bench: model remote object storage faithfully for --simulate-latency

benchmarks/Cargo.toml

@@ -67,6 +67,7 @@ toml = "1.1.3"
 [dev-dependencies]
 datafusion-proto = { workspace = true, features = ["parquet"] }
 tempfile = { workspace = true }
+tokio = { workspace = true, features = ["test-util"] }
 
 [[bench]]
 harness = false

benchmarks/bench.sh

@@ -188,7 +188,8 @@ CARGO_COMMAND       command that runs the benchmark binary
 DATAFUSION_DIR      directory to use (default $DATAFUSION_DIR)
 RESULTS_NAME        folder where the benchmark files are stored
 PREFER_HASH_JOIN    Prefer hash join algorithm (default true)
-SIMULATE_LATENCY    Simulate object store latency to mimic S3 (default false)
+SIMULATE_LATENCY    Serve local files as if they were remote object storage: per-request
+                    latency, per-connection bandwidth and paginated LIST (default false)
 DATAFUSION_*        Set the given datafusion configuration
 "
     exit 1

benchmarks/src/util/mod.rs

@@ -16,10 +16,10 @@
 // under the License.
 
 //! Shared benchmark utilities
-pub mod latency_object_store;
 mod memory;
 mod options;
 mod run;
+pub mod simulated_object_store;
 
 pub use memory::print_memory_stats;
 pub use options::CommonOpt;

benchmarks/src/util/options.rs

@@ -33,7 +33,9 @@ use datafusion::{
 use datafusion_common::{DataFusionError, Result};
 use object_store::local::LocalFileSystem;
 
-use super::latency_object_store::LatencyObjectStore;
+use super::simulated_object_store::{
+    BYTES_PER_SECOND, MAX_CONNECTIONS, SimulatedObjectStore,
+};
 
 // Common benchmark options (don't use doc comments otherwise this doc
 // shows up in help files)
@@ -69,8 +71,10 @@ pub struct CommonOpt {
     #[arg(short, long, env)]
     pub debug: bool,
 
-    /// Simulate object store latency to mimic remote storage (e.g. S3).
-    /// Adds random latency in the range 20-200ms to each object store operation.
+    /// Serve the local filesystem as if it were remote object storage (e.g. S3
+    /// or GCS): per-request latency, per-connection bandwidth, a bounded
+    /// connection pool, and paginated LIST. See
+    /// [`crate::util::simulated_object_store`].
     #[arg(long = "simulate-latency", env)]
     pub simulate_latency: bool,
 }
@@ -138,17 +142,18 @@ impl CommonOpt {
         Ok(rt_builder)
     }
 
-    /// Build the runtime environment, optionally wrapping the local filesystem
-    /// with a throttled object store to simulate remote storage latency.
+    /// Build the runtime environment, optionally presenting the local
+    /// filesystem as remote object storage.
     pub fn build_runtime(&self) -> Result<Arc<RuntimeEnv>> {
         let rt = self.runtime_env_builder()?.build_arc()?;
         if self.simulate_latency {
-            let store: Arc<dyn object_store::ObjectStore> =
-                Arc::new(LatencyObjectStore::new(LocalFileSystem::new()));
+            let store = SimulatedObjectStore::new(LocalFileSystem::new());
             let url = ObjectStoreUrl::parse("file:///")?;
-            rt.register_object_store(url.as_ref(), store);
+            rt.register_object_store(url.as_ref(), Arc::new(store));
             println!(
-                "Simulating S3-like object store latency (get: 25-200ms, list: 40-400ms)"
+                "Simulating remote object storage (get TTFB: 19-230ms, head TTFB: \
+                 8-28ms, list TTFB: 40-200ms/page, {} MB/s per connection, {MAX_CONNECTIONS} connections)",
+                BYTES_PER_SECOND / (1024 * 1024),
             );
         }
         Ok(rt)