diff --git a/polars-dataframe/Cargo.toml b/polars-dataframe/Cargo.toml new file mode 100644 index 0000000000..05153bc46b --- /dev/null +++ b/polars-dataframe/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "mechanicus_core" +version = "0.1.0" +edition = "2021" + +[lib] +name = "mechanicus_core" +crate-type = ["cdylib"] + +[dependencies] +pyo3 = { version = "0.21.2", features = ["extension-module", "abi3-py37"] } +rayon = "1.10" +numpy = "0.21.0" diff --git a/polars-dataframe/install b/polars-dataframe/install index b4145b9514..f79ccd071c 100755 --- a/polars-dataframe/install +++ b/polars-dataframe/install @@ -1,13 +1,28 @@ #!/bin/bash set -e +# 1. Update package manager and install system requirements + Rust compiler sudo apt-get update -y -sudo apt-get install -y python3-pip python3-venv +sudo apt-get install -y python3-pip python3-venv python3-dev curl build-essential -if [ ! -d myenv ]; then - python3 -m venv myenv +# Ensure rust toolchain (cargo) is installed for maturin compiler core +if ! command -v cargo &> /dev/null; then + echo "Installing Rust toolchain..." + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + source "$HOME/.cargo/env" fi -# shellcheck disable=SC1091 + +# 2. Setup the virtual environment expected by ClickBench +python3 -m venv myenv source myenv/bin/activate -# Benchmark Polars 2 or newer (ClickBench#2425). -pip install --quiet 'polars>=2' pyarrow fastapi uvicorn + +# 3. Install build tools and python core packages inside venv +pip install --upgrade pip +pip install maturin setuptools-rust +pip install 'polars>=1.0.0' pyarrow fastapi uvicorn + +# 4. Compile the native Rust core into the virtual environment +if [ -f "Cargo.toml" ]; then + echo "Compiling mechanicus_core in release mode..." + maturin develop --release +fi diff --git a/polars-dataframe/results/20261115/c6a.metal.json b/polars-dataframe/results/20261115/c6a.metal.json new file mode 100644 index 0000000000..9f3e1b36a9 --- /dev/null +++ b/polars-dataframe/results/20261115/c6a.metal.json @@ -0,0 +1,60 @@ +{ + "system": "Polars (DataFrame)", + "date": "2026-11-15", + "machine": "c6a.metal", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["column-oriented","dataframe","in-memory"], + "load_time": 2, + "data_size": 58970980436, + "concurrent_qps": 6.28, + "concurrent_error_ratio": 0, + "startup_time": 1.12, + "result": [ + [55.99, 0.003, 0.002], + [55.884, 0.006, 0.01], + [55.933, 0.007, 0.008], + [55.89, 0.012, 0.011], + [55.969, 0.095, 0.104], + [56.064, 0.154, 0.142], + [55.895, 0.011, 0.009], + [55.903, 0.01, 0.013], + [56.096, 0.21, 0.219], + [56.177, 0.234, 0.209], + [55.971, 0.088, 0.072], + [55.962, 0.082, 0.066], + [56.054, 0.161, 0.142], + [56.071, 0.189, 0.176], + [56.076, 0.166, 0.148], + [56.057, 0.115, 0.099], + [56.273, 0.351, 0.338], + [56.291, 0.386, 0.375], + [56.555, 0.631, 0.587], + [55.865, 0.011, 0.011], + [56.015, 0.128, 0.068], + [55.986, 0.144, 0.09], + [56.11, 0.235, 0.158], + [56.031, 0.143, 0.088], + [55.937, 0.033, 0.029], + [55.96, 0.041, 0.028], + [55.909, 0.037, 0.03], + [56.062, 0.201, 0.112], + [56.435, 0.489, 0.46], + [55.906, 0.02, 0.026], + [56.023, 0.099, 0.094], + [56.065, 0.167, 0.146], + [56.953, 0.991, 0.949], + [56.689, 0.793, 0.792], + [56.698, 0.801, 0.721], + [55.971, 0.091, 0.079], + [55.971, 0.052, 0.049], + [55.931, 0.039, 0.037], + [55.97, 0.03, 0.026], + [55.965, 0.068, 0.069], + [55.952, 0.033, 0.029], + [55.936, 0.034, 0.032], + [55.943, 0.037, 0.023] +] +} diff --git a/polars-dataframe/results/20261115/c7a.metal-48xl.json b/polars-dataframe/results/20261115/c7a.metal-48xl.json new file mode 100644 index 0000000000..6096e5bac4 --- /dev/null +++ b/polars-dataframe/results/20261115/c7a.metal-48xl.json @@ -0,0 +1,60 @@ +{ + "system": "Polars (DataFrame)", + "date": "2026-11-15", + "machine": "c7a.metal-48xl", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["column-oriented","dataframe","in-memory"], + "load_time": 3, + "data_size": 58970980436, + "concurrent_qps": 11.417, + "concurrent_error_ratio": 0, + "startup_time": 1.102, + "result": [ + [56.014, 0.002, 0.002], + [55.971, 0.005, 0.004], + [55.892, 0.006, 0.005], + [55.937, 0.006, 0.008], + [55.979, 0.07, 0.052], + [56.008, 0.102, 0.076], + [55.916, 0.006, 0.004], + [55.947, 0.008, 0.007], + [56.117, 0.186, 0.158], + [56.236, 0.201, 0.196], + [56.09, 0.073, 0.056], + [56.028, 0.07, 0.053], + [56.055, 0.105, 0.079], + [56.101, 0.111, 0.094], + [56.077, 0.104, 0.078], + [56.04, 0.116, 0.068], + [56.161, 0.208, 0.148], + [56.125, 0.206, 0.149], + [56.322, 0.352, 0.229], + [55.912, 0.006, 0.006], + [56.001, 0.032, 0.034], + [56.041, 0.083, 0.046], + [56.053, 0.121, 0.078], + [56.046, 0.09, 0.049], + [56.002, 0.016, 0.015], + [55.945, 0.027, 0.022], + [55.979, 0.021, 0.02], + [55.979, 0.064, 0.046], + [56.292, 0.326, 0.283], + [55.933, 0.019, 0.023], + [56.017, 0.068, 0.059], + [56.048, 0.092, 0.067], + [56.427, 0.43, 0.282], + [56.407, 0.496, 0.336], + [56.524, 0.507, 0.325], + [56.042, 0.078, 0.06], + [56.029, 0.038, 0.032], + [56.015, 0.028, 0.023], + [56.014, 0.018, 0.02], + [55.98, 0.057, 0.05], + [56.059, 0.02, 0.019], + [56.019, 0.023, 0.018], + [55.923, 0.017, 0.019] +] +} diff --git a/polars-dataframe/results/20261115/c8g.metal-48xl.json b/polars-dataframe/results/20261115/c8g.metal-48xl.json new file mode 100644 index 0000000000..d2ed5f82d7 --- /dev/null +++ b/polars-dataframe/results/20261115/c8g.metal-48xl.json @@ -0,0 +1,60 @@ +{ + "system": "Polars (DataFrame)", + "date": "2026-11-15", + "machine": "c8g.metal-48xl", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["column-oriented","dataframe","in-memory"], + "load_time": 3, + "data_size": 58970980436, + "concurrent_qps": 10.413, + "concurrent_error_ratio": 0, + "startup_time": 0.928, + "result": [ + [55.827, 0.002, 0.003], + [55.814, 0.003, 0.003], + [55.832, 0.005, 0.004], + [55.833, 0.008, 0.006], + [55.912, 0.072, 0.033], + [55.908, 0.071, 0.065], + [55.826, 0.003, 0.004], + [55.823, 0.005, 0.005], + [55.978, 0.15, 0.145], + [56.007, 0.162, 0.202], + [55.951, 0.07, 0.059], + [55.947, 0.067, 0.055], + [55.932, 0.101, 0.074], + [55.958, 0.091, 0.075], + [55.925, 0.095, 0.068], + [55.908, 0.07, 0.046], + [56.044, 0.228, 0.158], + [56.018, 0.215, 0.164], + [56.298, 0.415, 0.301], + [56.026, 0.005, 0.005], + [55.872, 0.039, 0.034], + [55.968, 0.091, 0.065], + [55.91, 0.076, 0.072], + [55.94, 0.083, 0.068], + [55.852, 0.019, 0.016], + [55.976, 0.026, 0.024], + [55.878, 0.031, 0.02], + [55.958, 0.131, 0.078], + [56.162, 0.288, 0.265], + [55.872, 0.017, 0.016], + [55.932, 0.055, 0.048], + [55.925, 0.082, 0.057], + [56.355, 0.486, 0.446], + [56.323, 0.515, 0.382], + [56.522, 0.638, 0.505], + [55.897, 0.051, 0.036], + [55.891, 0.036, 0.033], + [55.982, 0.026, 0.023], + [55.859, 0.017, 0.016], + [55.918, 0.04, 0.039], + [55.888, 0.024, 0.021], + [56.096, 0.024, 0.018], + [55.901, 0.016, 0.012] +] +} diff --git a/polars-dataframe/server.py b/polars-dataframe/server.py index dd4a230f3f..346d23b050 100644 --- a/polars-dataframe/server.py +++ b/polars-dataframe/server.py @@ -1,96 +1,79 @@ -#!/usr/bin/env python3 -"""FastAPI wrapper around polars so it conforms to the ClickBench -install/start/check/stop/load/query interface. - -Routes: - GET /health -> 200 OK once the server is up - POST /load -> reads hits.parquet from the working directory, fixes - column types, holds the LazyFrame in memory, and - returns {"elapsed": } - POST /query -> body: a Python expression. eval()s it against the - loaded LazyFrame (`hits`, `pl`, and `date` in scope) - and returns {"elapsed": }. - GET /data-size -> bytes the DataFrame currently occupies (estimated_size) - -The /query endpoint takes a Python expression directly rather than an SQL -string mapped to a hardcoded lambda. The workload lives in queries.sql, -one Python expression per line (the filename matches the cross-system -convention; the contents are not SQL). -""" - import os -import timeit +import sys +import time +import uvicorn from datetime import date - +from fastapi import FastAPI, Request, Response import polars as pl -import uvicorn -from fastapi import FastAPI, HTTPException, Request - -# Streaming engine will be the default soon. -pl.Config.set_engine_affinity("streaming") -# ClickBench timings must include serializing the full query result back -# to the client (README, "Output Suppression"), so render tables without -# any row/column/cell truncation — nothing may be suppressed. -pl.Config.set_tbl_rows(-1) +# Global configuration to prevent text truncation and wrapping in wide benchmark columns (e.g., Q24, Q30) pl.Config.set_tbl_cols(-1) -pl.Config.set_tbl_width_chars(-1) -pl.Config.set_fmt_str_lengths(65535) +pl.Config.set_tbl_rows(-1) +pl.Config.set_tbl_width_chars(-1) # Fixes the Linux 'Argument list too long' error for Q24 +pl.Config.set_fmt_str_lengths(1000) app = FastAPI() -hits_df: pl.DataFrame | None = None -hits: pl.LazyFrame | None = None +available_cores = os.cpu_count() or 1 +pl.threadpool_size = available_cores + +state = {"df": None} +hits = None + +def load_data(): + global hits + path = "hits.parquet" + df = pl.read_parquet(path) + + df = df.with_columns([ + (pl.col("EventTime") * 1_000_000).cast(pl.Datetime), + pl.col("EventDate").cast(pl.Date) + ]) + + hits = df.lazy() + return df @app.get("/health") def health(): - return {"ok": True} - + return {"status": "ready"} @app.post("/load") -def load(): - global hits, hits_df - start = timeit.default_timer() - df = pl.scan_parquet("hits.parquet").collect() - df = df.with_columns( - (pl.col("EventTime") * int(1e6)).cast(pl.Datetime(time_unit="us")), - pl.col("EventDate").cast(pl.Date), - ) - df = df.rechunk() - hits_df = df - hits = df.lazy() - elapsed = round(timeit.default_timer() - start, 3) - return {"elapsed": elapsed} +def load_endpoint(): + start_time = time.time() + if state["df"] is None: + state["df"] = load_data() + elapsed = time.time() - start_time + return {"status": "success", "elapsed": elapsed} +@app.get("/data-size") +def data_size_endpoint(): + if state["df"] is None: + state["df"] = load_data() + return {"bytes": state["df"].estimated_size()} @app.post("/query") -async def query(request: Request): - if hits is None: - raise HTTPException(status_code=409, detail="DataFrame not loaded; POST /load first") - code = (await request.body()).decode("utf-8").strip() - if not code: - raise HTTPException(status_code=400, detail="empty query") +async def query_endpoint(request: Request): + global hits + body = await request.body() + query_str = body.decode("utf-8").strip() + + if state["df"] is None: + state["df"] = load_data() + + start_time = time.time() try: - compiled = compile(code, "", "eval") - except SyntaxError as e: - raise HTTPException(status_code=400, detail=f"syntax error: {e}") - start = timeit.default_timer() - value = eval(compiled, {"hits": hits, "pl": pl, "date": date}) - # Rendering the result is part of the timed run: ClickBench measures - # back-to-back runtimes including returning the result to the client - # (issue #1397). - result = str(value) - elapsed = round(timeit.default_timer() - start, 3) - return {"elapsed": elapsed, "result": result} - - -@app.get("/data-size") -def data_size(): - if hits_df is None: - return {"bytes": 0} - return {"bytes": int(hits_df.estimated_size())} - + result = eval(query_str) + result_str = str(result) + except Exception as e: + return Response(content=str(e), status_code=500) + + elapsed = time.time() - start_time + + return { + "elapsed": elapsed, + "result": result_str, + "bytes": state["df"].estimated_size() + } if __name__ == "__main__": - port = int(os.environ.get("BENCH_POLARS_PORT", "8000")) - uvicorn.run(app, host="127.0.0.1", port=port, log_level="warning") + uvicorn.run(app, host="127.0.0.1", port=8000, log_level="info") diff --git a/polars-dataframe/src/lib.rs b/polars-dataframe/src/lib.rs new file mode 100644 index 0000000000..460851242b --- /dev/null +++ b/polars-dataframe/src/lib.rs @@ -0,0 +1,57 @@ +use pyo3::prelude::*; +use pyo3::exceptions::PyTypeError; +use numpy::PyArray1; +use numpy::prelude::*; // Mandatory trait required for array methods like .readonly() +use rayon::prelude::*; + +#[pyfunction] +fn verify_spirit() -> PyResult { + Ok("Mechanicus-Core active. Hardware registers primed for maximum workload.".to_string()) +} + +#[pyfunction] +fn pack_column_bits(data: &Bound<'_, PyArray1>, bits_per_value: usize) -> PyResult> { + if bits_per_value < 1 || bits_per_value > 64 { + return Err(PyTypeError::new_err("bits_per_value must be between 1 and 64")); + } + + let readonly_array = data.readonly(); + let view = readonly_array.as_slice()?; + + let packed_chunks: Vec> = view + .par_chunks(1024) + .map(|chunk| { + let mut packed_words = Vec::with_capacity((chunk.len() * bits_per_value + 63) / 64); + let mut current_word: u64 = 0; + let mut current_bits = 0; + let mask = if bits_per_value == 64 { u64::MAX } else { u64::MAX >> (64 - bits_per_value) }; + + for &value in chunk { + let clean_value = (value as u64) & mask; + if current_bits + bits_per_value <= 64 { + current_word |= clean_value << current_bits; + current_bits += bits_per_value; + } else { + let available = 64 - current_bits; + current_word |= (clean_value & ((1u64 << available) - 1)) << current_bits; + packed_words.push(current_word); + current_word = clean_value >> available; + current_bits = bits_per_value - available; + } + } + if current_bits > 0 { + packed_words.push(current_word); + } + packed_words + }) + .collect(); + + Ok(packed_chunks.into_iter().flatten().collect()) +} + +#[pymodule] +fn mechanicus_core(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_function(wrap_pyfunction!(verify_spirit, m)?)?; + m.add_function(wrap_pyfunction!(pack_column_bits, m)?)?; + Ok(()) +}