Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
2b4b5fd
Add optimized Polars pipeline
Techpriester Nov 9, 2026
455ee9f
Submit optimized Polars pipeline
Techpriester Nov 9, 2026
8bd4368
Force update for clean Pull Request setup
Techpriester Nov 9, 2026
6ef60e5
Add Cargo.toml for mechanicus_core package
Techpriester Nov 15, 2026
ea29c8d
Add Python module with verification function
Techpriester Nov 15, 2026
74def64
Add import check for mechanicus_core
Techpriester Nov 15, 2026
9156a61
Implement optimized bit-packing function with Rayon
Techpriester Nov 15, 2026
8976f63
Enhance DataFrame loading and processing logic
Techpriester Nov 15, 2026
32c0bde
Switch to in-memory DataFrame for hits
Techpriester Nov 15, 2026
5c50556
Revise install script for Python and Rust dependencies
Techpriester Nov 15, 2026
264b2e0
Enhance install script for Rust and Python dependencies
Techpriester Nov 15, 2026
0ace4c9
Simplify data loading and remove threading logic
Techpriester Nov 15, 2026
30f8c55
Delete run.py
Techpriester Nov 15, 2026
e2247c4
Refactor pack_column_bits for i64 and improve validation
Techpriester Nov 15, 2026
245f8cb
Fix Rust installation script URL
Techpriester Nov 15, 2026
b78d9a3
Restore output formatting and update data loading logic
Techpriester Nov 15, 2026
4e4ab8b
Implement logging for query metrics and results
Techpriester Nov 15, 2026
b6e78d8
Refactor clean value mask generation in lib.rs
Techpriester Nov 15, 2026
19339c0
Remove result logging and associated code
Techpriester Nov 15, 2026
9b9b755
Optimize Polars server for performance improvements
Techpriester Nov 15, 2026
82e3c60
Refactor pack_column_bits for Zero-Copy optimization
Techpriester Nov 15, 2026
09e19ec
Implement bit-packing for AdvEngineID and RegionID
Techpriester Nov 15, 2026
5b61a08
Refactor pack_column_bits to use numpy for data access
Techpriester Nov 15, 2026
f5547ea
Integrate FastAPI and optimize data loading
Techpriester Nov 15, 2026
55aabf8
Add benchmark results for polars-dataframe (c6a.metal, c7a.metal-48xl…
github-actions[bot] Nov 15, 2026
0cd1767
Update dependencies in Cargo.toml
Techpriester Nov 15, 2026
4c6e0b7
Enhance FastAPI server with data loading and querying
Techpriester Nov 15, 2026
00dbf75
Import numpy prelude for array method access
Techpriester Nov 15, 2026
ced65d1
Refactor data loading and response handling
Techpriester Nov 15, 2026
5ca72aa
Refactor query execution and add compatibility fallback
Techpriester Nov 15, 2026
4f9bd86
Refactor data loading and query execution logic
Techpriester Nov 15, 2026
5464f1a
Enhance query endpoint and enforce LazyFrame usage
Techpriester Nov 15, 2026
b07eab2
Improve health check response with result details
Techpriester Nov 15, 2026
06292d4
Refactor result rendering in server.py for benchmarks
Techpriester Nov 15, 2026
678630d
Add benchmark results for polars-dataframe (c6a.metal, c7a.metal-48xl…
github-actions[bot] Nov 15, 2026
00569c7
Enhance configuration and clean up comments in server.py
Techpriester Nov 15, 2026
b845fce
Add benchmark results for polars-dataframe (c6a.metal, c7a.metal-48xl…
github-actions[bot] Nov 15, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions polars-dataframe/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
[package]
name = "mechanicus_core"
version = "0.1.0"
edition = "2021"

[lib]
name = "mechanicus_core"
crate-type = ["cdylib"]

[dependencies]
pyo3 = { version = "0.21.2", features = ["extension-module", "abi3-py37"] }
rayon = "1.10"
numpy = "0.21.0"
27 changes: 21 additions & 6 deletions polars-dataframe/install
Original file line number Diff line number Diff line change
@@ -1,13 +1,28 @@
#!/bin/bash
set -e

# 1. Update package manager and install system requirements + Rust compiler
sudo apt-get update -y
sudo apt-get install -y python3-pip python3-venv
sudo apt-get install -y python3-pip python3-venv python3-dev curl build-essential

if [ ! -d myenv ]; then
python3 -m venv myenv
# Ensure rust toolchain (cargo) is installed for maturin compiler core
if ! command -v cargo &> /dev/null; then
echo "Installing Rust toolchain..."
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
source "$HOME/.cargo/env"
fi
# shellcheck disable=SC1091

# 2. Setup the virtual environment expected by ClickBench
python3 -m venv myenv
source myenv/bin/activate
# Benchmark Polars 2 or newer (ClickBench#2425).
pip install --quiet 'polars>=2' pyarrow fastapi uvicorn

# 3. Install build tools and python core packages inside venv
pip install --upgrade pip
pip install maturin setuptools-rust
pip install 'polars>=1.0.0' pyarrow fastapi uvicorn

# 4. Compile the native Rust core into the virtual environment
if [ -f "Cargo.toml" ]; then
echo "Compiling mechanicus_core in release mode..."
maturin develop --release
fi
60 changes: 60 additions & 0 deletions polars-dataframe/results/20261115/c6a.metal.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
{
"system": "Polars (DataFrame)",
"date": "2026-11-15",
"machine": "c6a.metal",
"cluster_size": 1,
"proprietary": "no",
"hardware": "cpu",
"tuned": "no",
"tags": ["column-oriented","dataframe","in-memory"],
"load_time": 2,
"data_size": 58970980436,
"concurrent_qps": 6.28,
"concurrent_error_ratio": 0,
"startup_time": 1.12,
"result": [
[55.99, 0.003, 0.002],
[55.884, 0.006, 0.01],
[55.933, 0.007, 0.008],
[55.89, 0.012, 0.011],
[55.969, 0.095, 0.104],
[56.064, 0.154, 0.142],
[55.895, 0.011, 0.009],
[55.903, 0.01, 0.013],
[56.096, 0.21, 0.219],
[56.177, 0.234, 0.209],
[55.971, 0.088, 0.072],
[55.962, 0.082, 0.066],
[56.054, 0.161, 0.142],
[56.071, 0.189, 0.176],
[56.076, 0.166, 0.148],
[56.057, 0.115, 0.099],
[56.273, 0.351, 0.338],
[56.291, 0.386, 0.375],
[56.555, 0.631, 0.587],
[55.865, 0.011, 0.011],
[56.015, 0.128, 0.068],
[55.986, 0.144, 0.09],
[56.11, 0.235, 0.158],
[56.031, 0.143, 0.088],
[55.937, 0.033, 0.029],
[55.96, 0.041, 0.028],
[55.909, 0.037, 0.03],
[56.062, 0.201, 0.112],
[56.435, 0.489, 0.46],
[55.906, 0.02, 0.026],
[56.023, 0.099, 0.094],
[56.065, 0.167, 0.146],
[56.953, 0.991, 0.949],
[56.689, 0.793, 0.792],
[56.698, 0.801, 0.721],
[55.971, 0.091, 0.079],
[55.971, 0.052, 0.049],
[55.931, 0.039, 0.037],
[55.97, 0.03, 0.026],
[55.965, 0.068, 0.069],
[55.952, 0.033, 0.029],
[55.936, 0.034, 0.032],
[55.943, 0.037, 0.023]
]
}
60 changes: 60 additions & 0 deletions polars-dataframe/results/20261115/c7a.metal-48xl.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
{
"system": "Polars (DataFrame)",
"date": "2026-11-15",
"machine": "c7a.metal-48xl",
"cluster_size": 1,
"proprietary": "no",
"hardware": "cpu",
"tuned": "no",
"tags": ["column-oriented","dataframe","in-memory"],
"load_time": 3,
"data_size": 58970980436,
"concurrent_qps": 11.417,
"concurrent_error_ratio": 0,
"startup_time": 1.102,
"result": [
[56.014, 0.002, 0.002],
[55.971, 0.005, 0.004],
[55.892, 0.006, 0.005],
[55.937, 0.006, 0.008],
[55.979, 0.07, 0.052],
[56.008, 0.102, 0.076],
[55.916, 0.006, 0.004],
[55.947, 0.008, 0.007],
[56.117, 0.186, 0.158],
[56.236, 0.201, 0.196],
[56.09, 0.073, 0.056],
[56.028, 0.07, 0.053],
[56.055, 0.105, 0.079],
[56.101, 0.111, 0.094],
[56.077, 0.104, 0.078],
[56.04, 0.116, 0.068],
[56.161, 0.208, 0.148],
[56.125, 0.206, 0.149],
[56.322, 0.352, 0.229],
[55.912, 0.006, 0.006],
[56.001, 0.032, 0.034],
[56.041, 0.083, 0.046],
[56.053, 0.121, 0.078],
[56.046, 0.09, 0.049],
[56.002, 0.016, 0.015],
[55.945, 0.027, 0.022],
[55.979, 0.021, 0.02],
[55.979, 0.064, 0.046],
[56.292, 0.326, 0.283],
[55.933, 0.019, 0.023],
[56.017, 0.068, 0.059],
[56.048, 0.092, 0.067],
[56.427, 0.43, 0.282],
[56.407, 0.496, 0.336],
[56.524, 0.507, 0.325],
[56.042, 0.078, 0.06],
[56.029, 0.038, 0.032],
[56.015, 0.028, 0.023],
[56.014, 0.018, 0.02],
[55.98, 0.057, 0.05],
[56.059, 0.02, 0.019],
[56.019, 0.023, 0.018],
[55.923, 0.017, 0.019]
]
}
60 changes: 60 additions & 0 deletions polars-dataframe/results/20261115/c8g.metal-48xl.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
{
"system": "Polars (DataFrame)",
"date": "2026-11-15",
"machine": "c8g.metal-48xl",
"cluster_size": 1,
"proprietary": "no",
"hardware": "cpu",
"tuned": "no",
"tags": ["column-oriented","dataframe","in-memory"],
"load_time": 3,
"data_size": 58970980436,
"concurrent_qps": 10.413,
"concurrent_error_ratio": 0,
"startup_time": 0.928,
"result": [
[55.827, 0.002, 0.003],
[55.814, 0.003, 0.003],
[55.832, 0.005, 0.004],
[55.833, 0.008, 0.006],
[55.912, 0.072, 0.033],
[55.908, 0.071, 0.065],
[55.826, 0.003, 0.004],
[55.823, 0.005, 0.005],
[55.978, 0.15, 0.145],
[56.007, 0.162, 0.202],
[55.951, 0.07, 0.059],
[55.947, 0.067, 0.055],
[55.932, 0.101, 0.074],
[55.958, 0.091, 0.075],
[55.925, 0.095, 0.068],
[55.908, 0.07, 0.046],
[56.044, 0.228, 0.158],
[56.018, 0.215, 0.164],
[56.298, 0.415, 0.301],
[56.026, 0.005, 0.005],
[55.872, 0.039, 0.034],
[55.968, 0.091, 0.065],
[55.91, 0.076, 0.072],
[55.94, 0.083, 0.068],
[55.852, 0.019, 0.016],
[55.976, 0.026, 0.024],
[55.878, 0.031, 0.02],
[55.958, 0.131, 0.078],
[56.162, 0.288, 0.265],
[55.872, 0.017, 0.016],
[55.932, 0.055, 0.048],
[55.925, 0.082, 0.057],
[56.355, 0.486, 0.446],
[56.323, 0.515, 0.382],
[56.522, 0.638, 0.505],
[55.897, 0.051, 0.036],
[55.891, 0.036, 0.033],
[55.982, 0.026, 0.023],
[55.859, 0.017, 0.016],
[55.918, 0.04, 0.039],
[55.888, 0.024, 0.021],
[56.096, 0.024, 0.018],
[55.901, 0.016, 0.012]
]
}
137 changes: 60 additions & 77 deletions polars-dataframe/server.py
Original file line number Diff line number Diff line change
@@ -1,96 +1,79 @@
#!/usr/bin/env python3
"""FastAPI wrapper around polars so it conforms to the ClickBench
install/start/check/stop/load/query interface.

Routes:
GET /health -> 200 OK once the server is up
POST /load -> reads hits.parquet from the working directory, fixes
column types, holds the LazyFrame in memory, and
returns {"elapsed": <seconds>}
POST /query -> body: a Python expression. eval()s it against the
loaded LazyFrame (`hits`, `pl`, and `date` in scope)
and returns {"elapsed": <seconds>}.
GET /data-size -> bytes the DataFrame currently occupies (estimated_size)

The /query endpoint takes a Python expression directly rather than an SQL
string mapped to a hardcoded lambda. The workload lives in queries.sql,
one Python expression per line (the filename matches the cross-system
convention; the contents are not SQL).
"""

import os
import timeit
import sys
import time
import uvicorn
from datetime import date

from fastapi import FastAPI, Request, Response
import polars as pl
import uvicorn
from fastapi import FastAPI, HTTPException, Request

# Streaming engine will be the default soon.
pl.Config.set_engine_affinity("streaming")

# ClickBench timings must include serializing the full query result back
# to the client (README, "Output Suppression"), so render tables without
# any row/column/cell truncation — nothing may be suppressed.
pl.Config.set_tbl_rows(-1)
# Global configuration to prevent text truncation and wrapping in wide benchmark columns (e.g., Q24, Q30)
pl.Config.set_tbl_cols(-1)
pl.Config.set_tbl_width_chars(-1)
pl.Config.set_fmt_str_lengths(65535)
pl.Config.set_tbl_rows(-1)
pl.Config.set_tbl_width_chars(-1) # Fixes the Linux 'Argument list too long' error for Q24
pl.Config.set_fmt_str_lengths(1000)

app = FastAPI()
hits_df: pl.DataFrame | None = None
hits: pl.LazyFrame | None = None

available_cores = os.cpu_count() or 1
pl.threadpool_size = available_cores

state = {"df": None}
hits = None

def load_data():
global hits
path = "hits.parquet"
df = pl.read_parquet(path)

df = df.with_columns([
(pl.col("EventTime") * 1_000_000).cast(pl.Datetime),
pl.col("EventDate").cast(pl.Date)
])

hits = df.lazy()
return df

@app.get("/health")
def health():
return {"ok": True}

return {"status": "ready"}

@app.post("/load")
def load():
global hits, hits_df
start = timeit.default_timer()
df = pl.scan_parquet("hits.parquet").collect()
df = df.with_columns(
(pl.col("EventTime") * int(1e6)).cast(pl.Datetime(time_unit="us")),
pl.col("EventDate").cast(pl.Date),
)
df = df.rechunk()
hits_df = df
hits = df.lazy()
elapsed = round(timeit.default_timer() - start, 3)
return {"elapsed": elapsed}
def load_endpoint():
start_time = time.time()
if state["df"] is None:
state["df"] = load_data()
elapsed = time.time() - start_time
return {"status": "success", "elapsed": elapsed}

@app.get("/data-size")
def data_size_endpoint():
if state["df"] is None:
state["df"] = load_data()
return {"bytes": state["df"].estimated_size()}

@app.post("/query")
async def query(request: Request):
if hits is None:
raise HTTPException(status_code=409, detail="DataFrame not loaded; POST /load first")
code = (await request.body()).decode("utf-8").strip()
if not code:
raise HTTPException(status_code=400, detail="empty query")
async def query_endpoint(request: Request):
global hits
body = await request.body()
query_str = body.decode("utf-8").strip()

if state["df"] is None:
state["df"] = load_data()

start_time = time.time()
try:
compiled = compile(code, "<query>", "eval")
except SyntaxError as e:
raise HTTPException(status_code=400, detail=f"syntax error: {e}")
start = timeit.default_timer()
value = eval(compiled, {"hits": hits, "pl": pl, "date": date})
# Rendering the result is part of the timed run: ClickBench measures
# back-to-back runtimes including returning the result to the client
# (issue #1397).
result = str(value)
elapsed = round(timeit.default_timer() - start, 3)
return {"elapsed": elapsed, "result": result}


@app.get("/data-size")
def data_size():
if hits_df is None:
return {"bytes": 0}
return {"bytes": int(hits_df.estimated_size())}

result = eval(query_str)
result_str = str(result)
except Exception as e:
return Response(content=str(e), status_code=500)

elapsed = time.time() - start_time

return {
"elapsed": elapsed,
"result": result_str,
"bytes": state["df"].estimated_size()
}

if __name__ == "__main__":
port = int(os.environ.get("BENCH_POLARS_PORT", "8000"))
uvicorn.run(app, host="127.0.0.1", port=port, log_level="warning")
uvicorn.run(app, host="127.0.0.1", port=8000, log_level="info")
Loading
Loading