diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 9ec587bd100ec5..f9b1a70245bc64 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1298,6 +1298,89 @@ DEFINE_mDouble(inverted_index_ram_buffer_size, "512"); // -1 indicates not working. // Normally we should not change this, it's useful for testing. DEFINE_mInt32(inverted_index_max_buffered_docs, "-1"); +// SNII phrase-bigram df-prune threshold: <0 auto (max(64, doc_count/10000)), +// 0 disable min-df pruning (full legacy layout ALSO needs +// snii_bigram_prune_max_df_ratio outside (0, 1)), >0 fixed min-df. +DEFINE_mInt32(snii_bigram_prune_min_df, "-1"); +// SNII phrase-bigram df-prune UPPER bound, as a fraction of the segment doc +// count (G15): pairs whose final df exceeds ratio * doc_count are pruned from +// the hidden bigram dict (their 2-term phrase queries fall back to the +// positions path, same as min-df pruning); only a ratio in (0, 1) arms the +// gate (<= 0 disables; >= 1 can never prune, so it resolves to no gate). +// 0.25 keeps the 20-25% df band on the bigram fastpath (bench: "united +// states" sits at 21.2% df in wikipedia; pruning it costs a 2.7x cold-phrase +// cliff, keeping the whole band costs +0.85% table size on textbench). +DEFINE_mDouble(snii_bigram_prune_max_df_ratio, "0.25"); +// Defer the SNII hidden phrase-bigram build to compaction (default ON since +// the 4-corpus write-parity campaign: import index CPU -3.5%..-35% at zero +// compaction overhead; fresh segments answer phrases through the identical- +// results positions fallback until compaction rebuilds the bigrams). +// Full behavioral contract documented at the DECLARE in config.h (single +// source of truth: deferral scope, capture-once semantics, segcompaction +// caveat, and the never-compacted perf-cliff disclosure). +DEFINE_mBool(snii_bigram_defer_build_to_compaction, "true"); +// DIAGNOSTIC (default off): force SNII inverted-index reads onto NO_CACHE +// (precise S3 range GETs) instead of the 1MiB FILE_BLOCK_CACHE. Per-open on the +// SNII reader only -- does NOT touch global enable_file_cache, so cloud mode +// still boots. Measures the block-cache read amplification's true cost. Warm +// queries lose the local cache under this flag, so it is a measurement knob, not +// a production setting. +DEFINE_mBool(inverted_index_read_bypass_file_cache, "false"); +// G16-c: whether plain positions-tier (non-scoring) SNII indexes lay out freq +// regions. Freq bytes serve ONLY BM25 scoring, which the Doris integration +// does not reach yet (scoring_query has no production caller), so the default +// drops them (textbench: -2.2 GB index). Scoring-config indexes always write +// freq regardless. Applies at segment build (write side only); existing +// segments keep whatever layout they were written with (self-describing). +DEFINE_mBool(snii_positions_index_write_freq, "false"); +// G16-h: zstd levels for the SNII dict-block compression and the .prx window +// auto mode. Level 9 (vs the historical 3) shrinks the two largest compressed +// sections -- textbench: index -457 MB (0.918x -> 0.891x V3) -- for an import +// CPU cost inside the run-to-run variance band; zstd decode speed does not +// depend on the level, and warm/cold benches measured no query change. +// Write side only; segments self-describe their compression. +// Default 3 since the all-level-3 evaluation (2026-07-11, 4 corpora): vs +// level 9 the settled index grows only +0.6%..+6.3% (whole table +// +0.3%..+1.9%) while import index CPU drops 17-24% and full-compaction CPU +// 8-24%; settled cold-query latency is unchanged (interleaved A/B). The +// delta+varint-encoded payloads are high-entropy, so level 9's extra search +// buys almost no ratio. Raise only for size-critical deployments. +DEFINE_mInt32(snii_dict_block_zstd_level, "3"); +DEFINE_mInt32(snii_prx_zstd_level, "3"); +// Patch C prx tiering: zstd level for the prx region of DIRECT-LOAD segments +// only (stream/broker load, see IndexColumnWriter::set_direct_load). Inert at +// the defaults (both levels 3); it exists for size-critical deployments that +// RAISE snii_prx_zstd_level (e.g. 9) and still want cheap loads: compaction +// rewrites every segment at snii_prx_zstd_level, so SETTLED data (and the +// cold-query path over it) is unaffected by the load tier -- measured -290s +// (httplogs) / -204s (agentlogs) of import index CPU at 3 vs 9. Same clamp +// [3, 19]. Read at index flush like snii_prx_zstd_level (a mid-load change +// lands on in-flight segments); the direct-load BIT itself is captured once. +DEFINE_mInt32(snii_prx_zstd_level_direct_load, "3"); +// G16-d: target SNII dict block size in bytes; 0 uses the format default +// (64 KiB). Larger blocks compress better under the per-block zstd (the dict +// is the dominant physical section on high-cardinality corpora) at the cost +// of a larger fetch+decompress unit per cold dict-block miss. Write side +// only; the block size is self-described by the on-disk directory. +DEFINE_mInt32(snii_target_dict_block_bytes, "0"); +// SNII per-writer bigram intern-vocabulary cap (bytes): df==1 bigram terms are +// incrementally evicted (and bloom-recorded for the flush-time drop) once the +// live bigram intern storage crosses this. 0 = uncapped. Effective only when +// snii_bigram_prune_min_df != 0. Default 512 MiB. +DEFINE_mInt64(snii_bigram_vocab_cap_bytes, "536870912"); +// Process-wide SNII index-build RAM budget across ALL live segment writers +// (G09); the largest writers are asked to spill early once the sum crosses it. +// 0 disables. Default 8 GiB. +DEFINE_mInt64(snii_index_writer_global_memory_bytes, "0"); +// Minimum reclaimable posting-arena bytes before a G09 forced spill is honored +// (and before a writer is eligible as a spill victim): forced spills reclaim +// ONLY the arena, so smaller triggers cut tiny runs for near-zero relief. +// Default 64 MiB. +DEFINE_mInt64(snii_forced_spill_min_arena_bytes, "67108864"); +// Max spill-run files one SNII writer accumulates before its runs are +// merge-compacted into one (bounds the k-way merge fan-in and its open fds; +// every run is held open for the whole merge). 0 = uncapped. Default 64. +DEFINE_mInt32(snii_spill_max_run_files_per_buffer, "64"); // dict path for chinese analyzer DEFINE_String(inverted_index_dict_path, "${DORIS_HOME}/dict"); DEFINE_Int32(inverted_index_read_buffer_size, "4096"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 012f09a735c0cd..baab52d516e964 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1342,6 +1342,130 @@ DECLARE_Int32(ann_index_result_cache_stale_sweep_time_sec); // inverted index DECLARE_mDouble(inverted_index_ram_buffer_size); DECLARE_mInt32(inverted_index_max_buffered_docs); +// SNII phrase-bigram df-prune threshold (G01 "bigram diet"). Hidden adjacent-pair +// bigram terms whose final segment df is below the effective threshold are not +// materialized (their 2-term phrases fall back to positions verification), and +// surviving bigram postings are written docs+freq only (no positions). +// < 0 : auto -- max(64, segment_doc_count / 10000) per segment (default) +// = 0 : disable MIN-df pruning. NOTE (G15): the full legacy layout (every +// bigram materialized with positions, dict miss == "no adjacency") +// additionally requires snii_bigram_prune_max_df_ratio to resolve to +// no gate; with the default ratio the upper gate stays active. +// > 0 : use this fixed min-df threshold +DECLARE_mInt32(snii_bigram_prune_min_df); +// SNII phrase-bigram df-prune UPPER bound (G15), as a fraction of the segment +// doc count. A hidden adjacent-pair bigram term whose FINAL segment df exceeds +// ratio * segment_doc_count is pruned from the bigram dict at flush -- +// near-ubiquitous (stopword-like) pairs pay dict + posting bytes for almost no +// selectivity -- and its 2-term phrase queries fall back to the generic +// positions-verification path, exactly like min-df-pruned pairs (the reader +// falls back on any bigram dict miss once the meta declares either gate). The +// resolved absolute threshold is floored at 2 * effective min-df when the +// min-df gate is active, so the two gates never overlap; it is applied only at +// flush, where the final df and final doc count are both known (a partial +// mid-feed df can never prove the final df will exceed a doc-count-scaled +// bound). Only a ratio in (0, 1) arms the gate: <= 0 disables it, and >= 1 +// resolves to no gate too (df never exceeds the doc count, so such a bound +// could never prune -- recording it would only arm pointless dict-miss +// fallbacks). +DECLARE_mDouble(snii_bigram_prune_max_df_ratio); +// Defer the SNII hidden phrase-bigram build to compaction. When true, a direct +// non-ARRAY load (stream/broker load, DataWriteType::TYPE_DIRECT) skips feeding +// the hidden adjacent-pair bigram tokens AND the bigram sentinel term entirely +// -- measured at ~3/4 of the SNII import-index CPU gap vs V3 -- and persists a +// resident capability flag, so phrase queries on such a fresh segment skip +// impossible pair probes before taking the full-fidelity positions-verification +// fallback (results are identical, only slower) until compaction rewrites the +// segment. The omitted sentinel retains the existing fallback for older readers. +// ARRAY indexes keep the full build to preserve existing SNII full-build phrase +// behavior at element boundaries. +// Compaction re-feeds every token through the same writer with a +// non-TYPE_DIRECT write type, so it rebuilds the bigram index with zero extra +// mechanism; schema change and ADD INDEX builds likewise keep the full build. +// Captured once per index writer when the segment writer marks it direct-load +// (before any row is fed); a mid-load flip does not change an in-flight +// segment. NOTE: local-mode segcompaction (enable_segcompaction) rewrites +// merged segments as TYPE_COMPACTION during the load itself, so those segments +// still build the full bigram inside the load window -- the deferral saving +// applies only to segments that are not segcompacted; cloud mode has no +// segcompaction, so the saving is complete there. +// PERF-CLIFF DISCLOSURE (default is ON): compaction is the ONLY mechanism +// that rebuilds a deferred segment's bigrams, and nothing guarantees it ever +// runs -- a load-once-then-cold tablet (single rowset under the cumulative +// thresholds, compaction disabled, or policies that skip a lone rowset) keeps +// its phrase queries on the positions-verification path indefinitely (results +// identical, slower), and BUILD INDEX cannot rebuild an already-existing +// index. DISABLE this on deployments whose compaction policy cannot cover +// lone-load rowsets (until the deferred-rowset compaction backstop lands). +DECLARE_mBool(snii_bigram_defer_build_to_compaction); +// DIAGNOSTIC: force SNII inverted-index reads to bypass the 1MiB FILE_BLOCK_CACHE +// and issue precise S3 range GETs (NO_CACHE) instead. Applies ONLY to the SNII +// index file reader (per-open cache_type), NOT the global enable_file_cache, so +// cloud mode does not FATAL. Default false (keep block cache). Used to measure +// whether the block-cache read amplification hurts cold reads at the cost of +// re-reading S3 on every (warm) query. Not for production: warm loses the local +// cache. Read-side only. +DECLARE_mBool(inverted_index_read_bypass_file_cache); +// G16-c: whether plain positions-tier (non-scoring) SNII indexes lay out freq +// regions. Freq serves ONLY BM25 scoring (no production caller yet), so the +// default (false) drops the layout; scoring-config indexes always keep freq. +// Write-side only; segments are self-describing either way. +DECLARE_mBool(snii_positions_index_write_freq); +// G16-h: zstd levels for SNII dict blocks / prx windows. Default 3 (the +// all-level-3 evaluation showed level 9 buys <=6.3% index size for 17-24% +// import CPU; see the DEFINEs in config.cpp). +DECLARE_mInt32(snii_dict_block_zstd_level); +DECLARE_mInt32(snii_prx_zstd_level); +// Patch C: prx zstd level for DIRECT-LOAD segments only (default 3, cheaper +// import); compaction rewrites at snii_prx_zstd_level so settled segments are +// unaffected. Full contract at the DEFINE in config.cpp. +DECLARE_mInt32(snii_prx_zstd_level_direct_load); +// G16-d: target SNII dict block size in bytes; 0 = format default (64 KiB). +// Bigger blocks -> better per-block zstd on the dict region, larger cold +// fetch+decompress unit per dict-block miss. Write side only. +DECLARE_mInt32(snii_target_dict_block_bytes); +// SNII phrase-bigram vocabulary cap in bytes, PER index writer (G04 "bigram +// diet" phase 2). The SPIMI intern table otherwise accumulates every distinct +// hidden bigram string for the whole segment (the dominant import RSS +// overshoot on long-document corpora). When the live bigram intern storage +// (strings + fixed per-term overhead) crosses this cap, bigram terms whose +// current df == 1 (the Zipf long tail) are incrementally EVICTED from the +// intern table and postings buffer and recorded in an ever-dropped bloom; at +// flush any bigram in the bloom is dropped IN ADDITION to the df threshold +// (safe: a dropped bigram's 2-term phrase falls back to positions +// verification per the G01 reader contract). Only takes effect when bigram +// pruning is enabled (snii_bigram_prune_min_df != 0); 0 disables the cap +// (unbounded vocabulary, pre-G04 behavior). +DECLARE_mInt64(snii_bigram_vocab_cap_bytes); +// PROCESS-WIDE budget in bytes for SNII index-build RAM, summed across every +// live SNII segment writer of the BE (all tablets x all concurrent loads; G09). +// The per-writer cap (inverted_index_ram_buffer_size) bounds ONE writer, but a +// concurrent load keeps (tablets x concurrency) writers alive at once, none of +// which may ever reach its own cap -- their SUM is what this bounds. When the +// registered total exceeds the budget, the LARGEST writers are asked to spill +// their posting buffers to disk early (async-safe advisory requests, honored +// on each writer's own thread; output stays byte-identical). 0 disables the +// global limiter (per-writer spilling only). Checked at index-writer creation: +// a change applies to writers created afterwards. +DECLARE_mInt64(snii_index_writer_global_memory_bytes); +// G09 forced-spill floor: minimum reclaimable posting-arena bytes a SNII +// writer must hold before a process-wide forced-spill request is honored, and +// before the global limiter selects it as a spill victim. A forced spill +// reclaims ONLY the posting arena -- the persistent vocab / pair-map +// structures survive it -- so honoring below a real floor degenerates into a +// storm of tiny runs whenever the budget is dominated by persistent bytes +// (each run then costs a file, a sort and a merge-fd for near-zero memory +// relief). The global budget therefore bounds SPILLABLE memory, not +// persistent memory. Default 64 MiB. +DECLARE_mInt64(snii_forced_spill_min_arena_bytes); +// G09 run-file cap: maximum spill-run files one SNII writer may accumulate; +// on the next spill past the cap, the existing runs are merge-compacted into +// a single run first (term stream unchanged). Bounds the final k-way merge's +// fan-in and, decisively, its simultaneously-open file descriptors -- every +// run of a buffer is reopened and held open for the whole merge, so unbounded +// run counts across ~100 concurrent writers can exhaust the BE nofile rlimit +// ("Too many open files" at run reopen). 0 disables the cap. Default 64. +DECLARE_mInt32(snii_spill_max_run_files_per_buffer); // dict path for chinese analyzer DECLARE_String(inverted_index_dict_path); DECLARE_Int32(inverted_index_read_buffer_size); diff --git a/be/src/common/status.h b/be/src/common/status.h index d29b66459a3832..37ff6917b4f567 100644 --- a/be/src/common/status.h +++ b/be/src/common/status.h @@ -299,6 +299,7 @@ namespace ErrorCode { E(INVERTED_INDEX_COMPACTION_ERROR, -6010, false); \ E(INVERTED_INDEX_ANALYZER_ERROR, -6011, false); \ E(INVERTED_INDEX_FILE_CORRUPTED, -6012, false); \ + E(INVERTED_INDEX_SNII_NOT_FOUND, -6013, false); \ E(KEY_NOT_FOUND, -7000, false); \ E(KEY_ALREADY_EXISTS, -7001, false); \ E(ENTRY_NOT_FOUND, -7002, false); \ diff --git a/be/src/exec/scan/olap_scanner.cpp b/be/src/exec/scan/olap_scanner.cpp index a9ef277c373002..cb42420e1ab687 100644 --- a/be/src/exec/scan/olap_scanner.cpp +++ b/be/src/exec/scan/olap_scanner.cpp @@ -153,7 +153,10 @@ static bool has_file_cache_statistics(const io::FileCacheStatistics& stats) { stats.inverted_index_bytes_read_from_remote != 0 || stats.inverted_index_bytes_read_from_peer != 0 || stats.inverted_index_local_io_timer != 0 || stats.inverted_index_remote_io_timer != 0 || - stats.inverted_index_peer_io_timer != 0 || stats.inverted_index_io_timer != 0; + stats.inverted_index_peer_io_timer != 0 || stats.inverted_index_io_timer != 0 || + stats.inverted_index_request_bytes != 0 || stats.inverted_index_read_bytes != 0 || + stats.inverted_index_range_read_count != 0 || + stats.inverted_index_serial_read_rounds != 0; } Status OlapScanner::_prepare_impl() { diff --git a/be/src/io/cache/block_file_cache_profile.cpp b/be/src/io/cache/block_file_cache_profile.cpp index babee6ba072c8a..57bf6f403d525d 100644 --- a/be/src/io/cache/block_file_cache_profile.cpp +++ b/be/src/io/cache/block_file_cache_profile.cpp @@ -17,6 +17,7 @@ #include "io/cache/block_file_cache_profile.h" +#include #include #include #include @@ -106,10 +107,16 @@ FileCacheStatistics diff_file_cache_statistics(const FileCacheStatistics& curren SUBTRACT_FIELD(inverted_index_bytes_read_from_local); SUBTRACT_FIELD(inverted_index_bytes_read_from_remote); SUBTRACT_FIELD(inverted_index_bytes_read_from_peer); + SUBTRACT_FIELD(inverted_index_remote_physical_read_bytes); + SUBTRACT_FIELD(inverted_index_bytes_write_into_cache); SUBTRACT_FIELD(inverted_index_local_io_timer); SUBTRACT_FIELD(inverted_index_remote_io_timer); SUBTRACT_FIELD(inverted_index_peer_io_timer); SUBTRACT_FIELD(inverted_index_io_timer); + SUBTRACT_FIELD(inverted_index_request_bytes); + SUBTRACT_FIELD(inverted_index_read_bytes); + SUBTRACT_FIELD(inverted_index_range_read_count); + SUBTRACT_FIELD(inverted_index_serial_read_rounds); SUBTRACT_FIELD(segment_footer_index_num_local_io_total); SUBTRACT_FIELD(segment_footer_index_num_remote_io_total); @@ -124,6 +131,7 @@ FileCacheStatistics diff_file_cache_statistics(const FileCacheStatistics& curren return diff; } +// NOLINTBEGIN(readability-function-size): Counter registration stays grouped by profile layout. FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile) : _profile(profile) { static const char* cache_profile = "FileCache"; ADD_TIMER_WITH_LEVEL(profile, cache_profile, 2); @@ -170,6 +178,10 @@ FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile) : _p profile, "InvertedIndexBytesScannedFromRemote", TUnit::BYTES, cache_profile, 1); inverted_index_bytes_scanned_from_peer = ADD_CHILD_COUNTER_WITH_LEVEL( profile, "InvertedIndexBytesScannedFromPeer", TUnit::BYTES, cache_profile, 1); + inverted_index_remote_physical_read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "InvertedIndexRemotePhysicalReadBytes", TUnit::BYTES, cache_profile, 1); + inverted_index_bytes_write_into_cache = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "InvertedIndexBytesWriteIntoCache", TUnit::BYTES, cache_profile, 1); inverted_index_local_io_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "InvertedIndexLocalIOUseTimer", cache_profile, 1); inverted_index_remote_io_timer = @@ -178,6 +190,14 @@ FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile) : _p ADD_CHILD_TIMER_WITH_LEVEL(profile, "InvertedIndexPeerIOUseTimer", cache_profile, 1); inverted_index_io_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "InvertedIndexIOTimer", cache_profile, 1); + inverted_index_request_bytes = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "InvertedIndexRequestBytes", TUnit::BYTES, cache_profile, 1); + inverted_index_read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "InvertedIndexReadBytes", + TUnit::BYTES, cache_profile, 1); + inverted_index_range_read_count = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "InvertedIndexRangeReadCount", TUnit::UNIT, cache_profile, 1); + inverted_index_serial_read_rounds = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "InvertedIndexSerialReadRounds", TUnit::UNIT, cache_profile, 1); segment_footer_index_num_local_io_total = ADD_CHILD_COUNTER_WITH_LEVEL( profile, "SegmentFooterIndexNumLocalIOTotal", TUnit::UNIT, cache_profile, 1); @@ -219,7 +239,9 @@ FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile) : _p peer_lazy_fetch_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "PeerLazyFetchTime", cache_profile, 1); } +// NOLINTEND(readability-function-size) +// NOLINTNEXTLINE(readability-function-size): Counter updates stay grouped by profile layout. void FileCacheProfileReporter::update(const FileCacheStatistics* statistics) const { COUNTER_UPDATE(num_local_io_total, statistics->num_local_io_total); COUNTER_UPDATE(num_remote_io_total, statistics->num_remote_io_total); @@ -251,10 +273,19 @@ void FileCacheProfileReporter::update(const FileCacheStatistics* statistics) con statistics->inverted_index_bytes_read_from_remote); COUNTER_UPDATE(inverted_index_bytes_scanned_from_peer, statistics->inverted_index_bytes_read_from_peer); + COUNTER_UPDATE(inverted_index_remote_physical_read_bytes, + statistics->inverted_index_remote_physical_read_bytes); + COUNTER_UPDATE(inverted_index_bytes_write_into_cache, + statistics->inverted_index_bytes_write_into_cache); COUNTER_UPDATE(inverted_index_local_io_timer, statistics->inverted_index_local_io_timer); COUNTER_UPDATE(inverted_index_remote_io_timer, statistics->inverted_index_remote_io_timer); COUNTER_UPDATE(inverted_index_peer_io_timer, statistics->inverted_index_peer_io_timer); COUNTER_UPDATE(inverted_index_io_timer, statistics->inverted_index_io_timer); + COUNTER_UPDATE(inverted_index_request_bytes, statistics->inverted_index_request_bytes); + COUNTER_UPDATE(inverted_index_read_bytes, statistics->inverted_index_read_bytes); + COUNTER_UPDATE(inverted_index_range_read_count, statistics->inverted_index_range_read_count); + COUNTER_UPDATE(inverted_index_serial_read_rounds, + statistics->inverted_index_serial_read_rounds); COUNTER_UPDATE(segment_footer_index_num_local_io_total, statistics->segment_footer_index_num_local_io_total); diff --git a/be/src/io/cache/block_file_cache_profile.h b/be/src/io/cache/block_file_cache_profile.h index b24cce69186ec4..d873748b943b1e 100644 --- a/be/src/io/cache/block_file_cache_profile.h +++ b/be/src/io/cache/block_file_cache_profile.h @@ -19,6 +19,7 @@ #include +#include #include #include #include @@ -60,7 +61,6 @@ class FileCacheMetrics { void register_entity(); void update_metrics_callback(); -private: std::mutex _mtx; // use shared_ptr for concurrent std::shared_ptr _statistics; @@ -96,10 +96,16 @@ struct FileCacheProfileReporter { RuntimeProfile::Counter* inverted_index_bytes_scanned_from_cache = nullptr; RuntimeProfile::Counter* inverted_index_bytes_scanned_from_remote = nullptr; RuntimeProfile::Counter* inverted_index_bytes_scanned_from_peer = nullptr; + RuntimeProfile::Counter* inverted_index_remote_physical_read_bytes = nullptr; + RuntimeProfile::Counter* inverted_index_bytes_write_into_cache = nullptr; RuntimeProfile::Counter* inverted_index_local_io_timer = nullptr; RuntimeProfile::Counter* inverted_index_remote_io_timer = nullptr; RuntimeProfile::Counter* inverted_index_peer_io_timer = nullptr; RuntimeProfile::Counter* inverted_index_io_timer = nullptr; + RuntimeProfile::Counter* inverted_index_request_bytes = nullptr; + RuntimeProfile::Counter* inverted_index_read_bytes = nullptr; + RuntimeProfile::Counter* inverted_index_range_read_count = nullptr; + RuntimeProfile::Counter* inverted_index_serial_read_rounds = nullptr; RuntimeProfile::Counter* segment_footer_index_num_local_io_total = nullptr; RuntimeProfile::Counter* segment_footer_index_num_remote_io_total = nullptr; diff --git a/be/src/io/cache/cached_remote_file_reader.cpp b/be/src/io/cache/cached_remote_file_reader.cpp index 8868969b233170..9525af9caf6abd 100644 --- a/be/src/io/cache/cached_remote_file_reader.cpp +++ b/be/src/io/cache/cached_remote_file_reader.cpp @@ -316,7 +316,11 @@ Status execute_s3_read(size_t empty_start, size_t& size, std::unique_ptr s3_read_counter << 1; SCOPED_RAW_TIMER(&stats.remote_read_timer); stats.from_peer_cache = false; - return remote_file_reader->read_at(empty_start, Slice(buffer.get(), size), &size, io_ctx); + auto st = remote_file_reader->read_at(empty_start, Slice(buffer.get(), size), &size, io_ctx); + if (st.ok()) { + stats.remote_physical_read_bytes += size; + } + return st; } CloudWarmUpManager& get_warm_up_manager() { @@ -1390,6 +1394,8 @@ void CachedRemoteFileReader::_update_stats(const ReadStatistics& read_stats, statis->inverted_index_bytes_read_from_remote, statis->inverted_index_bytes_read_from_peer, statis->inverted_index_local_io_timer, statis->inverted_index_remote_io_timer, statis->inverted_index_peer_io_timer); + statis->inverted_index_remote_physical_read_bytes += read_stats.remote_physical_read_bytes; + statis->inverted_index_bytes_write_into_cache += read_stats.bytes_write_into_file_cache; break; case FileCacheReadType::SEGMENT_FOOTER_INDEX: update_index_stats(statis->segment_footer_index_num_local_io_total, diff --git a/be/src/io/cache/file_cache_common.h b/be/src/io/cache/file_cache_common.h index 4c60d2e86f3e60..93c18db36276de 100644 --- a/be/src/io/cache/file_cache_common.h +++ b/be/src/io/cache/file_cache_common.h @@ -74,6 +74,7 @@ struct ReadStatistics { int64_t bytes_read_from_local = 0; int64_t bytes_read_from_remote = 0; int64_t bytes_read_from_peer = 0; + int64_t remote_physical_read_bytes = 0; int64_t bytes_write_into_file_cache = 0; int64_t remote_read_timer = 0; int64_t peer_read_timer = 0; diff --git a/be/src/io/io_common.h b/be/src/io/io_common.h index e686c23a313c2f..151d34a3a0351f 100644 --- a/be/src/io/io_common.h +++ b/be/src/io/io_common.h @@ -19,6 +19,9 @@ #include +#include +#include +#include #include #include @@ -78,10 +81,16 @@ struct FileCacheStatistics { int64_t inverted_index_bytes_read_from_local = 0; int64_t inverted_index_bytes_read_from_remote = 0; int64_t inverted_index_bytes_read_from_peer = 0; + int64_t inverted_index_remote_physical_read_bytes = 0; + int64_t inverted_index_bytes_write_into_cache = 0; int64_t inverted_index_local_io_timer = 0; int64_t inverted_index_remote_io_timer = 0; int64_t inverted_index_peer_io_timer = 0; int64_t inverted_index_io_timer = 0; + int64_t inverted_index_request_bytes = 0; + int64_t inverted_index_read_bytes = 0; + int64_t inverted_index_range_read_count = 0; + int64_t inverted_index_serial_read_rounds = 0; int64_t segment_footer_index_num_local_io_total = 0; int64_t segment_footer_index_num_remote_io_total = 0; diff --git a/be/src/storage/compaction/compaction.cpp b/be/src/storage/compaction/compaction.cpp index f27f55d309661d..a93a7b2cbe1f20 100644 --- a/be/src/storage/compaction/compaction.cpp +++ b/be/src/storage/compaction/compaction.cpp @@ -1222,6 +1222,12 @@ static bool check_rowset_has_inverted_index(const RowsetSharedPtr& src_rs, int32 } void Compaction::construct_index_compaction_columns(RowsetWriterContext& ctx) { + if (_cur_tablet_schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::SNII) { + LOG(INFO) << "tablet[" << _tablet->tablet_id() + << "] uses SNII inverted index storage format, skip CLucene index compaction"; + return; + } for (const auto& index : _cur_tablet_schema->inverted_indexes()) { auto col_unique_ids = index->col_unique_ids(); // check if column unique ids is empty to avoid crash diff --git a/be/src/storage/index/index_file_reader.cpp b/be/src/storage/index/index_file_reader.cpp index 348e1399421e5a..7859270f214015 100644 --- a/be/src/storage/index/index_file_reader.cpp +++ b/be/src/storage/index/index_file_reader.cpp @@ -20,8 +20,11 @@ #include #include +#include "common/cast_set.h" +#include "common/config.h" #include "storage/index/inverted/inverted_index_compound_reader.h" #include "storage/index/inverted/inverted_index_fs_directory.h" +#include "storage/index/snii/format/per_index_meta.h" #include "storage/tablet/tablet_schema.h" #include "util/debug_points.h" @@ -31,7 +34,9 @@ Status IndexFileReader::init(int32_t read_buffer_size, const io::IOContext* io_c std::unique_lock lock(_mutex); // Lock for writing if (!_inited) { _read_buffer_size = read_buffer_size; - if (_storage_format >= InvertedIndexStorageFormatPB::V2) { + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + RETURN_IF_ERROR(_init_snii(io_ctx)); + } else if (_storage_format >= InvertedIndexStorageFormatPB::V2) { RETURN_IF_ERROR(_init_from(read_buffer_size, io_ctx)); } _inited = true; @@ -136,7 +141,46 @@ Status IndexFileReader::_init_from(int32_t read_buffer_size, const io::IOContext return Status::OK(); } +Status IndexFileReader::_init_snii(const io::IOContext* io_ctx) { + auto index_file_full_path = InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix); + int64_t file_size = -1; + if (_idx_file_info.has_index_size()) { + file_size = _idx_file_info.index_size(); + } + file_size = file_size == 0 ? -1 : file_size; + + io::FileReaderOptions opts; + // DIAGNOSTIC: inverted_index_read_bypass_file_cache forces NO_CACHE (precise S3 + // range GETs) for the SNII index reader only, without touching the global + // enable_file_cache (cloud mode requires the latter). Default path keeps the + // block cache. + opts.cache_type = (config::enable_file_cache && !config::inverted_index_read_bypass_file_cache) + ? io::FileCachePolicy::FILE_BLOCK_CACHE + : io::FileCachePolicy::NO_CACHE; + opts.is_doris_table = true; + opts.file_size = file_size; + opts.tablet_id = _tablet_id; + io::FileReaderSPtr reader; + RETURN_IF_ERROR(_fs->open_file(index_file_full_path, &reader, &opts)); + _snii_file_reader = std::make_shared(std::move(reader)); + _snii_segment_reader = std::make_unique(); + io::IOContext meta_io_ctx; + if (io_ctx != nullptr) { + meta_io_ctx = *io_ctx; + } + meta_io_ctx.is_inverted_index = true; + meta_io_ctx.is_index_data = true; + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(&meta_io_ctx); + RETURN_IF_ERROR(doris::snii::reader::SniiSegmentReader::open(_snii_file_reader.get(), + _snii_segment_reader.get())); + return Status::OK(); +} + Result IndexFileReader::get_all_directories() { + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + return ResultError(Status::Error( + "SNII format does not expose CLucene directories")); + } InvertedIndexDirectoryMap res; std::shared_lock lock(_mutex); // Lock for reading for (auto& [index, _] : _indices_entries) { @@ -155,6 +199,11 @@ Result> IndexFileReader:: int64_t index_id, const std::string& index_suffix, const io::IOContext* io_ctx) const { std::unique_ptr compound_reader; + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + return ResultError(Status::Error( + "SNII format does not open CLucene compound readers")); + } + if (_storage_format == InvertedIndexStorageFormatPB::V1) { auto index_file_path = InvertedIndexDescriptor::get_index_file_path_v1( _index_path_prefix, index_id, index_suffix); @@ -229,6 +278,47 @@ Result> IndexFileReader:: return compound_reader; } +Result> IndexFileReader::open_snii_index( + const TabletIndex* index_meta, const io::IOContext* io_ctx) const { + DCHECK(_storage_format == InvertedIndexStorageFormatPB::SNII); + std::shared_lock lock(_mutex); + if (_snii_segment_reader == nullptr) { + return ResultError(Status::Error( + "SNII index file {} is not opened", + InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix))); + } + io::IOContext meta_io_ctx; + if (io_ctx != nullptr) { + meta_io_ctx = *io_ctx; + } + meta_io_ctx.is_inverted_index = true; + meta_io_ctx.is_index_data = true; + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(&meta_io_ctx); + + std::vector meta_bytes; + auto status = + _snii_segment_reader->read_index_meta(cast_set(index_meta->index_id()), + index_meta->get_index_suffix(), &meta_bytes); + auto doris_status = status; + if (!doris_status.ok()) { + return ResultError(doris_status); + } + doris::snii::format::PerIndexMetaReader meta; + status = doris::snii::format::PerIndexMetaReader::open(doris::snii::Slice(meta_bytes), &meta); + doris_status = status; + if (!doris_status.ok()) { + return ResultError(doris_status); + } + auto logical_reader = std::make_unique(); + status = _snii_segment_reader->open_index_from_meta(doris::snii::Slice(meta_bytes), + logical_reader.get()); + doris_status = status; + if (!doris_status.ok()) { + return ResultError(doris_status); + } + return logical_reader; +} + Result> IndexFileReader::open( const TabletIndex* index_meta, const io::IOContext* io_ctx) const { auto index_id = index_meta->index_id(); @@ -254,6 +344,19 @@ Status IndexFileReader::index_file_exist(const TabletIndex* index_meta, bool* re auto index_file_path = InvertedIndexDescriptor::get_index_file_path_v1( _index_path_prefix, index_meta->index_id(), index_meta->get_index_suffix()); return _fs->exists(index_file_path, res); + } else if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + auto index_file_path = InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix); + RETURN_IF_ERROR(_fs->exists(index_file_path, res)); + if (!*res) { + return Status::OK(); + } + std::shared_lock lock(_mutex); + if (_snii_segment_reader == nullptr) { + *res = false; + return Status::OK(); + } + return _snii_segment_reader->index_exists(cast_set(index_meta->index_id()), + index_meta->get_index_suffix(), res); } else { std::shared_lock lock(_mutex); // Lock for reading if (_stream == nullptr) { @@ -279,6 +382,30 @@ Status IndexFileReader::has_null(const TabletIndex* index_meta, bool* res) const *res = true; return Status::OK(); } + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + std::shared_lock lock(_mutex); + if (_snii_segment_reader == nullptr) { + return Status::Error( + "SNII index file {} is not opened", + InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix)); + } + io::IOContext meta_io_ctx; + meta_io_ctx.is_inverted_index = true; + meta_io_ctx.is_index_data = true; + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(&meta_io_ctx); + + std::vector meta_bytes; + auto status = + _snii_segment_reader->read_index_meta(cast_set(index_meta->index_id()), + index_meta->get_index_suffix(), &meta_bytes); + RETURN_IF_ERROR(status); + doris::snii::format::PerIndexMetaReader meta; + status = doris::snii::format::PerIndexMetaReader::open(doris::snii::Slice(meta_bytes), + &meta); + RETURN_IF_ERROR(status); + *res = meta.section_refs().null_bitmap.length > 0; + return Status::OK(); + } std::shared_lock lock(_mutex); // Lock for reading if (_stream == nullptr) { return Status::Error( diff --git a/be/src/storage/index/index_file_reader.h b/be/src/storage/index/index_file_reader.h index fb4ec2b9a62fe3..50619381e187a0 100644 --- a/be/src/storage/index/index_file_reader.h +++ b/be/src/storage/index/index_file_reader.h @@ -35,6 +35,9 @@ #include "io/fs/file_system.h" #include "storage/index/index_file_writer.h" #include "storage/index/inverted/inverted_index_desc.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/snii_doris_adapter.h" namespace doris { class TabletIndex; @@ -60,7 +63,7 @@ class IndexFileReader { : _fs(std::move(fs)), _index_path_prefix(std::move(index_path_prefix)), _storage_format(storage_format), - _idx_file_info(idx_file_info), + _idx_file_info(std::move(idx_file_info)), _tablet_id(tablet_id) {} virtual ~IndexFileReader() = default; @@ -68,6 +71,8 @@ class IndexFileReader { const io::IOContext* io_ctx = nullptr); MOCK_FUNCTION Result> open( const TabletIndex* index_meta, const io::IOContext* io_ctx = nullptr) const; + Result> open_snii_index( + const TabletIndex* index_meta, const io::IOContext* io_ctx = nullptr) const; void debug_file_entries(); std::string get_index_file_cache_key(const TabletIndex* index_meta) const; std::string get_index_file_path(const TabletIndex* index_meta) const; @@ -75,12 +80,19 @@ class IndexFileReader { Status has_null(const TabletIndex* index_meta, bool* res) const; Result get_all_directories(); // open file v2, init _stream - int64_t get_inverted_file_size() const { return _stream == nullptr ? 0 : _stream->length(); } + int64_t get_inverted_file_size() const { + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + return _snii_file_reader == nullptr ? 0 : _snii_file_reader->size(); + } + return _stream == nullptr ? 0 : _stream->length(); + } const std::string& get_index_path_prefix() const { return _index_path_prefix; } + InvertedIndexStorageFormatPB get_storage_format() const { return _storage_format; } friend IndexFileWriter; protected: Status _init_from(int32_t read_buffer_size, const io::IOContext* io_ctx); + Status _init_snii(const io::IOContext* io_ctx); Result> _open( int64_t index_id, const std::string& index_suffix, const io::IOContext* io_ctx = nullptr) const; @@ -88,6 +100,8 @@ class IndexFileReader { private: IndicesEntriesMap _indices_entries; std::unique_ptr _stream = nullptr; + std::shared_ptr _snii_file_reader; + std::unique_ptr _snii_segment_reader; const io::FileSystemSPtr _fs; std::string _index_path_prefix; int32_t _read_buffer_size = -1; @@ -99,4 +113,4 @@ class IndexFileReader { }; } // namespace segment_v2 -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/storage/index/index_file_writer.cpp b/be/src/storage/index/index_file_writer.cpp index afd09c84620bb5..14e38b2ca8f567 100644 --- a/be/src/storage/index/index_file_writer.cpp +++ b/be/src/storage/index/index_file_writer.cpp @@ -19,9 +19,12 @@ #include +#include #include #include +#include "common/cast_set.h" +#include "common/config.h" #include "common/status.h" #include "io/fs/packed_file_writer.h" #include "io/fs/s3_file_writer.h" @@ -34,10 +37,82 @@ #include "storage/index/inverted/inverted_index_desc.h" #include "storage/index/inverted/inverted_index_fs_directory.h" #include "storage/index/inverted/inverted_index_reader.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/snii_doris_adapter.h" #include "storage/tablet/tablet_schema.h" namespace doris::segment_v2 { +namespace { + +// Resolves the EFFECTIVE SNII phrase-bigram df-prune threshold for one segment +// index (G01): 0 for non-positional configs (no bigrams exist there) and when +// config::snii_bigram_prune_min_df == 0 (pruning disabled, legacy layout); the +// fixed config value when > 0; otherwise (< 0, the default) the auto formula +// max(64, doc_count / 10000). The resolved value is what the writer applies AND +// records in the per-index segment meta. +uint32_t snii_effective_bigram_prune_min_df(uint32_t doc_count, + doris::snii::format::IndexConfig index_config) { + if (!doris::snii::format::has_positions(index_config)) { + return 0; + } + const int32_t conf = config::snii_bigram_prune_min_df; + if (conf == 0) { + return 0; + } + if (conf > 0) { + return static_cast(conf); + } + return doris::snii::format::default_phrase_bigram_prune_min_df(doc_count); +} + +// Resolves the EFFECTIVE SNII phrase-bigram df-prune UPPER bound for one +// segment index (G15): 0 (no upper gate) for non-positional configs and when +// config::snii_bigram_prune_max_df_ratio is outside (0, 1); otherwise +// ratio * doc_count, floored at 2 * effective_min_df when the min-df gate is +// active so the two gates never overlap. A rounded-to-0 result (tiny segment, +// no min floor) stays 0: recording 0 means "gate inactive" to the reader, so +// pruning with it would break the dict-miss fallback contract. Resolved ONLY +// here, at flush, where the final doc count is known -- the bound scales with +// the doc count, so no mid-feed partial-df comparison against it is ever +// valid. +uint64_t snii_effective_bigram_prune_max_df(uint32_t doc_count, uint32_t effective_min_df, + doris::snii::format::IndexConfig index_config) { + if (!doris::snii::format::has_positions(index_config)) { + return 0; + } + const double ratio = config::snii_bigram_prune_max_df_ratio; + // Gate only for ratio in (0, 1): df can never exceed doc_count, so a + // ratio >= 1.0 is a provably-dead gate -- resolving it to 0 keeps the meta + // from arming the dict-miss fallback (turning fast "pair absent == empty" + // answers into pointless positions-verification fallbacks) for a bound + // that can never prune. The (0, 1) window also keeps ratio * doc_count + // below 2^32, so the uint64 cast below is always defined (an absurd ratio + // like 1e18, or NaN -- which fails the conjunction -- could otherwise + // overflow the cast into UB). + if (!(ratio > 0.0 && ratio < 1.0)) { + return 0; + } + auto max_df = static_cast(ratio * doc_count); + if (effective_min_df > 0) { + max_df = std::max(max_df, 2ULL * effective_min_df); + } + return max_df; +} + +} // namespace + +// Resolves whether one segment index lays out freq regions (G16-c). Freq +// serves ONLY BM25 scoring: a scoring config always keeps it; a plain +// positions config keeps it only when the escape-hatch config asks for the +// full T2 layout. NOT in the anonymous namespace on purpose -- the UT covers +// this production policy line directly (a flipped operator or inverted flag +// here would otherwise stay green: no BE test drives add_snii_index). +bool snii_effective_write_freq(doris::snii::format::IndexConfig index_config) { + return doris::snii::format::has_scoring(index_config) || + config::snii_positions_index_write_freq; +} + IndexFileWriter::IndexFileWriter(io::FileSystemSPtr fs, std::string index_path_prefix, std::string rowset_id, int64_t seg_id, InvertedIndexStorageFormatPB storage_format, @@ -56,7 +131,7 @@ IndexFileWriter::IndexFileWriter(io::FileSystemSPtr fs, std::string index_path_p _tmp_dir = tmp_file_dir.native(); if (_storage_format == InvertedIndexStorageFormatPB::V1) { _index_storage_format = std::make_unique(this); - } else { + } else if (_storage_format != InvertedIndexStorageFormatPB::SNII) { _index_storage_format = std::make_unique(this); } } @@ -84,6 +159,10 @@ Status IndexFileWriter::_insert_directory_into_map(int64_t index_id, } Result> IndexFileWriter::open(const TabletIndex* index_meta) { + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + return ResultError(Status::Error( + "SNII format does not open CLucene directories")); + } auto local_fs_index_path = InvertedIndexDescriptor::get_temporary_index_path( _tmp_dir, _rowset_id, _seg_id, index_meta->index_id(), index_meta->get_index_suffix()); auto dir = std::shared_ptr(DorisFSDirectoryFactory::getDirectory( @@ -97,6 +176,94 @@ Result> IndexFileWriter::open(const TabletInde return dir; } +Status IndexFileWriter::add_snii_index(const TabletIndex* index_meta, uint32_t doc_count, + std::vector null_docids, + doris::snii::writer::SpimiTermBuffer* const term_buffer, + doris::snii::format::IndexConfig index_config, + const SniiAddIndexOptions& options, + doris::snii::writer::MemoryReporter* const mem_reporter) { + DCHECK(_storage_format == InvertedIndexStorageFormatPB::SNII); + DCHECK(index_meta != nullptr); + DCHECK(term_buffer != nullptr); + if (_idx_v2_writer == nullptr) { + return Status::Error( + "SNII index file writer is null for {}", _index_path_prefix); + } + if (_snii_file_writer == nullptr) { + _snii_file_writer = std::make_unique(_idx_v2_writer.get()); + _snii_compound_writer = + std::make_unique(_snii_file_writer.get()); + } + + doris::snii::writer::SniiIndexInput input; + input.index_id = cast_set(index_meta->index_id()); + input.index_suffix = index_meta->get_index_suffix(); + input.config = index_config; + input.doc_count = doc_count; + input.null_docids = std::move(null_docids); + input.term_source = term_buffer; + input.mem_reporter = mem_reporter; + // NOTE: deferred segments persist these thresholds too, although zero + // bigram terms were fed. On OLD readers (flag-blind) a non-zero threshold + // is the SECOND, redundant fallback trigger alongside the omitted sentinel + // -- either alone routes their pair-dict miss to positions verification -- + // so neither mechanism may be removed independently. + input.bigram_prune_min_df = snii_effective_bigram_prune_min_df(doc_count, index_config); + input.bigram_prune_max_df = + snii_effective_bigram_prune_max_df(doc_count, input.bigram_prune_min_df, index_config); + input.phrase_bigrams_deferred = options.phrase_bigrams_deferred; + // G16-c: freq regions serve only BM25 scoring; a plain positions index + // drops them unless the escape hatch asks for the full T2 layout. + input.write_freq = snii_effective_write_freq(index_config); + // G16-h: zstd levels. dict blocks accept zstd's full sane range; the prx + // level floor is 3 because the writer passes -level into the prx builders + // and -1 is the historic "auto at default level 3" sentinel -- a + // configured level 1 would silently resolve to 3 anyway (levels 1-2 buy + // nothing over 3 on these payloads). + input.dict_block_zstd_level = std::clamp(config::snii_dict_block_zstd_level, 1, 19); + // Patch C prx tiering: a direct load compresses prx at the cheaper load + // level; compaction / schema change / ADD INDEX keep snii_prx_zstd_level + // and compaction rewrites every segment with it, so settled segments (and + // the cold-query path over them) are byte-for-byte unaffected. + input.prx_zstd_level = + std::clamp(options.is_direct_load ? config::snii_prx_zstd_level_direct_load + : config::snii_prx_zstd_level, + 3, 19); + // G16-d: dict block size experiment knob; <= 0 keeps the format default. + if (config::snii_target_dict_block_bytes > 0) { + input.target_dict_block_bytes = static_cast(config::snii_target_dict_block_bytes); + } + // G04: hand the flush the buffer's ever-dropped bloom so vocab-cap-evicted + // bigram pairs that reappeared (incomplete postings) are dropped in addition + // to the df threshold. Null when no eviction ever fired (zero probe cost). + input.bigram_ever_dropped = term_buffer->bigram_dropped_filter(); + // GUARD against a mid-import config flip: the buffer suppressed bigram + // positions / evicted terms because pruning was enabled at init, but the + // mutable config now resolves the threshold to 0 (legacy layout). Flushing a + // legacy layout from a dieted buffer would either fail validation (missing + // positions) or -- worse -- materialize incomplete evicted pairs on a segment + // whose meta declares no fallback. Fail loudly instead. DEFERRED segments + // are exempt: they fed zero pair tokens, so the armed diet is provably + // inert (empty pair map, no evicted-incomplete pairs possible) and the + // resident deferred flag already routes readers to positions verification. + if (input.bigram_prune_min_df == 0 && !options.phrase_bigrams_deferred && + term_buffer->bigram_diet_configured()) { + return Status::InternalError( + "SNII: snii_bigram_prune_min_df was disabled while an import with the bigram " + "diet active was in flight for {}; keep the config stable during a load", + _index_path_prefix); + } + RETURN_IF_ERROR(_snii_compound_writer->add_logical_index(input)); + ++_snii_index_count; + return Status::OK(); +} + +void IndexFileWriter::retain_snii_memory_reporter( + std::unique_ptr mem_reporter) { + DCHECK(mem_reporter != nullptr); + _snii_memory_reporters.push_back(std::move(mem_reporter)); +} + Status IndexFileWriter::delete_index(const TabletIndex* index_meta) { DBUG_EXECUTE_IF("IndexFileWriter::delete_index_index_meta_nullptr", { index_meta = nullptr; }); if (!index_meta) { @@ -123,6 +290,9 @@ Status IndexFileWriter::delete_index(const TabletIndex* index_meta) { } Status IndexFileWriter::add_into_searcher_cache() { + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + return Status::OK(); + } auto index_file_reader = std::make_unique( _fs, _index_path_prefix, _storage_format, InvertedIndexFileInfo(), _tablet_id); auto st = index_file_reader->init(); @@ -196,6 +366,24 @@ Result> IndexFileWriter::_construct_index_ Status IndexFileWriter::begin_close() { DCHECK(!_closed) << debug_string(); _closed = true; + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + if (_snii_compound_writer == nullptr) { + if (_idx_v2_writer == nullptr) { + return Status::OK(); + } + _snii_file_writer = + std::make_unique(_idx_v2_writer.get()); + _snii_compound_writer = std::make_unique( + _snii_file_writer.get()); + } + RETURN_IF_ERROR(_snii_compound_writer->finish()); + _total_file_size = _idx_v2_writer == nullptr ? 0 : _idx_v2_writer->bytes_appended(); + _file_info.set_index_size(_total_file_size); + if (_idx_v2_writer != nullptr && _idx_v2_writer->state() != io::FileWriter::State::CLOSED) { + RETURN_IF_ERROR(_idx_v2_writer->close(true)); + } + return Status::OK(); + } if (_indices_dirs.empty()) { // An empty file must still be created even if there are no indexes to write if (dynamic_cast(_idx_v2_writer.get()) != nullptr || @@ -238,6 +426,12 @@ Status IndexFileWriter::begin_close() { Status IndexFileWriter::finish_close() { DCHECK(_closed) << debug_string(); + if (_storage_format == InvertedIndexStorageFormatPB::SNII) { + if (_idx_v2_writer != nullptr && _idx_v2_writer->state() != io::FileWriter::State::CLOSED) { + RETURN_IF_ERROR(_idx_v2_writer->close(false)); + } + return Status::OK(); + } if (_indices_dirs.empty()) { // An empty file must still be created even if there are no indexes to write if (dynamic_cast(_idx_v2_writer.get()) != nullptr || diff --git a/be/src/storage/index/index_file_writer.h b/be/src/storage/index/index_file_writer.h index a303de8b68c156..57126835232caf 100644 --- a/be/src/storage/index/index_file_writer.h +++ b/be/src/storage/index/index_file_writer.h @@ -24,6 +24,7 @@ #include #include +#include #include "common/be_mock_util.h" #include "io/fs/file_system.h" @@ -33,12 +34,24 @@ #include "storage/index/inverted/inverted_index_common.h" #include "storage/index/inverted/inverted_index_compound_reader.h" #include "storage/index/inverted/inverted_index_searcher.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/snii_doris_adapter.h" +#include "storage/index/snii/writer/snii_compound_writer.h" + +namespace doris::snii::writer { +class MemoryReporter; +class SpimiTermBuffer; +class SniiCompoundWriter; +} // namespace doris::snii::writer namespace doris { class TabletIndex; namespace segment_v2 { class DorisFSDirectory; +namespace snii_doris { +class DorisSniiFileWriter; +} // namespace snii_doris using InvertedIndexDirectoryMap = std::map, std::shared_ptr>; @@ -55,6 +68,26 @@ class IndexFileWriter { virtual ~IndexFileWriter() = default; MOCK_FUNCTION Result> open(const TabletIndex* index_meta); + // Write-path facts for one SNII index flush, captured per writer by + // SniiIndexColumnWriter::set_direct_load (latched before the first row). + struct SniiAddIndexOptions { + // The fresh direct-load segment deliberately omitted the hidden + // phrase-bigram build (pair postings + sentinel); persisted as the + // resident kPhraseBigramsDeferred capability flag. + bool phrase_bigrams_deferred = false; + // This flush serves a stream/broker load (DataWriteType::TYPE_DIRECT): + // the prx region compresses at snii_prx_zstd_level_direct_load; + // compaction / schema change / ADD INDEX keep snii_prx_zstd_level. + bool is_direct_load = false; + }; + Status add_snii_index(const TabletIndex* index_meta, uint32_t doc_count, + std::vector null_docids, + doris::snii::writer::SpimiTermBuffer* const term_buffer, + doris::snii::format::IndexConfig index_config, + const SniiAddIndexOptions& options, + doris::snii::writer::MemoryReporter* const mem_reporter); + void retain_snii_memory_reporter( + std::unique_ptr mem_reporter); Status delete_index(const TabletIndex* index_meta); Status initialize(InvertedIndexDirectoryMap& indices_dirs); Status add_into_searcher_cache(); @@ -113,6 +146,10 @@ class IndexFileWriter { IndexStorageFormatPtr _index_storage_format; int64_t _tablet_id = -1; + std::unique_ptr _snii_file_writer; + std::vector> _snii_memory_reporters; + std::unique_ptr _snii_compound_writer; + size_t _snii_index_count = 0; friend class IndexStorageFormatV1; friend class IndexStorageFormatV2; diff --git a/be/src/storage/index/index_query_context.h b/be/src/storage/index/index_query_context.h index 92fb698f4d3767..ad39075acf3ad4 100644 --- a/be/src/storage/index/index_query_context.h +++ b/be/src/storage/index/index_query_context.h @@ -32,6 +32,24 @@ struct IndexQueryContext { size_t query_limit = 0; bool is_asc = false; + + // G02 count-only fast-path handshake. Set by SegmentIterator ONLY while it + // evaluates the single pushed-down MATCH predicate of a COUNT_ON_INDEX scan + // whose row space is provably unfiltered (no deletes, no other conjuncts, + // full row bitmap, no row-id consumers -- see count_on_index_fastpath.h), + // and reset immediately after. When set, an index reader MAY answer the + // query with a bitmap whose CARDINALITY equals the match count without the + // row ids being real (SNII returns [0, df) straight from dict-entry df, + // skipping the posting decode). Readers must never cache such a bitmap + // under a key a row-accurate query could hit. + bool count_on_index_fastpath = false; + // G03 reply direction of the same handshake. Set by a reader iff it DID + // answer with such a fabricated count bitmap (never on a query-cache hit, + // a single-flight shared result, or any row-accurate decode). Read and + // reset by SegmentIterator right after the index apply; a true value is + // the precondition for the count-emission shortcut that materializes the + // remaining count as default rows without iterating the row bitmap. + bool count_on_index_fastpath_hit = false; }; using IndexQueryContextPtr = std::shared_ptr; diff --git a/be/src/storage/index/index_writer.cpp b/be/src/storage/index/index_writer.cpp index 2325d280471337..6fb23c3c107e51 100644 --- a/be/src/storage/index/index_writer.cpp +++ b/be/src/storage/index/index_writer.cpp @@ -18,6 +18,7 @@ #include "common/exception.h" #include "storage/index/ann/ann_index_writer.h" #include "storage/index/inverted/inverted_index_writer.h" +#include "storage/index/snii/snii_index_writer.h" #include "storage/tablet/tablet_schema.h" #include "storage/types.h" @@ -80,6 +81,22 @@ Status IndexColumnWriter::create(const TabletColumn* column, } } + if (storage_format == InvertedIndexStorageFormatPB::SNII) { + if (!is_string_type(type)) { + return Status::Error( + "SNII inverted index storage format does not support BKD index type {}", + type); + } + *res = std::make_unique(index_file_writer, index_meta, + single_field); + auto st = (*res)->init(); + if (!st.ok()) { + (*res)->close_on_error(); + return st; + } + return Status::OK(); + } + DBUG_EXECUTE_IF("InvertedIndexColumnWriter::create_unsupported_type_for_inverted_index", { type = FieldType::OLAP_FIELD_TYPE_JSONB; }) switch (type) { diff --git a/be/src/storage/index/index_writer.h b/be/src/storage/index/index_writer.h index a0760f99000fcb..1084ca6fc10af8 100644 --- a/be/src/storage/index/index_writer.h +++ b/be/src/storage/index/index_writer.h @@ -59,6 +59,17 @@ class IndexColumnWriter { virtual Status add_nulls(uint32_t count) = 0; virtual Status add_array_nulls(const uint8_t* null_map, size_t num_rows) = 0; + // Write-path hint from the segment writer: this writer serves a direct load + // (stream/broker load, DataWriteType::TYPE_DIRECT) as opposed to compaction + // / schema change / index build. The column writer forwards it + // unconditionally right after create() succeeds (i.e. after init(), before + // any value is added); creation paths outside the column writer (e.g. ADD + // INDEX in index_builder.cpp) never call it and keep the non-direct default. + // Default no-op: only SNII acts on it today (deferring the hidden + // phrase-bigram build to compaction, see + // config::snii_bigram_defer_build_to_compaction). + virtual void set_direct_load(bool /*is_direct_load*/) {} + virtual Status finish() = 0; virtual int64_t size() const = 0; diff --git a/be/src/storage/index/inverted/inverted_index_cache.h b/be/src/storage/index/inverted/inverted_index_cache.h index 0e33fe747b3523..fa2cd3ebf088de 100644 --- a/be/src/storage/index/inverted/inverted_index_cache.h +++ b/be/src/storage/index/inverted/inverted_index_cache.h @@ -35,6 +35,7 @@ #include "runtime/memory/lru_cache_policy.h" #include "runtime/memory/mem_tracker.h" #include "storage/index/inverted/inverted_index_searcher.h" +#include "storage/index/snii/reader/logical_index_reader.h" #include "util/lru_cache.h" #include "util/slice.h" #include "util/time.h" @@ -42,6 +43,7 @@ namespace doris { namespace segment_v2 { class InvertedIndexCacheHandle; +class IndexFileReader; class InvertedIndexSearcherCache { public: @@ -56,6 +58,8 @@ class InvertedIndexSearcherCache { class CacheValue : public LRUCacheValueBase { public: IndexSearcherPtr index_searcher; + std::shared_ptr snii_index_file_reader; + std::unique_ptr snii_logical_reader; size_t size = 0; int64_t last_visit_time; @@ -65,6 +69,14 @@ class InvertedIndexSearcherCache { size = mem_size; last_visit_time = visit_time; } + explicit CacheValue(std::unique_ptr logical_reader, + size_t mem_size, int64_t visit_time, + std::shared_ptr index_file_reader) + : snii_index_file_reader(std::move(index_file_reader)), + snii_logical_reader(std::move(logical_reader)) { + size = mem_size; + last_visit_time = visit_time; + } }; // Create global instance of this class. // "capacity" is the capacity of lru cache. @@ -166,6 +178,11 @@ class InvertedIndexCacheHandle { return ((InvertedIndexSearcherCache::CacheValue*)_cache->value(_handle))->index_searcher; } + doris::snii::reader::LogicalIndexReader* get_snii_logical_reader() { + return ((InvertedIndexSearcherCache::CacheValue*)_cache->value(_handle)) + ->snii_logical_reader.get(); + } + InvertedIndexSearcherCache::CacheValue* get_index_cache_value() { return ((InvertedIndexSearcherCache::CacheValue*)_cache->value(_handle)); } diff --git a/be/src/storage/index/inverted/inverted_index_fs_directory.cpp b/be/src/storage/index/inverted/inverted_index_fs_directory.cpp index e65025b25a4fc7..ea67973305030b 100644 --- a/be/src/storage/index/inverted/inverted_index_fs_directory.cpp +++ b/be/src/storage/index/inverted/inverted_index_fs_directory.cpp @@ -97,8 +97,14 @@ bool DorisFSDirectory::FSIndexInput::open(const io::FileSystemSPtr& fs, const ch auto h = std::make_shared(path); io::FileReaderOptions reader_options; - reader_options.cache_type = config::enable_file_cache ? io::FileCachePolicy::FILE_BLOCK_CACHE - : io::FileCachePolicy::NO_CACHE; + // DIAGNOSTIC: inverted_index_read_bypass_file_cache forces NO_CACHE (precise + // S3 range GETs) for inverted-index reads (both CLucene here and SNII), so a + // fair direct-vs-direct engine comparison can bypass the 1MiB block cache + // without disabling the global enable_file_cache (cloud mode requires it). + reader_options.cache_type = + (config::enable_file_cache && !config::inverted_index_read_bypass_file_cache) + ? io::FileCachePolicy::FILE_BLOCK_CACHE + : io::FileCachePolicy::NO_CACHE; reader_options.is_doris_table = true; reader_options.file_size = file_size; reader_options.tablet_id = tablet_id; @@ -179,16 +185,15 @@ void DorisFSDirectory::FSIndexInput::close() { } void DorisFSDirectory::FSIndexInput::setIoContext(const void* io_ctx) { + const bool is_index_data = _io_ctx.is_index_data; if (io_ctx) { const auto& ctx = static_cast(io_ctx); - _io_ctx.reader_type = ctx->reader_type; - _io_ctx.query_id = ctx->query_id; - _io_ctx.file_cache_stats = ctx->file_cache_stats; + _io_ctx = *ctx; } else { - _io_ctx.reader_type = ReaderType::UNKNOWN; - _io_ctx.query_id = nullptr; - _io_ctx.file_cache_stats = nullptr; + _io_ctx = io::IOContext {}; } + _io_ctx.is_index_data = is_index_data; + _io_ctx.is_inverted_index = true; } const void* DorisFSDirectory::FSIndexInput::getIoContext() { @@ -247,6 +252,10 @@ void DorisFSDirectory::FSIndexInput::readInternal(uint8_t* b, const int32_t len) if (_io_ctx.file_cache_stats != nullptr) { _io_ctx.file_cache_stats->inverted_index_io_timer += inverted_index_io_timer; + _io_ctx.file_cache_stats->inverted_index_request_bytes += len; + _io_ctx.file_cache_stats->inverted_index_read_bytes += len; + ++_io_ctx.file_cache_stats->inverted_index_range_read_count; + ++_io_ctx.file_cache_stats->inverted_index_serial_read_rounds; } } diff --git a/be/src/storage/index/inverted/inverted_index_reader.h b/be/src/storage/index/inverted/inverted_index_reader.h index 0e2f6a120d41e3..a2aa0533f2bf7b 100644 --- a/be/src/storage/index/inverted/inverted_index_reader.h +++ b/be/src/storage/index/inverted/inverted_index_reader.h @@ -230,9 +230,9 @@ class InvertedIndexReader : public IndexReader { const Field& query_value, InvertedIndexQueryType query_type, size_t* count) = 0; - Status read_null_bitmap(const IndexQueryContextPtr& context, - InvertedIndexQueryCacheHandle* cache_handle, - lucene::store::Directory* dir = nullptr); + virtual Status read_null_bitmap(const IndexQueryContextPtr& context, + InvertedIndexQueryCacheHandle* cache_handle, + lucene::store::Directory* dir = nullptr); virtual InvertedIndexReaderType type() = 0; @@ -335,7 +335,6 @@ class InvertedIndexVisitor : public lucene::util::bkd::bkd_reader::intersect_vis std::string query_min; std::string query_max; -public: InvertedIndexVisitor(const void* io_ctx, lucene::util::bkd::bkd_reader* r, roaring::Roaring* hits, bool only_count = false); ~InvertedIndexVisitor() override = default; diff --git a/be/src/storage/index/snii/common/single_flight.h b/be/src/storage/index/snii/common/single_flight.h new file mode 100644 index 00000000000000..f3cdbaa9547b14 --- /dev/null +++ b/be/src/storage/index/snii/common/single_flight.h @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace doris::snii { + +// Collapses concurrent identical operations (same key) into a single execution. +// +// Motivating case: under a cold cache, Doris parallel scanners _lazy_init the same segment +// concurrently and would each miss the inverted-index caches and redundantly open + decode +// that segment's index. With single-flight the first caller (the leader) executes while the +// concurrent callers (followers) await and reuse its result -- so the duplicated open/decode +// is eliminated. +// +// No blocking work runs under the internal mutex: the leader performs its (potentially IO +// bound) work *after* join_or_lead() returns, holding no lock, and followers block on a +// std::future rather than the mutex. This preserves the SNII "no IO under lock" rule. +// +// Result must be copyable -- each follower receives its own copy. For large payloads use a +// shared_ptr (which is also how Doris query bitmaps are shared across cache consumers), so +// the copy is just a refcount bump and the payload itself is shared read-only. +template +class SingleFlight { +public: + using ResultFuture = std::shared_future; + + // If `key` is already in flight, returns the leader's future and the caller is a + // FOLLOWER: await the future, do not publish. Otherwise registers `key` as in flight, + // returns std::nullopt, and the caller is the LEADER and MUST call publish(key, ...) + // exactly once -- use a scope guard so it runs even on an error/early return. + std::optional join_or_lead(const std::string& key) { + std::lock_guard guard(_mutex); + if (auto it = _inflight.find(key); it != _inflight.end()) { + return it->second->future; + } + auto flight = std::make_shared(); + flight->future = flight->promise.get_future().share(); + _inflight.emplace(key, std::move(flight)); + return std::nullopt; + } + + // Publishes the leader's result, wakes all waiting followers, and clears the in-flight + // entry so the next caller of the same key leads a fresh execution. A no-op if the entry + // is already gone (defensive; should not happen for a correct leader). + void publish(const std::string& key, Result result) { + std::shared_ptr flight; + { + std::lock_guard guard(_mutex); + auto it = _inflight.find(key); + if (it == _inflight.end()) { + return; + } + flight = std::move(it->second); + _inflight.erase(it); + } + // set_value() outside the lock: it may wake followers, and we never hold the mutex + // across that hand-off. + flight->promise.set_value(std::move(result)); + } + + // Number of keys currently in flight. For tests/observability only. + size_t inflight_size() const { + std::lock_guard guard(_mutex); + return _inflight.size(); + } + +private: + struct Flight { + std::promise promise; + ResultFuture future; + }; + + mutable std::mutex _mutex; + std::unordered_map> _inflight; +}; + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/common/slice.h b/be/src/storage/index/snii/common/slice.h new file mode 100644 index 00000000000000..e5b80932944df3 --- /dev/null +++ b/be/src/storage/index/snii/common/slice.h @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +namespace doris::snii { + +// Read-only byte view (does not own memory). Lifetime is managed by the underlying buffer. +class Slice { +public: + Slice() = default; + Slice(const uint8_t* d, size_t n) : data_(d), size_(n) {} + explicit Slice(const std::vector& v) : data_(v.data()), size_(v.size()) {} + explicit Slice(std::string_view sv) + : data_(reinterpret_cast(sv.data())), size_(sv.size()) {} + + const uint8_t* data() const { return data_; } + size_t size() const { return size_; } + bool empty() const { return size_ == 0; } + + uint8_t operator[](size_t i) const { + assert(i < size_); + return data_[i]; + } + + Slice subslice(size_t off, size_t n) const { + assert(off + n <= size_); + return Slice(data_ + off, n); + } + +private: + const uint8_t* data_ = nullptr; + size_t size_ = 0; +}; + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/common/uninitialized_buffer.h b/be/src/storage/index/snii/common/uninitialized_buffer.h new file mode 100644 index 00000000000000..1b1855db566d0c --- /dev/null +++ b/be/src/storage/index/snii/common/uninitialized_buffer.h @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +// Single auditable seam for "resize a decode buffer that is about to be fully +// overwritten". Collapses the scattered `assign(n, 0)` / `resize(n)` calls in the +// PFOR/zstd decode paths into one intent-revealing entry point and removes the +// redundant zero-fill on warm-reused buffers. +namespace doris::snii { + +// Resize a vector of trivially-copyable T to `n` without any zero-fill beyond what +// std::vector itself mandates. +// +// CONTRACT: the caller MUST fully overwrite [0, n) before reading any element. The +// PFOR (`pfor_decode`/`bitunpack`, including the w==0 memset and exception patch +// paths) and zstd (`ZSTD_decompress` + length check) decoders all satisfy this. +// +// For a warm-reused buffer whose current size() >= n this performs NO +// value-initialization (resize shrinks in place). For a cold/grown buffer +// std::vector still value-initializes the new tail -- that is unavoidable for +// std::vector and is intentional here. Do NOT use on any buffer that may be read +// before being fully overwritten. +template +inline void resize_uninitialized(std::vector& v, std::size_t n) { + static_assert(std::is_trivially_copyable_v, + "resize_uninitialized requires a trivially-copyable element type"); + v.resize(n); +} + +// Allocator that default-initializes (instead of value-initializes) on the no-arg +// construct path: for trivial T this skips zeroing entirely. Use only for buffers +// that are fully overwritten before any read. +template +struct default_init_allocator : std::allocator { + template + struct rebind { + using other = default_init_allocator; + }; + using std::allocator::allocator; + + template + void construct(U* p, Args&&... args) { + if constexpr (sizeof...(Args) == 0) { + ::new (static_cast(p)) U; // default-init: no zeroing for trivial U + } else { + ::new (static_cast(p)) U(std::forward(args)...); + } + } +}; + +// A vector whose grow path default-initializes, so even a cold grow avoids the +// zero-fill that std::vector would perform for trivial T. +template +using uninitialized_vector = std::vector>; + +template +inline void resize_uninitialized(uninitialized_vector& v, std::size_t n) { + v.resize(n); +} + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/byte_sink.cpp b/be/src/storage/index/snii/encoding/byte_sink.cpp new file mode 100644 index 00000000000000..d51d091bdc1ef7 --- /dev/null +++ b/be/src/storage/index/snii/encoding/byte_sink.cpp @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/byte_sink.h" + +#include "storage/index/snii/encoding/varint.h" + +namespace doris::snii { + +void ByteSink::put_fixed16(uint16_t v) { + for (int i = 0; i < 2; ++i) buf_.push_back(static_cast(v >> (8 * i))); +} + +void ByteSink::put_fixed32(uint32_t v) { + for (int i = 0; i < 4; ++i) buf_.push_back(static_cast(v >> (8 * i))); +} + +void ByteSink::put_fixed64(uint64_t v) { + for (int i = 0; i < 8; ++i) buf_.push_back(static_cast(v >> (8 * i))); +} + +void ByteSink::put_varint32(uint32_t v) { + uint8_t tmp[5]; + size_t n = encode_varint32(v, tmp); + buf_.insert(buf_.end(), tmp, tmp + n); +} + +void ByteSink::put_varint64(uint64_t v) { + uint8_t tmp[10]; + size_t n = encode_varint64(v, tmp); + buf_.insert(buf_.end(), tmp, tmp + n); +} + +void ByteSink::put_zigzag(int64_t v) { + put_varint64(zigzag_encode(v)); +} + +void ByteSink::put_bytes(Slice s) { + buf_.insert(buf_.end(), s.data(), s.data() + s.size()); +} + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/byte_sink.h b/be/src/storage/index/snii/encoding/byte_sink.h new file mode 100644 index 00000000000000..e96ae112e20a31 --- /dev/null +++ b/be/src/storage/index/snii/encoding/byte_sink.h @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "storage/index/snii/common/slice.h" + +namespace doris::snii { + +// append-only write cursor: all section serialization goes through this; manual byte assembly is forbidden. +// All multi-byte fixed-width fields are little-endian. +class ByteSink { +public: + void put_u8(uint8_t v) { buf_.push_back(v); } + void put_fixed16(uint16_t v); + void put_fixed32(uint32_t v); + void put_fixed64(uint64_t v); + void put_varint32(uint32_t v); + void put_varint64(uint64_t v); + void put_zigzag(int64_t v); + void put_bytes(Slice s); + + size_t size() const { return buf_.size(); } + const std::vector& buffer() const { return buf_; } + Slice view() const { return Slice(buf_); } + + // Reserves capacity for `additional` MORE bytes on top of the current size(), + // so a caller about to append a known-length run pays at most one reallocation + // instead of the geometric-growth reallocs a byte-at-a-time put_u8 loop would + // trigger. The argument is RELATIVE (absolute target = size() + additional): a + // sink REUSED across encodes (e.g. the shared PFOR-run `out`) keeps accumulating + // correctly rather than no-op'ing on a repeated per-run reserve of the same + // value. Affects only the backing buffer's capacity -- the emitted bytes are + // unchanged. + void reserve(size_t additional) { buf_.reserve(buf_.size() + additional); } + + // Resets the cursor to empty while RETAINING the backing capacity, so a sink can + // be reused across many small encodes (e.g. per-window region/prx scratch in the + // windowed posting builder) without re-allocating each time -- this avoids the + // cumulative small-allocation churn that fragments the heap arena and inflates + // peak RSS during the merge of a high-df term split into thousands of windows. + void clear() { buf_.clear(); } + + // Moves the backing buffer OUT to the caller (the sink is left empty), so an encoded + // section can be handed off without the copy (+ copy-induced capacity slack) that + // reading buffer() and copy-assigning would incur. Use only when the sink is not + // reused afterward (a stack-local about to die, or one that is clear()'d next). + std::vector take() { return std::move(buf_); } + +private: + std::vector buf_; +}; + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/byte_source.cpp b/be/src/storage/index/snii/encoding/byte_source.cpp new file mode 100644 index 00000000000000..4a7dd11322ce3c --- /dev/null +++ b/be/src/storage/index/snii/encoding/byte_source.cpp @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/byte_source.h" + +#include "storage/index/snii/encoding/varint.h" + +namespace doris::snii { + +Status ByteSource::get_u8(uint8_t* v) { + if (remaining() < 1) + return Status::Error("get_u8 overrun"); + *v = s_[pos_++]; + return Status::OK(); +} + +Status ByteSource::get_fixed16(uint16_t* v) { + if (remaining() < 2) + return Status::Error( + "get_fixed16 overrun"); + uint16_t r = 0; + for (int i = 0; i < 2; ++i) r |= static_cast(s_[pos_ + i]) << (8 * i); + pos_ += 2; + *v = r; + return Status::OK(); +} + +Status ByteSource::get_fixed32(uint32_t* v) { + if (remaining() < 4) + return Status::Error( + "get_fixed32 overrun"); + uint32_t r = 0; + for (int i = 0; i < 4; ++i) r |= static_cast(s_[pos_ + i]) << (8 * i); + pos_ += 4; + *v = r; + return Status::OK(); +} + +Status ByteSource::get_fixed64(uint64_t* v) { + if (remaining() < 8) + return Status::Error( + "get_fixed64 overrun"); + uint64_t r = 0; + for (int i = 0; i < 8; ++i) r |= static_cast(s_[pos_ + i]) << (8 * i); + pos_ += 8; + *v = r; + return Status::OK(); +} + +Status ByteSource::get_varint64(uint64_t* v) { + const uint8_t* p = s_.data() + pos_; + const uint8_t* next = nullptr; + RETURN_IF_ERROR(decode_varint64(p, s_.data() + s_.size(), v, &next)); + pos_ = static_cast(next - s_.data()); + return Status::OK(); +} + +Status ByteSource::get_varint32(uint32_t* v) { + uint64_t tmp; + RETURN_IF_ERROR(get_varint64(&tmp)); + if (tmp > 0xFFFFFFFFu) + return Status::Error("varint32 overflow"); + *v = static_cast(tmp); + return Status::OK(); +} + +Status ByteSource::decode_delta_run(size_t count, std::vector* out) { + const uint8_t* const begin = s_.data(); + const uint8_t* const end = begin + s_.size(); + const uint8_t* p = begin + pos_; + out->reserve(out->size() + count); + uint32_t running = 0; + for (size_t i = 0; i < count; ++i) { + if (p >= end) { + return Status::Error( + "byte_source: delta run past end"); + } + uint32_t b = *p++; + uint64_t delta = b & 0x7FU; + if (b >= 0x80) { // multi-byte (rare for position deltas) + uint32_t shift = 7; + for (;;) { + if (p >= end) { + return Status::Error( + "byte_source: delta run past end"); + } + b = *p++; + delta |= static_cast(b & 0x7FU) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + if (shift > 28) { // a 32-bit value fits in <= 5 bytes (35 bits) + if (b > 0x0F) { + return Status::Error( + "byte_source: delta varint32 overflow"); + } + break; + } + } + } + running += static_cast(delta); + out->push_back(running); + } + pos_ = static_cast(p - begin); + return Status::OK(); +} + +Status ByteSource::skip_varints(size_t count) { + const uint8_t* const begin = s_.data(); + const uint8_t* const end = begin + s_.size(); + const uint8_t* p = begin + pos_; + // Each varint ends at the first byte whose continuation bit (0x80) is clear. + // Scanning for `count` such terminators skips the values with one branch per + // byte -- no shift/accumulate/store and no per-value bounds Status. (A SIMD + // bulk terminator-count was tried and reverted: the skipped position runs + // between selected docs are almost always 1-3 varints -- far below a 16-byte + // block -- so the vector path never amortized, and the larger body stopped + // this function from inlining into the CSR reader, a net CPU regression on + // the httplogs/agentlogs phrase-prefix profiles.) + for (size_t k = 0; k < count; ++k) { + while (p < end && (*p & 0x80) != 0) { + ++p; + } + if (p >= end) { + return Status::Error( + "byte_source: varint skip past end"); + } + ++p; // consume the terminator byte + } + pos_ = static_cast(p - begin); + return Status::OK(); +} + +Status ByteSource::get_zigzag(int64_t* v) { + uint64_t tmp; + RETURN_IF_ERROR(get_varint64(&tmp)); + *v = zigzag_decode(tmp); + return Status::OK(); +} + +Status ByteSource::get_bytes(size_t n, Slice* out) { + if (remaining() < n) + return Status::Error("get_bytes overrun"); + *out = s_.subslice(pos_, n); + pos_ += n; + return Status::OK(); +} + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/byte_source.h b/be/src/storage/index/snii/encoding/byte_source.h new file mode 100644 index 00000000000000..34264166d715d2 --- /dev/null +++ b/be/src/storage/index/snii/encoding/byte_source.h @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" + +namespace doris::snii { + +// Slice read cursor: all section deserialization goes through this; any overrun returns Corruption. +class ByteSource { +public: + explicit ByteSource(Slice s) : s_(s) {} + + Status get_u8(uint8_t* v); + Status get_fixed16(uint16_t* v); + Status get_fixed32(uint32_t* v); + Status get_fixed64(uint64_t* v); + Status get_varint32(uint32_t* v); + Status get_varint64(uint64_t* v); + + // Single-byte fast-path varint32. The CSR position reader decodes one count + // header (`pos_count`) per doc in a window -- for every doc, selected or + // skipped -- and that per-doc read, routed through the out-of-line + // get_varint32 -> get_varint64 -> decode_varint64 chain, dominates the loop + // once positions themselves are skipped/inlined. Almost all counts are < 128 + // (one byte), so inline that case here and fall back to the full decoder for + // multi-byte values and bounds handling. + Status get_varint32_fast(uint32_t* v) { + const size_t p = pos_; + if (p < s_.size()) { + const uint8_t b0 = s_[p]; + if (b0 < 0x80) { + *v = b0; + pos_ = p + 1; + return Status::OK(); + } + } + return get_varint32(v); + } + // Advances past `count` LEB128 varints WITHOUT decoding their values -- just + // scans continuation bytes. Cheaper than get_varint* per value when the + // decoded value is unused (e.g. skipping a non-selected doc's position + // deltas in a CSR window, where the vast majority of docs in a window are + // not in the candidate set). Returns Corruption on truncation. + Status skip_varints(size_t count); + + // Decodes `count` LEB128 varints, treats them as ASCENDING deltas + // (running prefix sum starting at 0), and APPENDS the running values to + // `out`. A tight inline decoder -- no per-value get_varint32/get_varint64/ + // decode_varint64 call chain, no per-value Status, and a single-byte fast + // path (position deltas are almost always < 128). This is the hot loop of + // the CSR position reader for candidate (selected) docs. Returns Corruption + // on truncation or a >32-bit value. + Status decode_delta_run(size_t count, std::vector* out); + Status get_zigzag(int64_t* v); + Status get_bytes(size_t n, Slice* out); + + size_t remaining() const { return s_.size() - pos_; } + size_t position() const { return pos_; } + bool eof() const { return pos_ == s_.size(); } + + // Returns a sub-view starting at absolute offset start with length len (used by framer etc. to rewind over the CRC coverage region). + Slice slice_from(size_t start, size_t len) const { return s_.subslice(start, len); } + +private: + Slice s_; + size_t pos_ = 0; +}; + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/crc32c.cpp b/be/src/storage/index/snii/encoding/crc32c.cpp new file mode 100644 index 00000000000000..39d7c6f58fe487 --- /dev/null +++ b/be/src/storage/index/snii/encoding/crc32c.cpp @@ -0,0 +1,278 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/crc32c.h" + +// T21 test-seam implementation. Production crc32c()/crc32c_extend() (in the header) +// delegate to the bundled Google crc32c thirdparty, which already runs a +// runtime-dispatched, hardware-accelerated and interleaved CRC32C. The reference +// sub-paths defined here -- portable slice-by-8, serial SSE4.2 hardware, and the +// 3-way interleaved SSE4.2 hardware algorithm T21 specifies -- exist ONLY so that +// unit tests can prove, byte-for-byte across all sizes and alignments, that the +// production path equals the canonical CRC32C (same bit-reflected Castagnoli +// polynomial) and that the hardware path is engaged. All of them are compiled out +// of release builds by the BE_TEST gate, so production pays nothing: no extra code, +// no static slice-by-8 table, no startup CPUID probe. Every function here is a pure +// function with no shared mutable state, so it is trivially thread-safe. +#ifdef BE_TEST + +#include +#include +#include + +#if defined(__x86_64__) || defined(_M_X64) +#define SNII_CRC32C_X86 1 +#include // __get_cpuid, bit_SSE4_2 +#include // _mm_crc32_u8/u32/u64 (SSE4.2) +#endif + +namespace doris::snii { +namespace { + +// Bit-reflected Castagnoli polynomial (CRC32C / iSCSI). Identical to the constant +// the removed in-tree implementation used, so every reference path below yields +// the same on-disk checksum value as the production library. +constexpr uint32_t kPoly = 0x82F63B78U; + +// Below this length the 3-way path's lane setup and GF(2) shift-combine outweigh +// the throughput win, so crc32c_hw3 falls back to the serial hardware path. Small +// buffers (inline prx windows, small pod_ref regions) therefore never pay the +// combine cost. Exposed to tests via crc32c_interleave_threshold(). +constexpr size_t kInterleaveThreshold = 1024; + +// Builds the slice-by-8 lookup tables. Column 0 is the classic byte table; each +// successive column folds in one more byte of look-ahead, letting the inner loop +// consume 8 bytes per iteration with 8 table reads + XORs instead of 8 dependent +// shift/lookup steps. The checksum value is identical to the byte-at-a-time loop. +std::array, 8> make_slice8_table() { + std::array, 8> t {}; + for (uint32_t i = 0; i < 256; ++i) { + uint32_t c = i; + for (int k = 0; k < 8; ++k) { + c = (c & 1) ? (kPoly ^ (c >> 1)) : (c >> 1); + } + t[0][i] = c; + } + for (uint32_t i = 0; i < 256; ++i) { + uint32_t c = t[0][i]; + for (int s = 1; s < 8; ++s) { + c = t[0][c & 0xFF] ^ (c >> 8); + t[s][i] = c; + } + } + return t; +} + +const std::array, 8> kSlice8 = make_slice8_table(); + +inline uint32_t load_le32(const uint8_t* p) { + return static_cast(p[0]) | (static_cast(p[1]) << 8) | + (static_cast(p[2]) << 16) | (static_cast(p[3]) << 24); +} + +// Pure software slice-by-8 (used as the portable path and the hardware fallback). +// Operates on the raw (pre/post-inversion applied by the callers) CRC register. +uint32_t crc32c_slice8(uint32_t crc, const uint8_t* p, size_t n) { + while (n >= 8) { + crc ^= load_le32(p); + const uint32_t hi = load_le32(p + 4); + crc = kSlice8[7][crc & 0xFF] ^ kSlice8[6][(crc >> 8) & 0xFF] ^ + kSlice8[5][(crc >> 16) & 0xFF] ^ kSlice8[4][crc >> 24] ^ kSlice8[3][hi & 0xFF] ^ + kSlice8[2][(hi >> 8) & 0xFF] ^ kSlice8[1][(hi >> 16) & 0xFF] ^ kSlice8[0][hi >> 24]; + p += 8; + n -= 8; + } + while (n--) { + crc = kSlice8[0][(crc ^ *p++) & 0xFF] ^ (crc >> 8); + } + return crc; +} + +#if SNII_CRC32C_X86 +// Serial hardware CRC32C via the SSE4.2 crc32 instruction. The intrinsics operate +// on the same bit-reflected Castagnoli polynomial as the tables, so the result is +// byte-identical. This TU is compiled without -msse4.2, so gate the intrinsics +// behind a function-level target attribute and a runtime CPUID check. This is the +// authoritative serial hardware path that crc32c_hw3 reuses for each lane and tail. +__attribute__((target("sse4.2"))) uint32_t crc32c_hw_serial(uint32_t crc, const uint8_t* p, + size_t n) { + while (n >= 8) { + uint64_t v; + std::memcpy(&v, p, sizeof(v)); // unaligned-safe; x86 folds to a plain load + crc = static_cast(_mm_crc32_u64(crc, v)); + p += 8; + n -= 8; + } + if (n >= 4) { + crc = _mm_crc32_u32(crc, load_le32(p)); + p += 4; + n -= 4; + } + while (n--) { + crc = _mm_crc32_u8(crc, *p++); + } + return crc; +} + +// GF(2) 32x32 bit-matrix helpers (zlib crc32_combine style). A matrix column mat[i] +// is the image of the i-th unit CRC register; gf2_matrix_times sums (XORs) the +// columns selected by the set bits of vec. +uint32_t gf2_matrix_times(const uint32_t* mat, uint32_t vec) { + uint32_t sum = 0; + while (vec != 0) { + if (vec & 1) { + sum ^= *mat; + } + vec >>= 1; + ++mat; + } + return sum; +} + +void gf2_matrix_square(uint32_t* square, const uint32_t* mat) { + for (int n = 0; n < 32; ++n) { + square[n] = gf2_matrix_times(mat, mat[n]); + } +} + +// crc32c_shift(crc, bytes): advance the raw CRC32C register as if `bytes` zero +// bytes were appended, i.e. crc . x^(8*bytes) mod P in the bit-reflected domain. +// This is the linear operator the 3-way combine applies to a lane's partial CRC so +// it lines up with the following lanes -- a pure table/matrix computation with no +// PCLMULQDQ, hence no extra CPUID gate. bytes == 0 returns crc unchanged. +uint32_t crc32c_shift(uint32_t crc, size_t bytes) { + uint32_t even[32]; // operator for 2^k zero bits, doubled each round + uint32_t odd[32]; // operator for 2^(k-1) zero bits + + // odd = operator for a single zero bit: column 0 is the polynomial, columns + // 1..31 shift the register right by one (bit i maps to bit i-1). + odd[0] = kPoly; + uint32_t row = 1; + for (int n = 1; n < 32; ++n) { + odd[n] = row; + row <<= 1; + } + gf2_matrix_square(even, odd); // even = two zero bits + gf2_matrix_square(odd, even); // odd = four zero bits + + size_t len = bytes; + do { + gf2_matrix_square(even, odd); // first pass: even = one zero byte (8 bits) + if (len & 1) { + crc = gf2_matrix_times(even, crc); + } + len >>= 1; + if (len == 0) { + break; + } + gf2_matrix_square(odd, even); + if (len & 1) { + crc = gf2_matrix_times(odd, crc); + } + len >>= 1; + } while (len != 0); + return crc; +} + +// 3-way interleaved hardware CRC32C (rocksdb/folly/Intel style). Splits the buffer +// into three equal, 8-byte-aligned lanes processed by independent _mm_crc32_u64 +// accumulators (breaking the ~3-cycle loop-carried dependency of the serial path), +// then stitches them with the GF(2) shift-combine and finishes the remainder +// serially. Byte-identical to crc32c_hw_serial / crc32c_slice8. Below the +// threshold it defers to the serial path so small buffers skip the combine cost. +uint32_t crc32c_hw3(uint32_t crc, const uint8_t* p, size_t n) { + if (n < kInterleaveThreshold) { + return crc32c_hw_serial(crc, p, n); + } + // 8-byte-aligned lane length keeps every lane on the u64 fast path. n >= 1024 + // guarantees L >= 336 > 0, so 3L <= n and the tail (n - 3L) is well defined. + const size_t lane = (n / 3) & ~static_cast(7); + const uint32_t crc_a = crc32c_hw_serial(crc, p, lane); // seeded lane + const uint32_t crc_b = crc32c_hw_serial(0, p + lane, lane); // raw lane + const uint32_t crc_c = crc32c_hw_serial(0, p + 2 * lane, lane); // raw lane + // crc(seed, A||B) == shift(crc(seed, A), |B|) ^ crc(0, B), applied twice. + uint32_t comb = crc32c_shift(crc_a, lane) ^ crc_b; + comb = crc32c_shift(comb, lane) ^ crc_c; + return crc32c_hw_serial(comb, p + 3 * lane, n - 3 * lane); // serial tail +} + +bool detect_sse42() { + unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0; + if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx)) { + return false; + } + return (ecx & bit_SSE4_2) != 0; +} + +const bool kHasSse42 = detect_sse42(); +#endif // SNII_CRC32C_X86 + +} // namespace + +namespace detail { + +// Portable software path. Always available; the canonical scalar reference for +// every other path and for the on-disk checksum value. +uint32_t crc32c_slice8_extend(uint32_t crc, Slice data) { + crc = ~crc; + crc = crc32c_slice8(crc, data.data(), data.size()); + return ~crc; +} + +// Serial hardware path (falls back to slice8 without SSE4.2). +uint32_t crc32c_hw_serial_extend(uint32_t crc, Slice data) { + crc = ~crc; +#if SNII_CRC32C_X86 + if (kHasSse42) { + crc = crc32c_hw_serial(crc, data.data(), data.size()); + return ~crc; + } +#endif + crc = crc32c_slice8(crc, data.data(), data.size()); + return ~crc; +} + +// 3-way interleaved hardware path (falls back to slice8 without SSE4.2; internally +// falls back to the serial path below the interleave threshold). +uint32_t crc32c_hw3_extend(uint32_t crc, Slice data) { + crc = ~crc; +#if SNII_CRC32C_X86 + if (kHasSse42) { + crc = crc32c_hw3(crc, data.data(), data.size()); + return ~crc; + } +#endif + crc = crc32c_slice8(crc, data.data(), data.size()); + return ~crc; +} + +size_t crc32c_interleave_threshold() { + return kInterleaveThreshold; +} + +bool crc32c_has_hw() { +#if SNII_CRC32C_X86 + return kHasSse42; +#else + return false; +#endif +} + +} // namespace detail +} // namespace doris::snii + +#endif // BE_TEST diff --git a/be/src/storage/index/snii/encoding/crc32c.h b/be/src/storage/index/snii/encoding/crc32c.h new file mode 100644 index 00000000000000..775a781c3d39f0 --- /dev/null +++ b/be/src/storage/index/snii/encoding/crc32c.h @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +#include "storage/index/snii/common/slice.h" + +namespace doris::snii { + +// CRC32C (Castagnoli, polynomial 0x1EDC6F41). Used to checksum the tail of each +// format block. Thin inline adapter over Doris's bundled Google crc32c thirdparty +// (crc32c::Extend / crc32c::Crc32c). That library computes the same canonical +// CRC32C (same reflected polynomial, same standard pre/post inversion), so every +// on-disk checksum stays byte-identical to the previous in-tree slice-by-8 / +// SSE4.2 implementation -- this is an implementation swap, not a format change. +// The leading :: keeps the crc32c namespace distinct from crc32c() below. +inline uint32_t crc32c_extend(uint32_t crc, Slice data) { + return ::crc32c::Extend(crc, data.data(), data.size()); +} + +inline uint32_t crc32c(Slice data) { + return ::crc32c::Crc32c(data.data(), data.size()); +} + +#ifdef BE_TEST +// T21 test seam. The production crc32c()/crc32c_extend() above delegate to the +// bundled Google crc32c thirdparty (see commit d0416bb4129), which already runs a +// runtime-dispatched, hardware-accelerated and interleaved CRC32C -- so T21's +// "hardware interleaved CRC" goal is already met (and exceeded: that library adds +// a PCLMULQDQ fold a hand-rolled 3-way _mm_crc32_u64 lacks). Rather than regress +// that reuse, the reference sub-paths below let unit tests prove, byte-for-byte +// across all sizes/alignments, that the production path equals the canonical +// CRC32C and that the hardware path is engaged: +// * crc32c_slice8_extend -- portable software slice-by-8 (always available); +// * crc32c_hw_serial_extend -- serial SSE4.2 _mm_crc32 hardware path; +// * crc32c_hw3_extend -- 3-way interleaved SSE4.2 hardware path with a +// GF(2) shift-combine and a 1024-byte fall-back to +// the serial path (the algorithm T21 specifies). +// hw_serial/hw3 fall back to slice8 when SSE4.2 is absent. Each *_extend applies +// the standard ~crc pre/post inversion, so *_extend(0, d) == crc32c(d). The whole +// seam plus its static slice-by-8 table and startup CPUID probe are compiled out +// of release builds by this BE_TEST gate, so production carries no extra code. +// Pure functions with no shared mutable state (CONCURRENCY: N/A). +namespace detail { +uint32_t crc32c_slice8_extend(uint32_t crc, Slice data); +uint32_t crc32c_hw_serial_extend(uint32_t crc, Slice data); +uint32_t crc32c_hw3_extend(uint32_t crc, Slice data); +size_t crc32c_interleave_threshold(); +bool crc32c_has_hw(); +} // namespace detail +#endif // BE_TEST + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/pfor.cpp b/be/src/storage/index/snii/encoding/pfor.cpp new file mode 100644 index 00000000000000..dcc620fcc54c8f --- /dev/null +++ b/be/src/storage/index/snii/encoding/pfor.cpp @@ -0,0 +1,452 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/pfor.h" + +#include +#include +#include +#include +#include + +#include "storage/index/snii/common/slice.h" + +namespace doris::snii { +namespace { + +// Unaligned little-endian 64-bit load from a raw byte pointer (single +// instruction on x86; memcpy is the portable, UB-free spelling the compiler +// folds to a mov). +inline uint64_t load_u64_le(const uint8_t* p) { + uint64_t v; + std::memcpy(&v, p, sizeof(v)); +#if defined(__BIG_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + v = __builtin_bswap64(v); +#endif + return v; +} + +// TEST-ONLY seam backing doris::snii::testing::pfor_width_evals(). value_width() +// is the SINGLE per-value bit-width evaluation point on the encode path, so this +// counter equals the number of values processed per run -- the deterministic +// signal that the histogram path scans each value exactly once (vs the former +// O(maxw*n) re-scan). Compiled out entirely in non-test builds so production pays +// nothing; a relaxed atomic under BE_TEST keeps it data-race-free under TSAN. +#ifdef BE_TEST +std::atomic g_width_evals {0}; +#endif + +// Number of significant bits of v (its minimal bit_width); value_width(0) == 0 is +// preserved (matching the former bits_for). clz(0) is undefined behaviour, so +// v == 0 is mapped explicitly here -- never via clz(v | 1), which would mis-score +// 0 as width 1 and corrupt the histogram. +inline uint8_t value_width(uint32_t v) { +#ifdef BE_TEST + g_width_evals.fetch_add(1, std::memory_order_relaxed); +#endif + return v ? static_cast(32 - __builtin_clz(v)) : 0; +} + +// Choose the bit_width that minimizes total bytes (packed + exceptions), with +// exception cost estimated at ~6 bytes each. A single O(n) pass builds a bit-width +// histogram (recording each value's width into widths[] for the encoder to reuse), +// then an O(maxw) suffix-sum gives the exception count per candidate width -- +// replacing the former O(maxw*n) re-scan. The cost formula and the ascending-w +// strict-'<' tie-break are kept identical, so the chosen width (and hence every +// encoded byte) is unchanged. +uint8_t choose_width(const uint32_t* v, size_t n, uint8_t* widths) { + // hist[b] = #values whose bit-width == b. n is capped at kFrqBaseUnit (256) on + // the production path; uint32_t buckets keep a direct caller with a larger run + // correct without affecting the chosen width. + uint32_t hist[33] = {0}; + uint8_t maxw = 0; + for (size_t i = 0; i < n; ++i) { + const uint8_t b = value_width(v[i]); + widths[i] = b; + ++hist[b]; + maxw = std::max(b, maxw); + } + // suffix[k] = #values whose bit-width >= k, so the exceptions for candidate + // width w (values needing more than w bits) are exactly suffix[w + 1]. + size_t suffix[34] = {0}; + for (int k = 32; k >= 0; --k) { + suffix[k] = suffix[k + 1] + hist[k]; + } + uint8_t best = maxw; + size_t best_cost = SIZE_MAX; + for (uint8_t w = 0; w <= maxw; ++w) { + const size_t exc = suffix[w + 1]; + const size_t cost = (static_cast(w) * n + 7) / 8 + exc * 6; + if (cost < best_cost) { + best_cost = cost; + best = w; + } + } + return best; +} + +uint32_t low_mask(uint8_t w) { + return (w >= 32) ? 0xFFFFFFFFU : ((1U << w) - 1U); +} + +// Bit-pack the low w bits of each value, writing 0 at exception positions +// (widths[i] > w) instead of the value's low bits. This is byte-identical to +// packing a copy in which those slots were pre-zeroed (the former `low[i] = 0` +// placeholder), so the encoder no longer materializes that copy. +void bitpack_masked(const uint32_t* v, const uint8_t* widths, size_t n, uint8_t w, ByteSink* out) { + if (w == 0) { + return; + } + // Pre-size for the exact packed byte count (ceil(w*n/8)) so the per-byte put_u8 + // loop below never reallocates mid-pack. ByteSink::reserve is RELATIVE to out's + // current size, and `out` is reused across the PFOR runs of one region, so this + // accumulates run-to-run instead of no-op'ing. Capacity-only: the packed bytes + // and their values are byte-identical to the un-reserved path. + const size_t packed = (static_cast(w) * n + 7) / 8; + out->reserve(packed); + const uint32_t mask = low_mask(w); + uint64_t acc = 0; + int filled = 0; + for (size_t i = 0; i < n; ++i) { + const uint32_t lo = (widths[i] > w) ? 0U : (v[i] & mask); + acc |= static_cast(lo) << filled; + filled += w; + while (filled >= 8) { + out->put_u8(static_cast(acc)); + acc >>= 8; + filled -= 8; + } + } + if (filled > 0) { + out->put_u8(static_cast(acc)); + } +} + +void bitunpack_tail(const uint8_t* base, size_t packed, size_t n, uint8_t w, size_t i, + uint64_t mask, uint32_t* out) { + for (; i < n; ++i) { + const size_t bit_off = static_cast(w) * i; + const size_t byte_off = bit_off >> 3; + uint64_t word = 0; + for (size_t b = byte_off; b < packed && b < byte_off + 8; ++b) { + word |= static_cast(base[b]) << ((b - byte_off) * 8); + } + out[i] = static_cast((word >> (bit_off & 7)) & mask); + } +} + +void bitunpack_w1(const uint8_t* base, size_t n, uint32_t* out) { + size_t i = 0; + size_t byte = 0; + for (; i + 8 <= n; i += 8, ++byte) { + const uint8_t v = base[byte]; + out[i] = v & 1U; + out[i + 1] = (v >> 1) & 1U; + out[i + 2] = (v >> 2) & 1U; + out[i + 3] = (v >> 3) & 1U; + out[i + 4] = (v >> 4) & 1U; + out[i + 5] = (v >> 5) & 1U; + out[i + 6] = (v >> 6) & 1U; + out[i + 7] = (v >> 7) & 1U; + } + if (i < n) { + const uint8_t v = base[byte]; + for (uint8_t bit = 0; i < n; ++i, ++bit) { + out[i] = (v >> bit) & 1U; + } + } +} + +void bitunpack_w2(const uint8_t* base, size_t n, uint32_t* out) { + size_t i = 0; + size_t byte = 0; + for (; i + 4 <= n; i += 4, ++byte) { + const uint8_t v = base[byte]; + out[i] = v & 3U; + out[i + 1] = (v >> 2) & 3U; + out[i + 2] = (v >> 4) & 3U; + out[i + 3] = (v >> 6) & 3U; + } + if (i < n) { + const uint8_t v = base[byte]; + for (uint8_t shift = 0; i < n; ++i, shift += 2) { + out[i] = (v >> shift) & 3U; + } + } +} + +void bitunpack_w3(const uint8_t* base, size_t packed, size_t n, uint32_t* out) { + size_t i = 0; + size_t byte = 0; + for (; i + 8 <= n; i += 8, byte += 3) { + const uint32_t b0 = base[byte]; + const uint32_t b1 = base[byte + 1]; + const uint32_t b2 = base[byte + 2]; + out[i] = b0 & 7U; + out[i + 1] = (b0 >> 3) & 7U; + out[i + 2] = ((b0 >> 6) | (b1 << 2)) & 7U; + out[i + 3] = (b1 >> 1) & 7U; + out[i + 4] = (b1 >> 4) & 7U; + out[i + 5] = ((b1 >> 7) | (b2 << 1)) & 7U; + out[i + 6] = (b2 >> 2) & 7U; + out[i + 7] = (b2 >> 5) & 7U; + } + bitunpack_tail(base, packed, n, 3, i, 7U, out); +} + +void bitunpack_w4(const uint8_t* base, size_t n, uint32_t* out) { + size_t i = 0; + size_t byte = 0; + for (; i + 2 <= n; i += 2, ++byte) { + const uint8_t v = base[byte]; + out[i] = v & 15U; + out[i + 1] = (v >> 4) & 15U; + } + if (i < n) { + out[i] = base[byte] & 15U; + } +} + +void bitunpack_w5(const uint8_t* base, size_t packed, size_t n, uint32_t* out) { + size_t i = 0; + size_t byte = 0; + for (; i + 8 <= n; i += 8, byte += 5) { + const uint32_t b0 = base[byte]; + const uint32_t b1 = base[byte + 1]; + const uint32_t b2 = base[byte + 2]; + const uint32_t b3 = base[byte + 3]; + const uint32_t b4 = base[byte + 4]; + out[i] = b0 & 31U; + out[i + 1] = ((b0 >> 5) | (b1 << 3)) & 31U; + out[i + 2] = (b1 >> 2) & 31U; + out[i + 3] = ((b1 >> 7) | (b2 << 1)) & 31U; + out[i + 4] = ((b2 >> 4) | (b3 << 4)) & 31U; + out[i + 5] = (b3 >> 1) & 31U; + out[i + 6] = ((b3 >> 6) | (b4 << 2)) & 31U; + out[i + 7] = (b4 >> 3) & 31U; + } + bitunpack_tail(base, packed, n, 5, i, 31U, out); +} + +void bitunpack_w6(const uint8_t* base, size_t packed, size_t n, uint32_t* out) { + size_t i = 0; + size_t byte = 0; + for (; i + 4 <= n; i += 4, byte += 3) { + const uint32_t b0 = base[byte]; + const uint32_t b1 = base[byte + 1]; + const uint32_t b2 = base[byte + 2]; + out[i] = b0 & 63U; + out[i + 1] = ((b0 >> 6) | (b1 << 2)) & 63U; + out[i + 2] = ((b1 >> 4) | (b2 << 4)) & 63U; + out[i + 3] = (b2 >> 2) & 63U; + } + bitunpack_tail(base, packed, n, 6, i, 63U, out); +} + +void bitunpack_w7(const uint8_t* base, size_t packed, size_t n, uint32_t* out) { + size_t i = 0; + size_t byte = 0; + for (; i + 8 <= n; i += 8, byte += 7) { + const uint32_t b0 = base[byte]; + const uint32_t b1 = base[byte + 1]; + const uint32_t b2 = base[byte + 2]; + const uint32_t b3 = base[byte + 3]; + const uint32_t b4 = base[byte + 4]; + const uint32_t b5 = base[byte + 5]; + const uint32_t b6 = base[byte + 6]; + out[i] = b0 & 127U; + out[i + 1] = ((b0 >> 7) | (b1 << 1)) & 127U; + out[i + 2] = ((b1 >> 6) | (b2 << 2)) & 127U; + out[i + 3] = ((b2 >> 5) | (b3 << 3)) & 127U; + out[i + 4] = ((b3 >> 4) | (b4 << 4)) & 127U; + out[i + 5] = ((b4 >> 3) | (b5 << 5)) & 127U; + out[i + 6] = ((b5 >> 2) | (b6 << 6)) & 127U; + out[i + 7] = (b6 >> 1) & 127U; + } + bitunpack_tail(base, packed, n, 7, i, 127U, out); +} + +void bitunpack_w8(const uint8_t* base, size_t n, uint32_t* out) { + for (size_t i = 0; i < n; ++i) { + out[i] = base[i]; + } +} + +void bitunpack_generic(const uint8_t* base, size_t packed, size_t n, uint8_t w, uint32_t* out) { + const uint64_t mask = low_mask(w); + size_t i = 0; + if (packed >= 8) { + const size_t last_safe_byte = packed - 8; + for (; i < n; ++i) { + const size_t bit_off = static_cast(w) * i; + const size_t byte_off = bit_off >> 3; + if (byte_off > last_safe_byte) { + break; + } + out[i] = static_cast((load_u64_le(base + byte_off) >> (bit_off & 7)) & mask); + } + } + bitunpack_tail(base, packed, n, w, i, mask, out); +} + +Status bitunpack(ByteSource* src, size_t n, uint8_t w, uint32_t* out) { + if (w == 0) { + std::memset(out, 0, n * sizeof(uint32_t)); + return Status::OK(); + } + // Pull the packed run once and unpack from the contiguous slice; this keeps + // the hot decode path free of per-byte ByteSource calls. + const size_t packed = (static_cast(w) * n + 7) / 8; + Slice buf; + RETURN_IF_ERROR(src->get_bytes(packed, &buf)); + const uint8_t* base = buf.data(); + + switch (w) { + case 1: + bitunpack_w1(base, n, out); + break; + case 2: + bitunpack_w2(base, n, out); + break; + case 3: + bitunpack_w3(base, packed, n, out); + break; + case 4: + bitunpack_w4(base, n, out); + break; + case 5: + bitunpack_w5(base, packed, n, out); + break; + case 6: + bitunpack_w6(base, packed, n, out); + break; + case 7: + bitunpack_w7(base, packed, n, out); + break; + case 8: + bitunpack_w8(base, n, out); + break; + default: + bitunpack_generic(base, packed, n, w, out); + break; + } + return Status::OK(); +} + +} // namespace + +namespace testing { +// Test-only op-count seam; see pfor.h. Reports/resets the per-value bit-width +// evaluation counter, and is a no-op in non-test builds where the counter is +// compiled out. +uint64_t pfor_width_evals() { +#ifdef BE_TEST + return g_width_evals.load(std::memory_order_relaxed); +#else + return 0; +#endif +} +void reset_pfor_width_evals() { +#ifdef BE_TEST + g_width_evals.store(0, std::memory_order_relaxed); +#endif +} +} // namespace testing + +void pfor_encode(const uint32_t* values, size_t n, ByteSink* out) { + // n is hard-capped at kFrqBaseUnit (256) by encode_pfor_runs, so the stack + // buffer is used on every production call; the heap fallback only guards a + // direct caller passing a larger run. + uint8_t widths_stack[256]; + std::vector widths_heap; + uint8_t* widths = widths_stack; + if (n > sizeof(widths_stack)) { + widths_heap.resize(n); + widths = widths_heap.data(); + } + + const uint8_t w = choose_width(values, n, widths); + out->put_u8(w); + + // Count exceptions for the varint header by reusing the cached per-value + // widths -- no further bit-width evaluation. + uint32_t n_exc = 0; + for (size_t i = 0; i < n; ++i) { + n_exc += (widths[i] > w); + } + out->put_varint32(n_exc); + + // Pack the low w bits, writing 0 at exception slots (byte-identical to the + // former zeroed-copy approach) so no separate `low` buffer is materialized. + bitpack_masked(values, widths, n, w, out); + + // Exception table: (index_delta, full_value) in ascending index order. + uint32_t prev = 0; + for (size_t i = 0; i < n; ++i) { + if (widths[i] > w) { + out->put_varint32(static_cast(i) - prev); + out->put_varint32(values[i]); + prev = static_cast(i); + } + } +} + +Status pfor_decode(ByteSource* src, size_t n, uint32_t* out) { + uint8_t w; + RETURN_IF_ERROR(src->get_u8(&w)); + uint32_t n_exc; + RETURN_IF_ERROR(src->get_varint32(&n_exc)); + RETURN_IF_ERROR(bitunpack(src, n, w, out)); + uint32_t idx = 0; + for (uint32_t i = 0; i < n_exc; ++i) { + uint32_t d, val; + RETURN_IF_ERROR(src->get_varint32(&d)); + RETURN_IF_ERROR(src->get_varint32(&val)); + idx += d; + if (idx >= n) { + return Status::Error( + "pfor exception index out of range"); + } + out[idx] = val; + } + return Status::OK(); +} + +Status pfor_skip(ByteSource* src, size_t n) { + uint8_t w = 0; + RETURN_IF_ERROR(src->get_u8(&w)); + uint32_t n_exc = 0; + RETURN_IF_ERROR(src->get_varint32(&n_exc)); + const size_t packed = (static_cast(w) * n + 7) / 8; + Slice unused; + RETURN_IF_ERROR(src->get_bytes(packed, &unused)); + uint32_t idx = 0; + for (uint32_t i = 0; i < n_exc; ++i) { + uint32_t d = 0; + uint32_t val = 0; + RETURN_IF_ERROR(src->get_varint32(&d)); + RETURN_IF_ERROR(src->get_varint32(&val)); + idx += d; + if (idx >= n) { + return Status::Error( + "pfor exception index out of range"); + } + } + return Status::OK(); +} + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/pfor.h b/be/src/storage/index/snii/encoding/pfor.h new file mode 100644 index 00000000000000..709d35748249f1 --- /dev/null +++ b/be/src/storage/index/snii/encoding/pfor.h @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/byte_source.h" + +namespace doris::snii { + +// PFOR integer block encoder/decoder (unsigned uint32 array). +// Encoded layout: [u8 bit_width][varint n_exceptions][bit-packed low +// bits][exception table]. Selects the bit_width that minimizes total byte size; +// values exceeding it go into the exception table (index_delta, full_value). +// delta/zigzag is handled by the upper layer (.frq window); PFOR only processes +// unsigned integer arrays. +void pfor_encode(const uint32_t* values, size_t n, ByteSink* out); +Status pfor_decode(ByteSource* src, size_t n, uint32_t* out); +Status pfor_skip(ByteSource* src, size_t n); + +} // namespace doris::snii + +// Test-only instrumentation seam (mirrors the dict-block decode-counter pattern). +// pfor_width_evals() returns a process-global count of per-value bit-width +// evaluations performed by pfor_encode since the last reset -- one per +// value_width() call, the single evaluation point on the encode path. Deterministic +// perf tests assert it equals the number of encoded values per run, proving the +// histogram path scans each value exactly once (vs the former O(maxw*n) re-scan). +// Compiled to a no-op in non-test builds; reset between tests. +namespace doris::snii::testing { + +uint64_t pfor_width_evals(); +void reset_pfor_width_evals(); + +} // namespace doris::snii::testing diff --git a/be/src/storage/index/snii/encoding/section_framer.cpp b/be/src/storage/index/snii/encoding/section_framer.cpp new file mode 100644 index 00000000000000..e67b618758dae8 --- /dev/null +++ b/be/src/storage/index/snii/encoding/section_framer.cpp @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/section_framer.h" + +#include "storage/index/snii/encoding/crc32c.h" + +namespace doris::snii { + +void SectionFramer::write(ByteSink& sink, uint8_t section_type, Slice payload) { + // Single-copy framing: write [type][varint64 len][payload] straight into the + // target sink, then crc exactly those bytes. view() is taken AFTER the payload + // and BEFORE the crc, so subslice([start, framed_len)) is over a settled, + // contiguous buffer with no pending realloc/aliasing. Byte-identical to the + // former temp-ByteSink assembly, minus one heap alloc + one payload copy. + const size_t start = sink.size(); + sink.put_u8(section_type); + sink.put_varint64(payload.size()); + sink.put_bytes(payload); + const size_t framed_len = sink.size() - start; + const uint32_t crc = crc32c(sink.view().subslice(start, framed_len)); + sink.put_fixed32(crc); +} + +Status SectionFramer::read(ByteSource& src, FramedSection* out) { + size_t start = src.position(); + uint8_t type; + RETURN_IF_ERROR(src.get_u8(&type)); + uint64_t len; + RETURN_IF_ERROR(src.get_varint64(&len)); + Slice payload; + RETURN_IF_ERROR(src.get_bytes(static_cast(len), &payload)); + size_t framed_len = src.position() - start; + uint32_t stored; + RETURN_IF_ERROR(src.get_fixed32(&stored)); + if (crc32c(src.slice_from(start, framed_len)) != stored) { + return Status::Error( + "section crc mismatch"); + } + out->type = type; + out->payload = payload; + return Status::OK(); +} + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/section_framer.h b/be/src/storage/index/snii/encoding/section_framer.h new file mode 100644 index 00000000000000..9a248381ea97b6 --- /dev/null +++ b/be/src/storage/index/snii/encoding/section_framer.h @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/byte_source.h" + +namespace doris::snii { + +// A framed section: type + payload view. +struct FramedSection { + uint8_t type = 0; + Slice payload; +}; + +// Unified section framing: [u8 type][varint64 len][payload][fixed32 crc32c(type+len+payload)]. +// All full-format sections reuse this encode/checksum path to avoid ad-hoc hand-assembly. +// Unknown optional sections are dispatched by the caller based on type; read still verifies the CRC and skips the payload. +class SectionFramer { +public: + static void write(ByteSink& sink, uint8_t section_type, Slice payload); + static Status read(ByteSource& src, FramedSection* out); +}; + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/varint.cpp b/be/src/storage/index/snii/encoding/varint.cpp new file mode 100644 index 00000000000000..a53a08de6b8d3b --- /dev/null +++ b/be/src/storage/index/snii/encoding/varint.cpp @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/varint.h" + +namespace doris::snii { + +size_t varint_len(uint64_t v) { + size_t n = 1; + while (v >= 0x80) { + v >>= 7; + ++n; + } + return n; +} + +size_t encode_varint64(uint64_t v, uint8_t* out) { + size_t i = 0; + while (v >= 0x80) { + out[i++] = static_cast(v) | 0x80; + v >>= 7; + } + out[i++] = static_cast(v); + return i; +} + +size_t encode_varint32(uint32_t v, uint8_t* out) { + return encode_varint64(v, out); +} + +Status decode_varint64(const uint8_t* p, const uint8_t* end, uint64_t* v, const uint8_t** next) { + uint64_t result = 0; + int shift = 0; + while (p < end) { + uint8_t b = *p++; + result |= static_cast(b & 0x7F) << shift; + if ((b & 0x80) == 0) { + *v = result; + *next = p; + return Status::OK(); + } + shift += 7; + if (shift >= 64) + return Status::Error( + "varint64 overflow"); + } + return Status::Error("varint truncated"); +} + +Status decode_varint32(const uint8_t* p, const uint8_t* end, uint32_t* v, const uint8_t** next) { + uint64_t tmp; + RETURN_IF_ERROR(decode_varint64(p, end, &tmp, next)); + if (tmp > 0xFFFFFFFFu) + return Status::Error("varint32 overflow"); + *v = static_cast(tmp); + return Status::OK(); +} + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/varint.h b/be/src/storage/index/snii/encoding/varint.h new file mode 100644 index 00000000000000..978812ea6e66f6 --- /dev/null +++ b/be/src/storage/index/snii/encoding/varint.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" + +namespace doris::snii { + +// LEB128 variable-length integer encoding + zigzag. out buffer must be >=10 bytes; returns number of bytes written. +size_t varint_len(uint64_t v); +size_t encode_varint32(uint32_t v, uint8_t* out); +size_t encode_varint64(uint64_t v, uint8_t* out); + +// Decode a varint from the range [p, end); on success *next points to the next byte after the consumed input. +Status decode_varint32(const uint8_t* p, const uint8_t* end, uint32_t* v, const uint8_t** next); +Status decode_varint64(const uint8_t* p, const uint8_t* end, uint64_t* v, const uint8_t** next); + +inline uint64_t zigzag_encode(int64_t v) { + return (static_cast(v) << 1) ^ static_cast(v >> 63); +} +inline int64_t zigzag_decode(uint64_t v) { + return static_cast(v >> 1) ^ -static_cast(v & 1); +} + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/zstd_codec.cpp b/be/src/storage/index/snii/encoding/zstd_codec.cpp new file mode 100644 index 00000000000000..6c01c0cb2ebdef --- /dev/null +++ b/be/src/storage/index/snii/encoding/zstd_codec.cpp @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/zstd_codec.h" + +#include + +#include + +#include "storage/index/snii/common/uninitialized_buffer.h" + +namespace doris::snii { + +Status zstd_compress(Slice input, int level, std::vector* out) { + size_t bound = ZSTD_compressBound(input.size()); + out->resize(bound); + size_t n = ZSTD_compress(out->data(), bound, input.data(), input.size(), level); + if (ZSTD_isError(n)) { + return Status::Error(std::string("zstd compress: ") + + ZSTD_getErrorName(n)); + } + out->resize(n); + return Status::OK(); +} + +Status zstd_decompress(Slice input, size_t expected_uncomp_len, std::vector* out) { + // Sized then fully overwritten by ZSTD_decompress (length-checked below). + resize_uninitialized(*out, expected_uncomp_len); + size_t n = ZSTD_decompress(out->data(), expected_uncomp_len, input.data(), input.size()); + if (ZSTD_isError(n)) { + return Status::Error( + std::string("zstd decompress: ") + ZSTD_getErrorName(n)); + } + if (n != expected_uncomp_len) { + return Status::Error( + "zstd decompressed length mismatch"); + } + return Status::OK(); +} + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/encoding/zstd_codec.h b/be/src/storage/index/snii/encoding/zstd_codec.h new file mode 100644 index 00000000000000..c8291ce024fca7 --- /dev/null +++ b/be/src/storage/index/snii/encoding/zstd_codec.h @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" + +namespace doris::snii { + +// Thin ZSTD wrapper. Used for compressing large payloads such as .prx windows. Decompression requires the caller to supply the original uncompressed length (from the block header). +Status zstd_compress(Slice input, int level, std::vector* out); +Status zstd_decompress(Slice input, size_t expected_uncomp_len, std::vector* out); + +} // namespace doris::snii diff --git a/be/src/storage/index/snii/format/bootstrap_header.cpp b/be/src/storage/index/snii/format/bootstrap_header.cpp new file mode 100644 index 00000000000000..6cdc1869385bec --- /dev/null +++ b/be/src/storage/index/snii/format/bootstrap_header.cpp @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/bootstrap_header.h" + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" + +namespace doris::snii::format { + +namespace { + +// Number of bytes covered by header_checksum: everything except the trailing +// crc32c. +constexpr size_t kChecksumCoverage = kBootstrapHeaderSize - 4; + +// Writes all fixed fields except the trailing checksum. Field order is the +// on-disk contract; reuse ByteSink fixed-width primitives, never hand-assemble +// bytes. +void encode_fields(const BootstrapHeader& header, ByteSink* sink) { + sink->put_fixed32(header.magic); + sink->put_fixed32((static_cast(header.min_reader_version) << 16) | + header.format_version); + sink->put_fixed32(header.flags); + sink->put_fixed32(kBootstrapHeaderSize); // header_length is always derived + sink->put_u8(header.tail_pointer_size); +} + +} // namespace + +Status encode_bootstrap_header(const BootstrapHeader& header, ByteSink* sink) { + if (sink == nullptr) { + return Status::Error("bootstrap_header: null sink"); + } + ByteSink fields; + encode_fields(header, &fields); + const uint32_t checksum = crc32c(fields.view()); + sink->put_bytes(fields.view()); + sink->put_fixed32(checksum); + return Status::OK(); +} + +Status decode_bootstrap_header(Slice data, BootstrapHeader* out) { + if (out == nullptr) { + return Status::Error("bootstrap_header: null out"); + } + // Reject any size other than the exact fixed header: short input is + // truncation, longer input means stray trailing bytes the parser would + // otherwise ignore. + if (data.size() != kBootstrapHeaderSize) { + return Status::Error( + "bootstrap_header: wrong header size"); + } + + ByteSource src(data); + uint32_t magic = 0; + uint32_t version_pair = 0; + uint32_t flags = 0; + uint32_t header_length = 0; + uint8_t tail_pointer_size = 0; + uint32_t stored_checksum = 0; + RETURN_IF_ERROR(src.get_fixed32(&magic)); + RETURN_IF_ERROR(src.get_fixed32(&version_pair)); + RETURN_IF_ERROR(src.get_fixed32(&flags)); + RETURN_IF_ERROR(src.get_fixed32(&header_length)); + RETURN_IF_ERROR(src.get_u8(&tail_pointer_size)); + RETURN_IF_ERROR(src.get_fixed32(&stored_checksum)); + + if (magic != kContainerMagic) { + return Status::Error( + "bootstrap_header: bad container magic"); + } + const uint32_t computed = crc32c(data.subslice(0, kChecksumCoverage)); + if (computed != stored_checksum) { + return Status::Error( + "bootstrap_header: checksum mismatch"); + } + + const auto min_reader_version = static_cast((version_pair >> 16) & 0xFFFFu); + const auto format_version = static_cast(version_pair & 0xFFFFu); + if (format_version != kFormatVersion) { + return Status::Error( + "bootstrap_header: unsupported container format_version"); + } + if (min_reader_version > kFormatVersion) { + return Status::Error( + "bootstrap_header: container requires a newer reader version"); + } + + out->magic = magic; + out->format_version = format_version; + out->min_reader_version = min_reader_version; + out->flags = flags; + out->header_length = header_length; + out->tail_pointer_size = tail_pointer_size; + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/bootstrap_header.h b/be/src/storage/index/snii/format/bootstrap_header.h new file mode 100644 index 00000000000000..6a8c0e8083d3ac --- /dev/null +++ b/be/src/storage/index/snii/format/bootstrap_header.h @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::format { + +// Fixed container header at the very start of a {rowset_id}_{seg_id}.idx file. +// Identifies the SNII container and carries basic compatibility info so a +// reader can fail fast before touching any streamed section or the tail meta +// region. +// +// On-disk layout (all multi-byte fields little-endian, fixed width; NOT framed +// by SectionFramer because it must be parseable without prior knowledge of the +// file): +// u32 magic == kContainerMagic +// u16 format_version == kFormatVersion +// u16 min_reader_version readers with kFormatVersion < this MUST refuse to +// read u32 flags container-level feature flags u32 +// header_length total bytes of this header including the checksum u8 +// tail_pointer_size size of the fixed tail pointer at EOF (hint for the +// reader) u32 header_checksum crc32c over all preceding header bytes +struct BootstrapHeader { + uint32_t magic = kContainerMagic; + uint16_t format_version = kFormatVersion; + uint16_t min_reader_version = kMinReaderVersion; + uint32_t flags = 0; + uint32_t header_length = 0; + uint8_t tail_pointer_size = 0; +}; + +// Total fixed on-disk size of the header, including the trailing crc32c. +inline constexpr uint32_t kBootstrapHeaderSize = + 4 /*magic*/ + 2 /*format_version*/ + 2 /*min_reader_version*/ + 4 /*flags*/ + + 4 /*header_length*/ + 1 /*tail_pointer_size*/ + 4 /*header_checksum*/; + +// Serializes the header to sink: writes header_length = kBootstrapHeaderSize +// and appends a crc32c over all preceding bytes. The caller's header_length +// field is ignored on input (it is always derived). Returns OK. +Status encode_bootstrap_header(const BootstrapHeader& header, ByteSink* sink); + +// Parses and validates a bootstrap header from the front of data. +// - too short / trailing bytes beyond the fixed header -> kCorruption +// - magic != kContainerMagic -> kCorruption +// - checksum mismatch -> kCorruption +// - format_version != kFormatVersion -> kUnsupported +// - min_reader_version > kFormatVersion -> kUnsupported +Status decode_bootstrap_header(Slice data, BootstrapHeader* out); + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/bsbf.cpp b/be/src/storage/index/snii/format/bsbf.cpp new file mode 100644 index 00000000000000..d214a99e745ae9 --- /dev/null +++ b/be/src/storage/index/snii/format/bsbf.cpp @@ -0,0 +1,257 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/bsbf.h" + +#include + +#include "storage/index/snii/encoding/crc32c.h" + +#if defined(__x86_64__) || defined(_M_X64) +#include +#define SNII_BSBF_X86 1 +#endif + +#define XXH_INLINE_ALL +#include "xxhash.h" + +namespace doris::snii::format { + +const uint32_t kBsbfSalt[kBsbfBitsSetPerBlock] = {0x47b6137bU, 0x44974d91U, 0x8824ad5bU, + 0xa2b7289dU, 0x705495c7U, 0x2df1424bU, + 0x9efc4947U, 0x5c6bfb31U}; + +namespace { + +void store_le32(uint8_t* p, uint32_t v) { + p[0] = static_cast(v); + p[1] = static_cast(v >> 8); + p[2] = static_cast(v >> 16); + p[3] = static_cast(v >> 24); +} +uint32_t load_le32(const uint8_t* p) { + return static_cast(p[0]) | (static_cast(p[1]) << 8) | + (static_cast(p[2]) << 16) | (static_cast(p[3]) << 24); +} + +bool cpu_has_avx2() { +#if defined(SNII_BSBF_X86) + static const bool v = __builtin_cpu_supports("avx2"); + return v; +#else + return false; +#endif +} + +// --- scalar kernels --- +inline void masks_scalar(uint32_t key, uint32_t m[8]) { + for (int i = 0; i < 8; ++i) m[i] = 1u << ((key * kBsbfSalt[i]) >> 27); +} +bool block_contains_scalar(uint64_t hash, const uint8_t* block) { + const uint32_t* w = reinterpret_cast(block); // LE + uint32_t m[8]; + masks_scalar(static_cast(hash), m); + for (int i = 0; i < 8; ++i) + if ((load_le32(reinterpret_cast(w + i)) & m[i]) != m[i]) return false; + return true; +} +void insert_scalar(uint32_t* words, uint32_t block, uint32_t key) { + uint32_t m[8]; + masks_scalar(key, m); + for (int i = 0; i < 8; ++i) words[block * 8 + i] |= m[i]; +} +bool find_scalar(const uint32_t* words, uint32_t block, uint32_t key) { + uint32_t m[8]; + masks_scalar(key, m); + for (int i = 0; i < 8; ++i) + if ((words[block * 8 + i] & m[i]) != m[i]) return false; + return true; +} + +#if defined(SNII_BSBF_X86) +// --- AVX2 kernels: a 256-bit block is one YMM register --- +__attribute__((target("avx2"))) __m256i mask_avx2(uint32_t key) { + const __m256i salt = + _mm256_setr_epi32(static_cast(kBsbfSalt[0]), static_cast(kBsbfSalt[1]), + static_cast(kBsbfSalt[2]), static_cast(kBsbfSalt[3]), + static_cast(kBsbfSalt[4]), static_cast(kBsbfSalt[5]), + static_cast(kBsbfSalt[6]), static_cast(kBsbfSalt[7])); + const __m256i prod = _mm256_mullo_epi32(_mm256_set1_epi32(static_cast(key)), salt); + const __m256i shifts = _mm256_srli_epi32(prod, 27); // top 5 bits -> 0..31 + return _mm256_sllv_epi32(_mm256_set1_epi32(1), shifts); +} +__attribute__((target("avx2"))) bool block_contains_avx2(uint64_t hash, const uint8_t* block) { + const __m256i m = mask_avx2(static_cast(hash)); + const __m256i b = _mm256_loadu_si256(reinterpret_cast(block)); + return _mm256_testc_si256(b, m) != 0; // (~b & m) == 0 -> b contains m +} +__attribute__((target("avx2"))) void insert_avx2(uint32_t* words, uint32_t block, uint32_t key) { + __m256i* p = reinterpret_cast<__m256i*>(words + block * 8); + _mm256_storeu_si256(p, _mm256_or_si256(_mm256_loadu_si256(p), mask_avx2(key))); +} +__attribute__((target("avx2"))) bool find_avx2(const uint32_t* words, uint32_t block, + uint32_t key) { + const __m256i m = mask_avx2(key); + const __m256i b = _mm256_loadu_si256(reinterpret_cast(words + block * 8)); + return _mm256_testc_si256(b, m) != 0; +} +#endif + +} // namespace + +uint64_t bsbf_hash(std::string_view term) { + return XXH64(term.data(), term.size(), /*seed=*/0); +} + +uint32_t bsbf_optimal_num_bytes(uint32_t ndv, double fpp) { + // Parquet OptimalNumOfBits, then >>3 for bytes. + const double m = -8.0 * ndv / std::log(1 - std::pow(fpp, 1.0 / 8)); + uint32_t num_bits; + if (m < 0 || m > static_cast(kBsbfMaxBytes) * 8) { + num_bits = kBsbfMaxBytes << 3; + } else { + num_bits = static_cast(m); + } + if (num_bits < (kBsbfMinBytes << 3)) num_bits = kBsbfMinBytes << 3; + // G16-g: round up to the 32-byte block only, NOT to the next power of 2. + // The block index is fastrange ((h >> 32) * num_blocks >> 32, bsbf.h), + // which is uniform for ANY block count -- the old power-of-2 rounding was + // a sizing convention, not an addressing requirement, and cost up to ~2x + // bitset bytes (e.g. a 31M-term segment: 40.3 MB optimal -> 64 MB rounded) + // while silently over-delivering on fpp. Exact sizing keeps the fpp at + // the kBsbfFpp design point. + const uint32_t block_bits = kBsbfBytesPerBlock << 3; + num_bits = (num_bits + block_bits - 1) / block_bits * block_bits; + if (num_bits > (kBsbfMaxBytes << 3)) num_bits = kBsbfMaxBytes << 3; + return num_bits >> 3; +} + +bool bsbf_block_contains(uint64_t hash, const uint8_t block[kBsbfBytesPerBlock]) { +#if defined(SNII_BSBF_X86) + if (cpu_has_avx2()) return block_contains_avx2(hash, block); +#endif + return block_contains_scalar(hash, block); +} + +Status BsbfBuilder::create(uint32_t ndv, double fpp, BsbfBuilder* out) { + if (out == nullptr) return Status::Error("bsbf: null out"); + if (!(fpp > 0.0 && fpp < 1.0)) + return Status::Error("bsbf: fpp out of (0,1)"); + if (ndv == 0) ndv = 1; + out->num_bytes_ = bsbf_optimal_num_bytes(ndv, fpp); + out->num_blocks_ = out->num_bytes_ / kBsbfBytesPerBlock; + out->ndv_ = ndv; + out->words_.assign(out->num_bytes_ / 4, 0u); + return Status::OK(); +} + +void BsbfBuilder::insert(uint64_t hash) { + const uint32_t block = bsbf_block_index(hash, num_blocks_); + const uint32_t key = static_cast(hash); +#if defined(SNII_BSBF_X86) + if (cpu_has_avx2()) { + insert_avx2(words_.data(), block, key); + return; + } +#endif + insert_scalar(words_.data(), block, key); +} + +bool BsbfBuilder::maybe_contains(uint64_t hash) const { + const uint32_t block = bsbf_block_index(hash, num_blocks_); + const uint32_t key = static_cast(hash); +#if defined(SNII_BSBF_X86) + if (cpu_has_avx2()) return find_avx2(words_.data(), block, key); +#endif + return find_scalar(words_.data(), block, key); +} + +Status BsbfBuilder::serialize(ByteSink* sink) const { + if (sink == nullptr) + return Status::Error("bsbf: null sink"); + if (num_bytes_ == 0) + return Status::Error("bsbf: not built"); + uint8_t hdr[kBsbfHeaderSize] = {0}; + hdr[0] = 'B'; + hdr[1] = 'S'; + hdr[2] = 'B'; + hdr[3] = 'F'; + hdr[4] = 1; // version + hdr[5] = 0; // hash strategy: XXH64 seed 0 + hdr[6] = 0; // index strategy: fastrange + hdr[7] = 0; // pad + store_le32(hdr + 8, num_bytes_); + store_le32(hdr + 12, num_blocks_); + store_le32(hdr + 16, ndv_); + store_le32(hdr + 20, crc32c(Slice(hdr, 20))); // header crc over [0,20) + const uint8_t* bits = reinterpret_cast(words_.data()); + store_le32(hdr + 24, crc32c(Slice(bits, num_bytes_))); // bitset crc + sink->put_bytes(Slice(hdr, kBsbfHeaderSize)); + sink->put_bytes(Slice(bits, num_bytes_)); // contiguous, uncompressed, LE + return Status::OK(); +} + +Status BsbfHeader::parse(Slice h, uint64_t section_base, BsbfHeader* out) { + if (out == nullptr) return Status::Error("bsbf: null out"); + if (h.size() < kBsbfHeaderSize) + return Status::Error("bsbf: short header"); + const uint8_t* p = h.data(); + if (p[0] != 'B' || p[1] != 'S' || p[2] != 'B' || p[3] != 'F') + return Status::Error("bsbf: bad magic"); + if (p[4] != 1) + return Status::Error("bsbf: bad version"); + if (p[5] != 0) + return Status::Error( + "bsbf: unsupported hash strategy"); + if (p[6] != 0) + return Status::Error( + "bsbf: unsupported index strategy"); + if (crc32c(Slice(p, 20)) != load_le32(p + 20)) + return Status::Error( + "bsbf: header crc mismatch"); + const uint32_t nb = load_le32(p + 8); + const uint32_t nblk = load_le32(p + 12); + // G16-g: num_bytes only needs 32-byte block alignment -- the fastrange + // block index is uniform for any block count. Power-of-2 sizes (all + // pre-G16-g filters) remain valid as a subset. + if (nb < kBsbfMinBytes || nb > kBsbfMaxBytes || nb % kBsbfBytesPerBlock != 0) + return Status::Error( + "bsbf: num_bytes out of range or not block-aligned"); + if (nblk != nb / kBsbfBytesPerBlock) + return Status::Error( + "bsbf: num_blocks mismatch"); + out->num_bytes = nb; + out->num_blocks = nblk; + out->bitset_crc = load_le32(p + 24); + out->bitset_base = section_base + kBsbfHeaderSize; + return Status::OK(); +} + +Status bsbf_probe(io::FileReader* reader, const BsbfHeader& header, uint64_t hash, + bool* maybe_present) { + if (reader == nullptr || maybe_present == nullptr) + return Status::Error("bsbf: null arg"); + std::vector blk; + RETURN_IF_ERROR(reader->read_at(header.block_offset(hash), kBsbfBytesPerBlock, &blk)); + if (blk.size() < kBsbfBytesPerBlock) + return Status::Error( + "bsbf: short block read"); + *maybe_present = bsbf_block_contains(hash, blk.data()); + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/bsbf.h b/be/src/storage/index/snii/format/bsbf.h new file mode 100644 index 00000000000000..0cc2187efbb42f --- /dev/null +++ b/be/src/storage/index/snii/format/bsbf.h @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/io/file_reader.h" + +// Block-split bloom filter (BSBF) -- Apache Parquet split-block spec, with an +// S3-native on-demand single-block probe that none of the reference implementations +// (Apache Parquet, Doris storage, Doris format/parquet) ship. +// +// BIT FORMAT IS PARQUET-CANONICAL (interoperable with Apache Parquet / Doris +// format/parquet for the bitset bytes): +// - 256-bit (32-byte) blocks, 8 bits set per block. +// - key = XXH64(term, seed=0); high 32 bits select the block via FASTRANGE +// `block = ((hash>>32) * num_blocks) >> 32` (no power-of-2 requirement); low 32 +// bits select 8 in-block positions `1 << ((key * SALT[i]) >> 27)`. +// - num_bytes via Parquet OptimalNumOfBytes: power of 2 in [32, 128 MiB]. +// +// SNII WRAPPER (NOT Parquet's variable thrift header): a FIXED 28-byte header, then +// the contiguous, uncompressed, little-endian bitset. Because the header size is a +// constant, the bitset start is a constant offset (`section_base + 28`) and block i +// is at `section_base + 28 + i*32` -- so a single 32-byte block can be range-read on +// demand WITHOUT parsing a variable-length header and WITHOUT loading the whole blob. +namespace doris::snii::format { + +constexpr uint32_t kBsbfBytesPerBlock = 32; // 256-bit block +constexpr uint32_t kBsbfBitsSetPerBlock = 8; // 8 uint32 words / block +constexpr uint32_t kBsbfMinBytes = 32; +constexpr uint32_t kBsbfMaxBytes = 128u * 1024 * 1024; // Parquet kMaximumBloomFilterBytes +constexpr uint32_t kBsbfHeaderSize = 28; // FIXED (constant bitset offset) +// L0/L1 tiering threshold (the "fast-reject absent terms" design): a bsbf section whose total +// size is <= this is loaded WHOLE into the resident reader at open (L0 -> free +// in-memory probe, no per-lookup round); larger filters stay L1 (header-only, probed +// one 32-byte block on demand). 256 KiB fits in a single cloud FileCache block. +constexpr uint32_t kBsbfResidentMaxBytes = 256u * 1024; + +// Canonical Parquet/Doris split-block SALT (8 odd 32-bit constants). +extern const uint32_t kBsbfSalt[kBsbfBitsSetPerBlock]; + +// XXH64(term, seed=0) -- the Parquet-canonical key (NOT XXH3, NOT Doris murmur). +uint64_t bsbf_hash(std::string_view term); + +// Parquet OptimalNumOfBytes(ndv, fpp): power of 2 in [32, 128 MiB]. +uint32_t bsbf_optimal_num_bytes(uint32_t ndv, double fpp); + +// Fastrange block index from a 64-bit hash and the block count. +inline uint32_t bsbf_block_index(uint64_t hash, uint32_t num_blocks) { + return static_cast(((hash >> 32) * num_blocks) >> 32); +} + +// Pure 32-byte-block kernel: does `block` contain the key's 8 bits? SIMD (AVX2) +// accelerated at runtime when available, scalar otherwise. Returns true => the term +// MAY be present (could be a false positive); false => DEFINITELY ABSENT. +bool bsbf_block_contains(uint64_t hash, const uint8_t block[kBsbfBytesPerBlock]); + +// In-memory builder + serializer. +class BsbfBuilder { +public: + BsbfBuilder() = default; + + // Sizes the filter for `ndv` distinct keys at target `fpp`. fpp in (0,1). + static Status create(uint32_t ndv, double fpp, BsbfBuilder* out); + + // Insert a key / term. SIMD-accelerated. + void insert(uint64_t hash); + void insert_term(std::string_view term) { insert(bsbf_hash(term)); } + + // In-memory probe over the resident bitset (build/warm path). SIMD-accelerated. + bool maybe_contains(uint64_t hash) const; + bool maybe_contains_term(std::string_view term) const { + return maybe_contains(bsbf_hash(term)); + } + + // Serialize [28-byte header][contiguous LE bitset] into `sink`. The header carries + // magic/version/hash+index strategy/num_bytes/num_blocks/ndv + header & bitset + // crc32c. The bitset is Parquet-canonical bytes. + Status serialize(ByteSink* sink) const; + + uint32_t num_bytes() const { return num_bytes_; } + uint32_t num_blocks() const { return num_blocks_; } + +private: + std::vector words_; // num_bytes_/4, blocks of 8 words + uint32_t num_bytes_ = 0; + uint32_t num_blocks_ = 0; + uint32_t ndv_ = 0; +}; + +// Resident header (28 bytes), parsed once at open. Validates magic/version/crc/bounds. +struct BsbfHeader { + uint32_t num_bytes = 0; + uint32_t num_blocks = 0; + uint32_t bitset_crc = 0; // stored crc32c of the bitset body (for L0 verification) + uint64_t bitset_base = 0; // absolute file offset of block 0 = section_base + 28 + + // Parse a 28-byte header located at `section_base` in the file. The bitset_base + // is set to section_base + kBsbfHeaderSize. + static Status parse(Slice header28, uint64_t section_base, BsbfHeader* out); + + // Absolute file offset of the 32-byte block this hash maps to. + uint64_t block_offset(uint64_t hash) const { + return bitset_base + + static_cast(bsbf_block_index(hash, num_blocks)) * kBsbfBytesPerBlock; + } +}; + +// On-demand probe: read EXACTLY ONE 32-byte block via `reader`, then test. No whole +// blob load, no deep copy. *maybe_present=false means DEFINITELY ABSENT. +Status bsbf_probe(io::FileReader* reader, const BsbfHeader& header, uint64_t hash, + bool* maybe_present); + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/dict_block.cpp b/be/src/storage/index/snii/format/dict_block.cpp new file mode 100644 index 00000000000000..c7697b3eeea3c1 --- /dev/null +++ b/be/src/storage/index/snii/format/dict_block.cpp @@ -0,0 +1,492 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/dict_block.h" + +#include +#include + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/encoding/varint.h" +#include "storage/index/snii/format/phrase_bigram.h" // is_phrase_bigram_term (G16 accounting) +#include "storage/index/snii/format/sampled_term_index.h" // std_string_heap_bytes + +namespace doris::snii::format { + +namespace { + +constexpr size_t kFooterBytes = sizeof(uint32_t); // trailing crc32c +constexpr size_t kNAnchorsBytes = sizeof(uint32_t); // n_anchors u32 +constexpr size_t kAnchorOffBytes = sizeof(uint32_t); // per-anchor offset u32 + +// Estimate the encoded upper-bound byte size of one entry (no actual encoding; used by estimated_bytes). +// Take the maximum varint width of each variable-length field plus payload bytes to guarantee an upper bound. +size_t estimate_entry_bytes(const DictEntry& e) { + size_t body = 0; + body += varint_len(static_cast(e.term.size())); // prefix_len upper bound + body += varint_len(static_cast(e.term.size())); // suffix_len upper bound + body += e.term.size(); // suffix bytes upper bound + body += 1; // flags + body += 10; // df + ttf + max_freq upper bound + body += 10; // ttf_delta + body += 10; // max_freq + if (e.kind == DictEntryKind::kInline) { + body += 10 + e.frq_bytes.size(); + body += 10 + e.prx_bytes.size(); + } else { + body += 10 * 5; // frq_off/frq_len/prelude/prx_off/prx_len upper bound + } + return varint_len(static_cast(body)) + body; // entry_len + body +} + +} // namespace + +// ---- DictBlockBuilder ---- + +DictBlockBuilder::DictBlockBuilder(IndexTier tier, bool has_positions, uint64_t frq_base, + uint64_t prx_base, uint32_t anchor_interval, bool term_stats) + : tier_(tier), + has_positions_(has_positions), + term_stats_(term_stats), + frq_base_(frq_base), + prx_base_(prx_base), + anchor_interval_(anchor_interval == 0 ? 1 : anchor_interval) {} + +void DictBlockBuilder::add_entry(const DictEntry& entry) { + if (is_anchor(n_entries_)) { + ++n_anchors_; + } + entries_est_ += estimate_entry_bytes(entry); + entries_.push_back(entry); + ++n_entries_; +} + +void DictBlockBuilder::add_entry(DictEntry&& entry) { + if (is_anchor(n_entries_)) { + ++n_anchors_; + } + // estimate_entry_bytes reads `entry`, so it MUST run before the move below: + // sizing a moved-from (empty) entry would undercount entries_est_ and split + // blocks incorrectly. finish() output is unaffected either way -- it depends + // only on the entries actually queued, not on how they were appended. + entries_est_ += estimate_entry_bytes(entry); + entries_.push_back(std::move(entry)); + ++n_entries_; +} + +size_t DictBlockBuilder::estimated_bytes() const { + size_t header = varint_len(static_cast(n_entries_)) + 2; // +ver +flags + header += varint_len(frq_base_); + if (has_positions_) { + header += varint_len(prx_base_); + } + const size_t anchors = n_anchors_ * kAnchorOffBytes + kNAnchorsBytes; + return header + entries_est_ + anchors + kFooterBytes; +} + +void DictBlockBuilder::finish(ByteSink* sink) const { + ByteSink body; // header + entries + anchor_offsets + n_anchors (crc covered region) + + // header. + body.put_varint64(static_cast(n_entries_)); + body.put_u8(kDictBlockFormatVer); + body.put_u8(static_cast((has_positions_ ? dict_block_flags::kHasPositions : 0U) | + (term_stats_ ? 0U : dict_block_flags::kNoTermStats))); + body.put_varint64(frq_base_); + if (has_positions_) { + body.put_varint64(prx_base_); + } + + // entries: anchor entries use prev_term="" and record their byte offset within the block. + std::vector anchor_offsets; + anchor_offsets.reserve(n_anchors_); + std::string prev; + entry_bytes_total_ = 0; + entry_bytes_bigram_ = 0; + for (uint32_t i = 0; i < n_entries_; ++i) { + const bool anchor = is_anchor(i); + if (anchor) { + anchor_offsets.push_back(static_cast(body.size())); + } + const std::string_view prev_term = anchor ? std::string_view {} : std::string_view(prev); + // finish() is void and entry encoding into an in-memory ByteSink cannot fail; + // explicitly discard the (now [[nodiscard]] Status) return. + const uint64_t before = body.size(); + static_cast(encode_dict_entry(entries_[i], prev_term, tier_, &body, term_stats_)); + const uint64_t encoded = body.size() - before; + entry_bytes_total_ += encoded; + if (is_phrase_bigram_term(entries_[i].term)) { + entry_bytes_bigram_ += encoded; + } + prev = entries_[i].term; + } + + // anchor_offsets[] + n_anchors. + for (uint32_t off : anchor_offsets) { + body.put_fixed32(off); + } + body.put_fixed32(static_cast(anchor_offsets.size())); + + // Write the entire block (including crc footer) to sink. + sink->put_bytes(body.view()); + sink->put_fixed32(crc32c(body.view())); +} + +// ---- DictBlockReader ---- + +namespace { + +// Verify the block length is sufficient and validate the trailing crc; return a Slice of the covered region (excluding crc footer). +Status verify_crc(Slice block, Slice* covered) { + if (block.size() < kFooterBytes + kNAnchorsBytes) { + return Status::Error( + "dict_block: block too short to contain footer"); + } + const size_t covered_len = block.size() - kFooterBytes; + *covered = block.subslice(0, covered_len); + + ByteSource crc_src(block.subslice(covered_len, kFooterBytes)); + uint32_t stored = 0; + RETURN_IF_ERROR(crc_src.get_fixed32(&stored)); + if (crc32c(*covered) != stored) { + return Status::Error( + "dict_block: crc32c checksum mismatch"); + } + return Status::OK(); +} + +// Read and verify that block_flags is consistent with has_positions. +Status check_flags(uint8_t flags, bool has_positions) { + const bool flag_pos = (flags & dict_block_flags::kHasPositions) != 0; + if (flag_pos != has_positions) { + return Status::Error( + "dict_block: has_positions inconsistent with block_flags"); + } + return Status::OK(); +} + +} // namespace + +Status DictBlockReader::open(Slice block, IndexTier tier, bool has_positions, + DictBlockReader* out) { + if (out == nullptr) { + return Status::Error("dict_block: out is null"); + } + *out = DictBlockReader {}; + + // Decode instrumentation seam: one increment per block materialization (the + // CRC verify + anchor parse below, preceded by a zstd decompress for a + // compressed block). A dict-block cache eliminates repeats of exactly this. + testing::note_dict_block_decode(); + + Slice covered; + RETURN_IF_ERROR(verify_crc(block, &covered)); + out->block_ = covered; + out->tier_ = tier; + out->has_positions_ = has_positions; + + // header. + ByteSource src(covered); + uint64_t n_entries = 0; + RETURN_IF_ERROR(src.get_varint64(&n_entries)); + uint8_t ver = 0; + uint8_t flags = 0; + RETURN_IF_ERROR(src.get_u8(&ver)); + RETURN_IF_ERROR(src.get_u8(&flags)); + if (ver != kDictBlockFormatVer) { + return Status::Error( + "dict_block: unsupported entry_format_ver"); + } + RETURN_IF_ERROR(check_flags(flags, has_positions)); + out->term_stats_ = (flags & dict_block_flags::kNoTermStats) == 0; + RETURN_IF_ERROR(src.get_varint64(&out->frq_base_)); + if (has_positions) { + RETURN_IF_ERROR(src.get_varint64(&out->prx_base_)); + } + + out->n_entries_ = static_cast(n_entries); + out->entries_begin_ = src.position(); + + // The anchor table is at the tail of covered: [... anchor_offsets[n] n_anchors(u32)]. + if (covered.size() < kNAnchorsBytes) { + return Status::Error( + "dict_block: missing n_anchors"); + } + ByteSource na_src(covered.subslice(covered.size() - kNAnchorsBytes, kNAnchorsBytes)); + uint32_t n_anchors = 0; + RETURN_IF_ERROR(na_src.get_fixed32(&n_anchors)); + + const size_t anchor_table_bytes = static_cast(n_anchors) * kAnchorOffBytes; + if (covered.size() < kNAnchorsBytes + anchor_table_bytes || + out->entries_begin_ + anchor_table_bytes + kNAnchorsBytes > covered.size()) { + return Status::Error( + "dict_block: anchor table out of range"); + } + const size_t anchor_table_begin = covered.size() - kNAnchorsBytes - anchor_table_bytes; + + ByteSource at_src(covered.subslice(anchor_table_begin, anchor_table_bytes)); + out->anchor_offsets_.resize(n_anchors); + out->anchor_terms_.resize(n_anchors); + for (uint32_t i = 0; i < n_anchors; ++i) { + uint32_t off = 0; + RETURN_IF_ERROR(at_src.get_fixed32(&off)); + if (off >= anchor_table_begin) { + return Status::Error( + "dict_block: anchor offset out of range"); + } + // Anchor offsets must be strictly monotonically increasing, and the first anchor must be exactly the start of the entries region (entry 0 is always an anchor). + // Otherwise scan_from_anchor's segment-length computation seg_end-seg_begin would underflow as size_t and cause an out-of-range read, + // guarding against non-monotonic offset tables with a re-stamped crc (remote on-demand read / cache misalignment scenarios). + if (i == 0) { + if (off != out->entries_begin_) { + return Status::Error( + "dict_block: first anchor offset is not the start of entries"); + } + } else if (off <= out->anchor_offsets_[i - 1]) { + return Status::Error( + "dict_block: anchor offsets are not strictly increasing"); + } + out->anchor_offsets_[i] = off; + // Anchor entries are encoded with prev_term="" and can be decoded independently to retrieve their term. + ByteSource e_src(covered.subslice(off, anchor_table_begin - off)); + DictEntry probe; + RETURN_IF_ERROR(decode_dict_entry(&e_src, std::string_view {}, tier, &probe, + (flags & dict_block_flags::kNoTermStats) == 0)); + out->anchor_terms_[i] = std::move(probe.term); + } + return Status::OK(); +} + +size_t DictBlockReader::heap_bytes() const { + size_t bytes = anchor_offsets_.capacity() * sizeof(uint32_t) + + anchor_terms_.capacity() * sizeof(std::string); + for (const auto& term : anchor_terms_) { + bytes += std_string_heap_bytes(term); + } + return bytes; +} + +bool DictBlockReader::locate_anchor(std::string_view target, size_t* anchor_idx) const { + if (anchor_terms_.empty()) { + return false; + } + if (target < std::string_view(anchor_terms_.front())) { + return false; + } + // The last anchor_term <= target. + size_t lo = 0; + size_t hi = anchor_terms_.size(); // open interval + while (lo + 1 < hi) { + const size_t mid = lo + (hi - lo) / 2; + if (std::string_view(anchor_terms_[mid]) <= target) { + lo = mid; + } else { + hi = mid; + } + } + *anchor_idx = lo; + return true; +} + +Status DictBlockReader::decode_all(std::vector* out) const { + if (out == nullptr) { + return Status::Error("dict_block: out is null"); + } + out->clear(); + out->reserve(n_entries_); + for (size_t a = 0; a < anchor_offsets_.size(); ++a) { + const size_t seg_begin = anchor_offsets_[a]; + const bool is_last = a + 1 == anchor_offsets_.size(); + const size_t seg_end = is_last ? (block_.size() - kNAnchorsBytes - + anchor_offsets_.size() * kAnchorOffBytes) + : anchor_offsets_[a + 1]; + if (seg_end < seg_begin || seg_end > block_.size()) { + return Status::Error( + "dict_block: anchor segment range invalid"); + } + ByteSource src(block_.subslice(seg_begin, seg_end - seg_begin)); + std::string prev; // first entry of a segment is an anchor (prev_term="") + while (!src.eof()) { + DictEntry e; + RETURN_IF_ERROR( + decode_dict_entry(&src, std::string_view(prev), tier_, &e, term_stats_)); + prev = e.term; + out->push_back(std::move(e)); + } + } + if (out->size() != n_entries_) { + return Status::Error( + "dict_block: decoded entry count mismatch"); + } + return Status::OK(); +} + +Status DictBlockReader::scan_from_anchor(size_t anchor_idx, std::string_view target, bool* found, + DictEntry* out) const { + // Byte range of this anchor segment: [anchor_offset, next anchor offset or anchor table start). + const size_t seg_begin = anchor_offsets_[anchor_idx]; + const bool is_last = anchor_idx + 1 == anchor_offsets_.size(); + const size_t seg_end = + is_last ? (block_.size() - kNAnchorsBytes - anchor_offsets_.size() * kAnchorOffBytes) + : anchor_offsets_[anchor_idx + 1]; + + // Fallback: open() has already verified anchor monotonicity; this additionally guards against seg_end block_.size()) { + return Status::Error( + "dict_block: anchor segment range invalid"); + } + ByteSource src(block_.subslice(seg_begin, seg_end - seg_begin)); + std::string prev; // the first entry in the segment is an anchor, prev_term="" + while (!src.eof()) { + // Key-first: decode only the (front-coded) term key, then decide whether + // the body is worth materializing. Non-matching entries skip their body + // entirely -- with anchor_interval=16 that turns ~16 body decodes per + // lookup into 1 (the matched entry) or 0 (a miss). + DictEntry e; + size_t body_start = 0; + uint64_t entry_total = 0; + RETURN_IF_ERROR( + decode_dict_entry_key(&src, std::string_view(prev), &e, &body_start, &entry_total)); + if (e.term == target) { + RETURN_IF_ERROR( + decode_dict_entry_rest(&src, tier_, body_start, entry_total, &e, term_stats_)); + *found = true; + *out = std::move(e); + return Status::OK(); + } + if (std::string_view(e.term) > target) { + *found = false; // already past target; entries are sorted so it does not exist + return Status::OK(); + } + // Before target: skip the body but keep the key as the front-coding base. + RETURN_IF_ERROR(skip_dict_entry_body(&src, body_start, entry_total)); + prev = std::move(e.term); + } + *found = false; + return Status::OK(); +} + +Status DictBlockReader::find_term(std::string_view target, bool* found, DictEntry* out) const { + if (found == nullptr || out == nullptr) { + return Status::Error("dict_block: found / out is null"); + } + *found = false; + size_t anchor_idx = 0; + if (!locate_anchor(target, &anchor_idx)) { + return Status::OK(); + } + return scan_from_anchor(anchor_idx, target, found, out); +} + +Status DictBlockReader::visit_prefix_range(std::string_view prefix, + const std::function& accept_key, + const std::function& on_hit, + bool* prefix_exhausted) const { + if (!on_hit || prefix_exhausted == nullptr) { + return Status::Error( + "dict_block: null visit_prefix_range args"); + } + *prefix_exhausted = false; + if (anchor_offsets_.empty()) { + return Status::OK(); // empty block: nothing to enumerate + } + + // Anchor-jump: start at the anchor segment that may contain prefix. Earlier + // segments hold only terms < prefix (every term is < the next anchor term + // <= prefix), so they are skipped without any decode. An empty prefix or one + // sorting before the first anchor starts at anchor 0. + size_t anchor_idx = 0; + if (!prefix.empty()) { + locate_anchor(prefix, &anchor_idx); // false leaves anchor_idx at 0 + } + + // Scan from the chosen anchor to the end of the entries region: the prefix + // range may span several anchor segments. Anchor entries are encoded with + // prefix_len=0, so a single running `prev` reconstructs every term correctly + // even as the scan crosses segment boundaries. + const size_t seg_begin = anchor_offsets_[anchor_idx]; + const size_t entries_end = + block_.size() - kNAnchorsBytes - anchor_offsets_.size() * kAnchorOffBytes; + if (entries_end < seg_begin || entries_end > block_.size()) { + return Status::Error( + "dict_block: entries region range invalid"); + } + + ByteSource src(block_.subslice(seg_begin, entries_end - seg_begin)); + std::string prev; // the chosen anchor segment starts at an anchor (prev="") + while (!src.eof()) { + DictEntry e; + size_t body_start = 0; + uint64_t entry_total = 0; + RETURN_IF_ERROR( + decode_dict_entry_key(&src, std::string_view(prev), &e, &body_start, &entry_total)); + const std::string_view t(e.term); + if (t < prefix) { + // Still before the range (only reachable inside the anchor segment + // that straddles prefix): skip the body, keep the front-coding base. + RETURN_IF_ERROR(skip_dict_entry_body(&src, body_start, entry_total)); + prev = std::move(e.term); + continue; + } + const bool has_prefix = t.size() >= prefix.size() && t.starts_with(prefix); + if (!has_prefix) { + *prefix_exhausted = true; // sorted: no further matches here or later + return Status::OK(); + } + if (accept_key && !accept_key(t)) { + // Key-only rejection: never pay for this entry's body. + RETURN_IF_ERROR(skip_dict_entry_body(&src, body_start, entry_total)); + prev = std::move(e.term); + continue; + } + // Accepted: materialize the body and hand the entry to the visitor. + RETURN_IF_ERROR( + decode_dict_entry_rest(&src, tier_, body_start, entry_total, &e, term_stats_)); + prev = e.term; // copy the key before the entry is moved into on_hit + bool stop = false; + RETURN_IF_ERROR(on_hit(std::move(e), &stop)); + if (stop) { + return Status::OK(); + } + } + return Status::OK(); +} + +} // namespace doris::snii::format + +namespace doris::snii::testing { +namespace { +std::atomic& dict_decode_atomic() { + static std::atomic counter {0}; + return counter; +} +} // namespace + +uint64_t dict_decode_counter() { + return dict_decode_atomic().load(std::memory_order_relaxed); +} + +void reset_dict_decode_counter() { + dict_decode_atomic().store(0, std::memory_order_relaxed); +} + +void note_dict_block_decode() { + dict_decode_atomic().fetch_add(1, std::memory_order_relaxed); +} + +} // namespace doris::snii::testing diff --git a/be/src/storage/index/snii/format/dict_block.h b/be/src/storage/index/snii/format/dict_block.h new file mode 100644 index 00000000000000..fcf92914aa4e34 --- /dev/null +++ b/be/src/storage/index/snii/format/dict_block.h @@ -0,0 +1,228 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" + +// DICT block —— a positioning unit mapping term → postings read plan, and also +// the unit for remote on-demand fetching, caching, and CRC checksum +// verification (see docs/design/SNII-design-spec.source.md "DICT block" and +// "dict lookup flow summary" sections). +// +// Byte layout (strictly implemented; multi-byte fixed-width fields are +// little-endian, variable-length integers use LEB128): +// header: +// n_entries varint +// entry_format_ver u8 # = kDictBlockFormatVer +// block_flags u8 # bit0 = has_positions (consistency check +// against the value passed to reader) frq_base varint64 prx_base +// varint64 # present only when has_positions is set +// entries[n_entries] # variable-length DictEntry, front-coded in +// lexicographic order anchor_offsets[n_anchors] # u32 * n_anchors, byte +// offset of each anchor entry within the block n_anchors u32 crc32c +// u32 # covers [header .. n_anchors], detects corruption (sole CRC +// layer) +// +// Anchor rule: every anchor_interval entries, one "term anchor" is forced — +// that entry is encoded with prev_term="" (prefix_len=0, storing the full +// term), and its byte offset is recorded in anchor_offsets; non-anchor entries +// use the preceding entry's term as prev_term for front coding. The reader can +// start from any anchor and scan independently without needing earlier terms, +// enabling anchor binary search + local scan for exact term lookup. +namespace doris::snii::format { + +// DICT block entry_format_ver: self-describing version of the DictEntry +// encoding. Reader rejects a mismatch so a query-only run cannot silently read +// an older dict-entry layout as the current one. +inline constexpr uint8_t kDictBlockFormatVer = 2; + +// block_flags bit definitions. +namespace dict_block_flags { +inline constexpr uint8_t kHasPositions = 1U << 0; // whether to write prx_base / .prx fields +// G16-f: entries omit the ttf_delta/max_freq varints (freq-dropped index -- +// the stats serve only BM25 scoring). Self-describing per block; absent on +// pre-G16-f blocks, whose entries always carry the stats on tier>=T2. +inline constexpr uint8_t kNoTermStats = 1U << 1; +// bit1-7 reserved +} // namespace dict_block_flags + +// DICT block writer: entries are added in lexicographic order via add_entry; +// internally determines anchors and accumulates size estimates, and on finish +// serializes header + entries + anchor table + CRC in one pass. The front-coding +// base is rebuilt from a local prev inside finish(), so no prev_term is retained +// as builder state. +class DictBlockBuilder { +public: + DictBlockBuilder(IndexTier tier, bool has_positions, uint64_t frq_base, uint64_t prx_base, + uint32_t anchor_interval = 16, bool term_stats = true); + + // Append one entry (caller must guarantee lexicographic term order). + // Internally decides whether it becomes an anchor. The copy overload is kept + // for callers that must retain their entry afterwards (materialized fallback, + // tests); the move overload avoids the per-term DictEntry copy -- which for an + // inline entry is two std::vector heap allocations plus the term + // copy -- on the SPIMI build path. + void add_entry(const DictEntry& entry); + void add_entry(DictEntry&& entry); + + // Upper-bound estimate of the serialized size of the current block (including + // header + entries + anchor table + CRC footer), used by the upper layer to + // decide when to cut a new block based on target_dict_block_bytes. + size_t estimated_bytes() const; + + // Number of entries. + uint32_t n_entries() const { return n_entries_; } + + // Serialize the entire block and append it to sink. + void finish(ByteSink* sink) const; + + // G16 byte accounting, valid AFTER finish(): serialized entry-body bytes + // (uncompressed, prefix-coded, anchors included) split by term class. + // Measurement-only -- the writer aggregates these into the per-index + // section-stats log line; nothing on disk depends on them. + uint64_t entry_bytes_total() const { return entry_bytes_total_; } + uint64_t entry_bytes_bigram() const { return entry_bytes_bigram_; } + +private: + bool is_anchor(uint32_t index) const { return index % anchor_interval_ == 0; } + + IndexTier tier_; + bool has_positions_; + bool term_stats_ = true; // false: entries omit ttf/max_freq (kNoTermStats) + uint64_t frq_base_; + uint64_t prx_base_; + uint32_t anchor_interval_; + + uint32_t n_entries_ = 0; + std::vector entries_; + size_t entries_est_ = 0; // accumulated byte estimate for the entries section + size_t n_anchors_ = 0; // number of anchors + // G16 accounting, filled by finish() (which stays logically const: these + // never affect serialization). + mutable uint64_t entry_bytes_total_ = 0; + mutable uint64_t entry_bytes_bigram_ = 0; +}; + +// DICT block reader: on open, verifies the CRC and parses the header / anchor +// table; find_term uses anchor binary search + local scan to locate a +// DictEntry. Holds a byte view of the block (non-owning); lifetime is managed +// by the caller. +class DictBlockReader { +public: + DictBlockReader() = default; + + // Parse and verify the entire block. CRC mismatch / truncation / invalid + // structure → Corruption; has_positions in the header inconsistent with the + // supplied argument → InvalidArgument. + static Status open(Slice block, IndexTier tier, bool has_positions, DictBlockReader* out); + + // Anchor binary search + local scan to locate target. Hit → *found=true and + // *out is filled; miss (including out-of-range, gap) → *found=false. + // Structural error → non-OK Status. + Status find_term(std::string_view target, bool* found, DictEntry* out) const; + + // Decodes EVERY entry in the block in lexicographic order into *out (each a + // self-contained DictEntry, owning its term). Used for ordered term + // enumeration (prefix / range scans). Resets the front-coding base at each + // anchor segment. Retained as the golden reference; the prefix path now + // streams via visit_prefix_range. + Status decode_all(std::vector* out) const; + + // Streams the entries of this block whose term lies in [prefix, prefix+) in + // lexicographic order, materializing only the bodies a caller keeps (T07): + // 1) anchor-jump to the anchor segment containing prefix (segments whose + // terms are all < prefix are skipped without any decode); an empty + // prefix or one before the first anchor starts at anchor 0; + // 2) within range, decode each entry's term key only; term < prefix skips + // the body and continues; a term that leaves the prefix range sets + // *prefix_exhausted=true and ends the scan (sorted order guarantees no + // further matches here or in later blocks); + // 3) accept_key(term)==false skips the body (lets callers push a key-only + // predicate down so a non-match never pays for its body); + // 4) only accepted entries have their body decoded and are handed to + // on_hit, which may request an early stop via *stop. + // accept_key may be empty (treated as accept-all). prefix_exhausted is set + // false on entry and true only when a term past the range is seen. + Status visit_prefix_range(std::string_view prefix, + const std::function& accept_key, + const std::function& on_hit, + bool* prefix_exhausted) const; + + uint64_t frq_base() const { return frq_base_; } + uint64_t prx_base() const { return prx_base_; } + uint32_t n_entries() const { return n_entries_; } + + // Resident heap held beyond sizeof(*this): the anchor_offsets_ / anchor_terms_ + // vector buffers plus each non-SSO anchor term's heap allocation. block_ is a + // NON-owning view and is deliberately NOT counted here -- its owning buffer is + // charged by the caller (the resident block's `bytes` vector, or a + // request-scoped decoded block). Summed into + // LogicalIndexReader::memory_usage() per resident block. + size_t heap_bytes() const; + +private: + // Sequentially scan from anchor anchor_idx to the end of that anchor segment, + // searching for target. + Status scan_from_anchor(size_t anchor_idx, std::string_view target, bool* found, + DictEntry* out) const; + + // Find the last anchor index where first_term(anchor) <= target; return false + // if none exists. + bool locate_anchor(std::string_view target, size_t* anchor_idx) const; + + Slice block_; // [header .. crc) full block view + IndexTier tier_ = IndexTier::kT1; + bool has_positions_ = false; + bool term_stats_ = true; // from block flags (kNoTermStats absent => true) + uint64_t frq_base_ = 0; + uint64_t prx_base_ = 0; + uint32_t n_entries_ = 0; + + size_t entries_begin_ = 0; // absolute offset of the start of the entries section + std::vector anchor_offsets_; // byte offset within the block for each anchor entry + std::vector + anchor_terms_; // full term of each anchor entry (used for binary search) +}; + +} // namespace doris::snii::format + +// Test-only instrumentation seam. dict_decode_counter() returns a process-global +// count of DICT block decodes performed by DictBlockReader::open -- i.e. the +// optional zstd decompress + CRC verify + anchor parse that turns on-disk block +// bytes into a usable reader. This is precisely the unit a dict-block cache +// eliminates on repeat, so tests assert dict_decode_counter() == unique_blocks. +// In production DICT blocks are zstd-compressed, so this equals the zstd +// decompress count. Counters use relaxed atomics; reset between tests. +namespace doris::snii::testing { + +uint64_t dict_decode_counter(); +void reset_dict_decode_counter(); +void note_dict_block_decode(); + +} // namespace doris::snii::testing diff --git a/be/src/storage/index/snii/format/dict_block_directory.cpp b/be/src/storage/index/snii/format/dict_block_directory.cpp new file mode 100644 index 00000000000000..67d4d672d04305 --- /dev/null +++ b/be/src/storage/index/snii/format/dict_block_directory.cpp @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/dict_block_directory.h" + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/section_framer.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::format { + +namespace { + +// Each block_ref has a fixed field order; reuse ByteSink varint/fixed primitives — do not hand-craft bytes manually. +// uncomp_len trails only when the kZstd flag is set, so uncompressed-block +// directories keep their compact (v1-identical) per-ref byte layout. +void encode_ref(const BlockRef& ref, ByteSink* payload) { + payload->put_varint64(ref.offset); + payload->put_varint64(ref.length); + payload->put_varint32(ref.n_entries); + payload->put_u8(ref.flags); + payload->put_fixed32(ref.checksum); + if (ref.flags & block_ref_flags::kZstd) payload->put_varint64(ref.uncomp_len); +} + +Status decode_ref(ByteSource* ps, BlockRef* ref) { + RETURN_IF_ERROR(ps->get_varint64(&ref->offset)); + RETURN_IF_ERROR(ps->get_varint64(&ref->length)); + RETURN_IF_ERROR(ps->get_varint32(&ref->n_entries)); + RETURN_IF_ERROR(ps->get_u8(&ref->flags)); + RETURN_IF_ERROR(ps->get_fixed32(&ref->checksum)); + if (ref->flags & block_ref_flags::kZstd) { + RETURN_IF_ERROR(ps->get_varint64(&ref->uncomp_len)); + } + return Status::OK(); +} + +Status decode_payload(Slice payload, std::vector* refs) { + ByteSource ps(payload); + uint32_t n_blocks = 0; + RETURN_IF_ERROR(ps.get_varint32(&n_blocks)); + // Guard against a corrupted, inflated count from untrusted bytes: each BlockRef + // needs >= 8 bytes (flags u8 + checksum u32 + >= 1 byte for each of 3 varints), + // so cap before reserve to avoid a huge allocation. + constexpr size_t kMinRefBytes = 8; + if (n_blocks > ps.remaining() / kMinRefBytes) { + return Status::Error( + "dict_block_directory: n_blocks exceeds payload capacity"); + } + refs->clear(); + refs->reserve(n_blocks); + for (uint32_t i = 0; i < n_blocks; ++i) { + BlockRef ref {}; + RETURN_IF_ERROR(decode_ref(&ps, &ref)); + refs->push_back(ref); + } + if (!ps.eof()) { + return Status::Error( + "dict_block_directory: trailing bytes in payload"); + } + return Status::OK(); +} + +} // namespace + +void DictBlockDirectoryBuilder::finish(ByteSink* sink) const { + ByteSink payload; + payload.put_varint32(static_cast(refs_.size())); + for (const auto& ref : refs_) { + encode_ref(ref, &payload); + } + SectionFramer::write(*sink, static_cast(SectionType::kDictBlockDirectory), + payload.view()); +} + +Status DictBlockDirectoryReader::open(Slice section, DictBlockDirectoryReader* out) { + ByteSource src(section); + FramedSection sec; + RETURN_IF_ERROR(SectionFramer::read(src, &sec)); + if (sec.type != static_cast(SectionType::kDictBlockDirectory)) { + return Status::Error( + "dict_block_directory: unexpected section type"); + } + return decode_payload(sec.payload, &out->refs_); +} + +Status DictBlockDirectoryReader::get(uint32_t ordinal, BlockRef* out) const { + if (ordinal >= refs_.size()) { + return Status::Error( + "dict_block_directory: ordinal out of range"); + } + *out = refs_[ordinal]; + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/dict_block_directory.h b/be/src/storage/index/snii/format/dict_block_directory.h new file mode 100644 index 00000000000000..909b32f29d1374 --- /dev/null +++ b/be/src/storage/index/snii/format/dict_block_directory.h @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +namespace doris::snii::format { + +// BlockRef.flags bit definitions. +namespace block_ref_flags { +// bit0: the on-disk block bytes are zstd(uncompressed_block). When set, the +// directory also stores uncomp_len, and the reader zstd-decompresses the fetched +// [offset, offset+length) range to uncomp_len before parsing the dict block. The +// block-level crc32c (and BlockRef.checksum) cover the UNCOMPRESSED bytes, so a +// zstd block shrinks the bytes fetched from S3 while keeping the same integrity +// guarantees after decompression in RAM. +inline constexpr uint8_t kZstd = 1u << 0; +} // namespace block_ref_flags + +// Physical location and checksum info for a single DICT block. Aligned with SampledTermIndex by ordinal: +// SampledTermIndex[i]'s first_term corresponds to DictBlockDirectory[i] (see design spec +// "sampled dict index"). The read path issues a single range read over [offset, offset+length). +struct BlockRef { + uint64_t offset = 0; // absolute byte offset of the block within the container + uint64_t length = 0; // ON-DISK byte length of the block (compressed when kZstd) + uint32_t n_entries = 0; // number of DictEntry records within this block + uint8_t flags = 0; // block-level flags (block_ref_flags::*) + uint32_t checksum = 0; // crc32c of the block's UNCOMPRESSED content (verified after read) + uint64_t uncomp_len = 0; // uncompressed block byte length (stored only when kZstd set) +}; + +// DICT block directory: block ordinal → physical location mapping. +// +// on-disk layout (framed by SectionFramer with a unified type+len+crc32c wrapper): +// [u8 type=kDictBlockDirectory][varint64 payload_len][payload][fixed32 crc32c] +// payload = varint32 n_blocks +// then n_blocks × block_ref{ +// varint64 offset, varint64 length, varint32 n_entries, +// u8 flags, fixed32 checksum } +// Section-level crc detects truncation/corruption; block_ref.checksum is the per-block crc. +class DictBlockDirectoryBuilder { +public: + void add(const BlockRef& ref) { refs_.push_back(ref); } + + // Encodes as a kDictBlockDirectory framed section (with embedded crc32c) and appends to sink. + void finish(ByteSink* sink) const; + +private: + std::vector refs_; +}; + +// Reads and verifies a kDictBlockDirectory framed section; provides ordinal → BlockRef lookup. +// After parsing, all block_refs reside in the reader (entering the searcher cache along with meta). +class DictBlockDirectoryReader { +public: + // Verifies the section crc and deserializes all block_refs. + // crc mismatch / truncation / trailing bytes → kCorruption; wrong section type → kInvalidArgument. + static Status open(Slice section, DictBlockDirectoryReader* out); + + uint32_t n_blocks() const { return static_cast(refs_.size()); } + + // Resident heap held beyond sizeof(*this): the refs_ vector buffer. BlockRef + // is trivially copyable (no per-element heap), so the vector buffer is the + // whole charge. Summed into LogicalIndexReader::memory_usage(). + size_t heap_bytes() const { return refs_.capacity() * sizeof(BlockRef); } + + // Returns the ordinal-th block_ref; ordinal >= n_blocks → kNotFound. + Status get(uint32_t ordinal, BlockRef* out) const; + +private: + std::vector refs_; +}; + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/dict_entry.cpp b/be/src/storage/index/snii/format/dict_entry.cpp new file mode 100644 index 00000000000000..aced84e939e705 --- /dev/null +++ b/be/src/storage/index/snii/format/dict_entry.cpp @@ -0,0 +1,374 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/dict_entry.h" + +#include +#include + +#include "storage/index/snii/common/slice.h" + +namespace doris::snii::format { + +namespace { + +// Body-decode counter seam (T07). Incremented once per decode_dict_entry_rest +// call so tests can assert a key-first scan only materializes the bodies it +// actually produces. Relaxed atomic: read by tests only, never a sync point. +std::atomic& body_decode_atomic() { + static std::atomic counter {0}; + return counter; +} + +// Pure-function assembly / parsing of flags bits; avoids a long inline if-else +// chain. +uint8_t pack_flags(const DictEntry& e) { + uint8_t f = 0; + if (e.kind == DictEntryKind::kInline) f |= dict_flags::kKind; + if (e.enc == DictEntryEnc::kWindowed) f |= dict_flags::kEnc; + if (e.has_sb) f |= dict_flags::kHasSb; + // bit3 has_champion / bit4 offsets_ref are always 0 in v1. + return f; +} + +void apply_flags(uint8_t f, DictEntry* e) { + e->kind = (f & dict_flags::kKind) ? DictEntryKind::kInline : DictEntryKind::kPodRef; + e->enc = (f & dict_flags::kEnc) ? DictEntryEnc::kWindowed : DictEntryEnc::kSlim; + e->has_sb = (f & dict_flags::kHasSb) != 0; +} + +// Length of the longest common prefix between term and prev_term. +uint32_t common_prefix_len(std::string_view term, std::string_view prev) { + uint32_t n = 0; + const uint32_t lim = static_cast(std::min(term.size(), prev.size())); + while (n < lim && term[n] == prev[n]) ++n; + return n; +} + +bool tier_has_stats(IndexTier tier) { + return tier >= IndexTier::kT2; +} + +// ---- Encode entry body (excluding entry_len and trailing crc) ---- + +void write_term_key(const DictEntry& e, std::string_view prev, ByteSink* sink) { + const uint32_t prefix = common_prefix_len(e.term, prev); + const std::string_view suffix = std::string_view(e.term).substr(prefix); + sink->put_varint32(prefix); + sink->put_varint32(static_cast(suffix.size())); + sink->put_bytes(Slice(suffix)); +} + +void write_stats(const DictEntry& e, IndexTier tier, bool term_stats, ByteSink* sink) { + sink->put_varint32(e.df); + // G16-f: term_stats == false (freq-dropped index, block header flag) omits + // the ttf_delta/max_freq varints -- BM25-only fields, dead without freq. + if (!tier_has_stats(tier) || !term_stats) return; + sink->put_varint64(e.ttf_delta); + sink->put_varint64(e.max_freq); +} + +// Per-window codec mode byte shared by slim/inline single-window regions. +uint8_t pack_win_mode(const DictEntry& e) { + uint8_t mode = 0; + if (e.dd_meta.zstd) mode |= 1u << 0; // dd_zstd + if (e.freq_meta.zstd) mode |= 1u << 1; // freq_zstd + return mode; +} + +// Writes the slim/inline region codec metadata (dd always; freq when tier>=T2). +// store_crc=false (INLINE entries, format v2) omits the redundant per-region +// crc32c: the inline bytes already sit inside the dict block, whose own +// block-level crc32c covers them. POD-ref entries pass store_crc=true (their +// regions live in the separately-fetched .frq POD, uncovered by the block crc). +void write_region_meta(const DictEntry& e, IndexTier tier, bool store_crc, ByteSink* sink) { + sink->put_u8(pack_win_mode(e)); + sink->put_varint64(e.dd_meta.uncomp_len); + if (store_crc) sink->put_fixed32(e.dd_meta.crc); + if (!tier_has_stats(tier)) return; + sink->put_varint64(e.freq_meta.uncomp_len); + if (store_crc) sink->put_fixed32(e.freq_meta.crc); +} + +void write_pod_ref(const DictEntry& e, IndexTier tier, ByteSink* sink) { + sink->put_varint64(e.frq_off_delta); + sink->put_varint64(e.frq_len); + if (e.enc == DictEntryEnc::kWindowed) { + sink->put_varint64(e.prelude_len); + sink->put_varint64(e.frq_docs_len); + } else { + sink->put_varint64(e.frq_docs_len); // slim pod_ref: dd region on-disk length + // POD-ref regions live in the .frq POD (not covered by the block crc): keep + // crc. + write_region_meta(e, tier, /*store_crc=*/true, sink); + } + if (!tier_has_stats(tier)) return; + sink->put_varint64(e.prx_off_delta); + sink->put_varint64(e.prx_len); +} + +void write_inline(const DictEntry& e, IndexTier tier, ByteSink* sink) { + sink->put_varint64(static_cast(e.frq_bytes.size())); + sink->put_bytes(Slice(e.frq_bytes)); + sink->put_varint64(e.inline_dd_disk_len); + // INLINE bytes are covered by the dict block crc32c: omit the redundant + // per-region crc. + write_region_meta(e, tier, /*store_crc=*/false, sink); + if (!tier_has_stats(tier)) return; + sink->put_varint64(static_cast(e.prx_bytes.size())); + sink->put_bytes(Slice(e.prx_bytes)); +} + +void write_body(const DictEntry& e, std::string_view prev, IndexTier tier, bool term_stats, + ByteSink* sink) { + write_term_key(e, prev, sink); + sink->put_u8(pack_flags(e)); + write_stats(e, tier, term_stats, sink); + if (e.kind == DictEntryKind::kInline) { + write_inline(e, tier, sink); + } else { + write_pod_ref(e, tier, sink); + } +} + +// ---- Decode entry body ---- + +Status read_term_key(ByteSource* src, std::string_view prev, DictEntry* out) { + uint32_t prefix = 0; + uint32_t suffix_len = 0; + RETURN_IF_ERROR(src->get_varint32(&prefix)); + RETURN_IF_ERROR(src->get_varint32(&suffix_len)); + if (prefix > prev.size()) { + return Status::Error( + "dict_entry: prefix_len exceeds prev_term length"); + } + Slice suffix; + RETURN_IF_ERROR(src->get_bytes(suffix_len, &suffix)); + out->term.assign(prev.substr(0, prefix)); + out->term.append(reinterpret_cast(suffix.data()), suffix.size()); + return Status::OK(); +} + +Status read_stats(ByteSource* src, IndexTier tier, bool term_stats, DictEntry* out) { + RETURN_IF_ERROR(src->get_varint32(&out->df)); + out->term_stats_present = tier_has_stats(tier) && term_stats; + if (!out->term_stats_present) return Status::OK(); + RETURN_IF_ERROR(src->get_varint64(&out->ttf_delta)); + RETURN_IF_ERROR(src->get_varint64(&out->max_freq)); + return Status::OK(); +} + +// Reads the slim/inline region codec metadata (mode/uncomp/[crc]) and fills the +// dd/freq region disk_len from the supplied total/split lengths. has_crc=false +// (INLINE entries, format v2) means no per-region crc was stored: the on-disk +// crc field is absent and region decode must skip crc verification (verify_crc= +// false) since the dict block's own crc32c already covers the inline bytes. +Status read_region_meta(ByteSource* src, IndexTier tier, bool has_crc, uint64_t dd_disk_len, + uint64_t freq_disk_len, DictEntry* out) { + uint8_t mode = 0; + RETURN_IF_ERROR(src->get_u8(&mode)); + if ((mode & ~0x3u) != 0) { + return Status::Error( + "dict_entry: unknown win_mode bits"); + } + out->dd_meta.zstd = (mode & (1u << 0)) != 0; + out->dd_meta.disk_len = dd_disk_len; + out->dd_meta.verify_crc = has_crc; + RETURN_IF_ERROR(src->get_varint64(&out->dd_meta.uncomp_len)); + if (has_crc) RETURN_IF_ERROR(src->get_fixed32(&out->dd_meta.crc)); + if (!tier_has_stats(tier)) { + if (mode & (1u << 1)) { + return Status::Error( + "dict_entry: freq mode set without freq tier"); + } + return Status::OK(); + } + out->freq_meta.zstd = (mode & (1u << 1)) != 0; + out->freq_meta.disk_len = freq_disk_len; + out->freq_meta.verify_crc = has_crc; + RETURN_IF_ERROR(src->get_varint64(&out->freq_meta.uncomp_len)); + if (has_crc) RETURN_IF_ERROR(src->get_fixed32(&out->freq_meta.crc)); + return Status::OK(); +} + +Status read_pod_ref(ByteSource* src, IndexTier tier, DictEntry* out) { + RETURN_IF_ERROR(src->get_varint64(&out->frq_off_delta)); + RETURN_IF_ERROR(src->get_varint64(&out->frq_len)); + if (out->enc == DictEntryEnc::kWindowed) { + RETURN_IF_ERROR(src->get_varint64(&out->prelude_len)); + RETURN_IF_ERROR(src->get_varint64(&out->frq_docs_len)); + if (out->prelude_len == 0 || out->prelude_len > out->frq_docs_len || + out->frq_docs_len > out->frq_len) { + return Status::Error( + "dict_entry: invalid windowed docs prefix"); + } + } else { + RETURN_IF_ERROR(src->get_varint64(&out->frq_docs_len)); + if (out->frq_docs_len > out->frq_len) { + return Status::Error( + "dict_entry: frq_docs_len exceeds frq_len"); + } + RETURN_IF_ERROR(read_region_meta(src, tier, /*has_crc=*/true, out->frq_docs_len, + out->frq_len - out->frq_docs_len, out)); + } + if (!tier_has_stats(tier)) return Status::OK(); + RETURN_IF_ERROR(src->get_varint64(&out->prx_off_delta)); + RETURN_IF_ERROR(src->get_varint64(&out->prx_len)); + return Status::OK(); +} + +Status read_byte_blob(ByteSource* src, std::vector* out) { + uint64_t len = 0; + RETURN_IF_ERROR(src->get_varint64(&len)); + Slice bytes; + RETURN_IF_ERROR(src->get_bytes(static_cast(len), &bytes)); + out->assign(bytes.data(), bytes.data() + bytes.size()); + return Status::OK(); +} + +Status read_inline(ByteSource* src, IndexTier tier, DictEntry* out) { + RETURN_IF_ERROR(read_byte_blob(src, &out->frq_bytes)); + RETURN_IF_ERROR(src->get_varint64(&out->inline_dd_disk_len)); + if (out->inline_dd_disk_len > out->frq_bytes.size()) { + return Status::Error( + "dict_entry: inline_dd_disk_len exceeds frq_bytes"); + } + const uint64_t freq_disk_len = + static_cast(out->frq_bytes.size()) - out->inline_dd_disk_len; + // INLINE entries store no per-region crc (covered by the block crc): + // has_crc=false. + RETURN_IF_ERROR(read_region_meta(src, tier, /*has_crc=*/false, out->inline_dd_disk_len, + freq_disk_len, out)); + if (!tier_has_stats(tier)) return Status::OK(); + RETURN_IF_ERROR(read_byte_blob(src, &out->prx_bytes)); + return Status::OK(); +} + +Status read_locator(ByteSource* src, IndexTier tier, DictEntry* out) { + if (out->kind == DictEntryKind::kInline) return read_inline(src, tier, out); + return read_pod_ref(src, tier, out); +} + +// Read entry_len (= body length) and verify that src has enough remaining +// bytes. +Status read_entry_len(ByteSource* src, uint64_t* total) { + RETURN_IF_ERROR(src->get_varint64(total)); + if (*total > src->remaining()) { + return Status::Error( + "dict_entry: entry_len out of range"); + } + return Status::OK(); +} + +} // namespace + +Status encode_dict_entry(const DictEntry& entry, std::string_view prev_term, IndexTier tier, + ByteSink* sink, bool term_stats) { + if (sink == nullptr) + return Status::Error("dict_entry: sink is null"); + + // Serialize the body into a temporary buffer first to obtain the exact + // length, then write entry_len + body. CRC verification is done uniformly at + // the DICT block level (covering block header + all entries + anchor table); + // CRC is not repeated at the entry level, to keep slim/inline low-frequency + // terms maximally compact (spec §DICT block/§dict entry). + ByteSink body; + write_body(entry, prev_term, tier, term_stats, &body); + sink->put_varint64(static_cast(body.size())); + sink->put_bytes(body.view()); + return Status::OK(); +} + +Status decode_dict_entry_key(ByteSource* src, std::string_view prev_term, DictEntry* out, + size_t* body_start, uint64_t* entry_total) { + if (src == nullptr || out == nullptr) { + return Status::Error("dict_entry: src / out is null"); + } + *out = DictEntry {}; + + uint64_t total = 0; + RETURN_IF_ERROR(read_entry_len(src, &total)); + *body_start = src->position(); + *entry_total = total; + + return read_term_key(src, prev_term, out); +} + +Status decode_dict_entry_rest(ByteSource* src, IndexTier tier, size_t body_start, + uint64_t entry_total, DictEntry* out, bool term_stats) { + if (src == nullptr || out == nullptr) { + return Status::Error("dict_entry: src / out is null"); + } + // Body-decode seam: count exactly the entries whose body we materialize. + body_decode_atomic().fetch_add(1, std::memory_order_relaxed); + + uint8_t flags = 0; + RETURN_IF_ERROR(src->get_u8(&flags)); + apply_flags(flags, out); + RETURN_IF_ERROR(read_stats(src, tier, term_stats, out)); + RETURN_IF_ERROR(read_locator(src, tier, out)); + + // The body must consume exactly entry_len bytes; otherwise the structure is + // inconsistent with the tier. + const size_t consumed = src->position() - body_start; + if (consumed != static_cast(entry_total)) { + return Status::Error( + "dict_entry: body length does not match entry_len"); + } + return Status::OK(); +} + +Status skip_dict_entry_body(ByteSource* src, size_t body_start, uint64_t entry_total) { + if (src == nullptr) { + return Status::Error("dict_entry: src is null"); + } + const size_t consumed = src->position() - body_start; + if (consumed > static_cast(entry_total)) { + return Status::Error( + "dict_entry: term key overruns entry body"); + } + const size_t advance = static_cast(entry_total) - consumed; + Slice unused; + return src->get_bytes(advance, &unused); +} + +Status decode_dict_entry(ByteSource* src, std::string_view prev_term, IndexTier tier, + DictEntry* out, bool term_stats) { + size_t body_start = 0; + uint64_t entry_total = 0; + RETURN_IF_ERROR(decode_dict_entry_key(src, prev_term, out, &body_start, &entry_total)); + return decode_dict_entry_rest(src, tier, body_start, entry_total, out, term_stats); +} + +Status skip_dict_entry(ByteSource* src) { + if (src == nullptr) + return Status::Error("dict_entry: src is null"); + uint64_t total = 0; + RETURN_IF_ERROR(read_entry_len(src, &total)); + Slice unused; + return src->get_bytes(static_cast(total), &unused); +} + +uint64_t dict_entry_body_decode_count() { + return body_decode_atomic().load(std::memory_order_relaxed); +} + +void reset_dict_entry_counters() { + body_decode_atomic().store(0, std::memory_order_relaxed); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/dict_entry.h b/be/src/storage/index/snii/format/dict_entry.h new file mode 100644 index 00000000000000..f35fb680c29aeb --- /dev/null +++ b/be/src/storage/index/snii/format/dict_entry.h @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/frq_pod.h" + +// DictEntry —— on-disk encoding/decoding of a dict entry. +// +// Byte layout (see docs/design/SNII-design-spec.source.md "dict entry" +// section): +// entry_len varint # byte length of entry body, allowing reader to skip +// unknown extensions or fast-skip entries +// --- entry body begins here, covered by entry_len --- +// prefix_len varint # length of shared prefix with prev_term +// suffix_len varint # number of suffix bytes +// suffix u8[] # suffix bytes that differ from prev_term +// flags u8 # bit0 kind / bit1 enc / bit2 has_sb / bit3 +// has_champion(=0) / bit4 offsets_ref(=0) df varint ttf_delta varint +// # only when tier>=T2 max_freq varint # only when tier>=T2 locator: +// pod_ref: frq_off_delta varint, frq_len varint, +// [prelude_len varint, frq_docs_len varint when enc=windowed] +// # docs-only prefix [prelude][dd-block]; windowed entries +// carry # per-window region metadata in the prelude. +// [frq_docs_len varint, slim region meta when enc=slim]: +// # frq_docs_len == dd region on-disk length; the docs-only +// prefix # [frq_off, frq_off+frq_docs_len) a docid-only reader +// fetches # without the freq region. win_mode u8 (bit0 +// dd_zstd, bit1 freq_zstd) dd_uncomp_len varint, crc_dd u32 +// [freq_uncomp_len varint, crc_freq u32 when tier>=T2] +// # The single slim window is [dd_region][freq_region]; +// dd_disk_len # = frq_docs_len, freq_disk_len = frq_len - +// frq_docs_len. +// [prx_off_delta varint, prx_len varint when tier>=T2] +// inline: frq_len varint, frq_bytes u8[], # frq_bytes = +// [dd_region][freq_region] +// slim region meta (as above, sans frq_docs_len which == dd disk +// len +// carried as inline_dd_disk_len varint), +// [prx_len varint, prx_bytes u8[] when tier>=T2] +// --- entry body ends --- +// +// CRC verification is performed at the DICT block level (covering block header +// + all entries + anchor offset table), no per-entry CRC to keep slim/inline +// low-frequency terms compact (spec §DICT block line 330/348). tier and +// positions capability are provided by per-index meta (not stored redundantly +// inside entries): when tier>=T2, ttf_delta / max_freq and .prx locator/bytes +// are written. +namespace doris::snii::format { + +// Dict entry: inline or pod-ref (two states), self-described length, supports +// intra-block front coding. +struct DictEntry { + // term key (front coding relative to prev_term is applied during + // encode/decode; full term stored here). + std::string term; + + // flags. + DictEntryKind kind = DictEntryKind::kPodRef; + DictEntryEnc enc = DictEntryEnc::kSlim; + bool has_sb = false; + + // term stats. + uint32_t df = 0; + uint64_t ttf_delta = 0; // only when tier>=T2 AND the block carries term stats + // G16-f: false when decoded from a kNoTermStats block (freq-dropped index): + // ttf_delta/max_freq above are then meaningless defaults, NOT real zeros. + // Consumers that need them (stats provider / BM25) must check this flag. + bool term_stats_present = true; + uint64_t max_freq = 0; // only when tier>=T2 + + // pod_ref locator. + uint64_t frq_off_delta = 0; + uint64_t frq_len = 0; + uint64_t prelude_len = 0; // only when enc=windowed + uint64_t frq_docs_len = 0; // pod_ref docs-only prefix length + uint64_t prx_off_delta = 0; // only when tier>=T2 + uint64_t prx_len = 0; // only when tier>=T2 + + // slim/inline single-window region codecs. The window is + // [dd_region][freq_region] (no self-describing header). dd_meta drives the + // docs-only decode; freq_meta the scoring decode (only when tier>=T2). For + // slim pod_ref dd_meta.disk_len == frq_docs_len; for inline it is stored as + // inline_dd_disk_len. + FrqRegionMeta dd_meta; + FrqRegionMeta freq_meta; // only when tier>=T2 + uint64_t inline_dd_disk_len = 0; // only for inline: dd region on-disk length + + // inline payload. + std::vector frq_bytes; // = [dd_region][freq_region] + std::vector prx_bytes; // only when tier>=T2 +}; + +// Encodes an entry into sink (appending) using the layout above, with front +// coding relative to prev_term. tier determines whether optional fields are +// written. term_stats == false (G16-f: freq-dropped indexes, declared by the +// block header's kNoTermStats flag) omits the ttf_delta/max_freq varints -- +// they serve only BM25 scoring, dead on an index that dropped freq; df stays. +// Region metadata (freq/prx locators) remains tier-conditioned. +Status encode_dict_entry(const DictEntry& entry, std::string_view prev_term, IndexTier tier, + ByteSink* sink, bool term_stats = true); + +// Decodes one entry from the current position of src; term is reconstructed +// from prev_term + suffix. Verifies the trailing CRC; out-of-range / CRC +// mismatch / invalid prefix_len all return Corruption. term_stats must match +// the writer's choice (the dict block header flag carries it). +Status decode_dict_entry(ByteSource* src, std::string_view prev_term, IndexTier tier, + DictEntry* out, bool term_stats = true); + +// Skips one entry using only entry_len (does not parse internal fields or +// verify CRC). +Status skip_dict_entry(ByteSource* src); + +// ---- Key-first decode primitives (T07) ---- +// +// decode_dict_entry is split into a "key" stage and a "rest" (body) stage so a +// caller scanning many entries can decide on the (front-coded) term key alone +// whether the entry's body is worth materializing. decode_dict_entry below is +// re-expressed as key + rest and stays byte-for-byte identical in output. + +// Reads only entry_len + the front-coded term key. On return src is positioned +// at the start of the entry body (the flags byte). out is reset to defaults and +// out->term is reconstructed from prev_term + suffix; no other field is touched. +// *body_start receives the absolute src position of the body (right after the +// entry_len varint) and *entry_total the body byte length (already bounds-checked +// against src->remaining()). Used to drive key-first scans (find_term, prefix +// streaming) that skip non-matching bodies. +Status decode_dict_entry_key(ByteSource* src, std::string_view prev_term, DictEntry* out, + size_t* body_start, uint64_t* entry_total); + +// Continues from the position decode_dict_entry_key left at: reads +// flags/stats/locator into *out and verifies the body consumed exactly +// entry_total bytes (body_start anchors the consumed count). Increments the +// body-decode counter seam (see dict_entry_body_decode_count) at its top so +// tests can assert how many entry bodies a scan actually materialized. +Status decode_dict_entry_rest(ByteSource* src, IndexTier tier, size_t body_start, + uint64_t entry_total, DictEntry* out, bool term_stats = true); + +// Skips the remaining body bytes after decode_dict_entry_key, advancing src to +// the next entry without parsing flags/stats/locator. advance = entry_total - +// (src.position() - body_start); the term key already consumed by the key stage +// is accounted for. +Status skip_dict_entry_body(ByteSource* src, size_t body_start, uint64_t entry_total); + +// Test-only instrumentation seam: dict_entry_body_decode_count() returns a +// process-global count of decode_dict_entry_rest calls -- i.e. how many entry +// bodies (flags/stats/locator + any inline byte copies) a scan materialized. +// Key-first find_term / prefix streaming drive this toward the number of +// produced hits instead of the number of entries walked. Counters use relaxed +// atomics; reset between tests. +uint64_t dict_entry_body_decode_count(); +void reset_dict_entry_counters(); + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/format_constants.h b/be/src/storage/index/snii/format/format_constants.h new file mode 100644 index 00000000000000..4e86283d76db76 --- /dev/null +++ b/be/src/storage/index/snii/format/format_constants.h @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +// SNII container and per-section on-disk contract constants. +// Once published, these values are format semantics; changes require bumping +// format_version and maintaining a compatibility policy. All multi-byte +// fixed-width fields are little-endian; variable-length integers use LEB128 +// (see snii/encoding/varint.h). +namespace doris::snii::format { + +// ---- Container-level magic / version ---- +// "SNII" reads as 0x49494E53 in little-endian. +inline constexpr uint32_t kContainerMagic = 0x49494E53u; // 'S''N''I''I' +inline constexpr uint32_t kTailMagic = 0x4C494154u; // 'T''A''I''L' +inline constexpr uint16_t kFormatVersion = 2; +inline constexpr uint16_t kMinReaderVersion = 2; +// Self-describing version of the meta layout (the per-index meta header AND the +// tail meta region share this single constant; a reader fails fast with +// Corruption on any mismatch). This is a from-scratch, pre-launch format: there +// is exactly ONE meta layout, so the value is 1. Bump it only AFTER launch, +// when a real on-disk change must coexist with already-written indexes -- +// pre-launch changes just fold into v1. +inline constexpr uint16_t kMetaFormatVersion = 1; + +// ---- SectionFramer section type ids (within per-index meta / tail region) +// ---- +enum class SectionType : uint8_t { + kStatsBlock = 1, + kSampledTermIndex = 2, + kDictBlockDirectory = 3, + kXFilter = 4, // reserved: legacy embedded XFilter; meta no longer emits/reads it + kSectionRefs = 5, + kPerIndexMetaHeader = 6, + kLogicalIndexDirectory = 7, + kTailMetaHeader = 8, + kFeatureBits = 9, + // OPTIONAL per-index meta section (G01): payload = varint64 + // bigram_prune_min_df, then (G15) varint64 bigram_prune_max_df -- the + // effective phrase-bigram df-prune LOWER and UPPER bounds the writer + // applied to THIS index (absolute doc counts; 0 == that gate inactive). + // Emitted only when pruning was active (either bound > 0); absent == both + // 0 == legacy semantics (every adjacent pair materialized, so a bigram + // dict miss means "no adjacency"). Pre-G15 writers emitted only the min + // varint: decode defaults the missing max to 0, and trailing payload bytes + // past the known fields stay IGNORED for future extension. Readers that + // predate this section skip it (unknown optional type), keeping old + // binaries readable on new segments and new binaries on old segments. + // CAVEAT (version skew): a pre-G15 reader arms its dict-miss fallback off + // the min varint ALONE, so a segment written with only the max gate armed + // (min == 0, a non-default config) would answer 2-term phrases for its + // max-pruned pairs as EMPTY on such a binary. Tolerable only because no + // released reader predates G15 (SNII ships with both gates); any future + // field that widens the dict-miss-fallback trigger must instead arm one of + // the fields old readers already key on. + kBigramPruneInfo = 10, + // G13: zstd-compressed carriers for the two large embedded meta sub-sections + // (they are highly compressible sorted string/offset tables and dominate the + // per-segment meta blob fetched serially at open). Payload = varint64 + // uncomp_len followed by zstd(original full frame), where "original full + // frame" is the byte-exact kSampledTermIndex / kDictBlockDirectory frame + // (type+len+payload+crc32c) the raw layout would have embedded. Decompression + // therefore reproduces the legacy frame verbatim and the sub-module readers + // (which re-verify the inner crc) stay unchanged. The writer emits these ONLY + // when the raw frame reaches kMetaSectionCompressMinBytes AND compression + // shrinks it; otherwise the raw frame is emitted as before, so small/legacy + // segments remain byte-identical and always readable. + kSampledTermIndexZstd = 11, + kDictBlockDirectoryZstd = 12, +}; + +// ---- Logical index postings storage content configuration (fixed per logical +// index, not per-term) ---- Determines whether to write freq / positions / +// norms+stats. +enum class IndexConfig : uint8_t { + kDocsOnly = 0, // docid only: term/match filtering + kDocsPositions = 1, // docid+positions (+freq only when the caller keeps + // it -- SniiIndexInput::write_freq, G16-c): MATCH_PHRASE + kDocsPositionsScoring = 2, // + norms + stats: phrase + BM25 + kPositionsOffsets = 3, // reserved (highlight/RAG), not implemented in this release +}; + +// term stats / postings capability tiers: only tier>=kT2 writes +// ttf_delta/max_freq and .prx. +enum class IndexTier : uint8_t { + kT1 = 1, // docs-only + kT2 = 2, // docs-positions + kT3 = 3, // docs-positions-scoring +}; + +inline constexpr IndexTier tier_of(IndexConfig cfg) { + return cfg == IndexConfig::kDocsOnly ? IndexTier::kT1 + : cfg == IndexConfig::kDocsPositions ? IndexTier::kT2 + : IndexTier::kT3; // scoring / offsets +} +inline constexpr bool has_positions(IndexConfig cfg) { + return cfg != IndexConfig::kDocsOnly; +} +inline constexpr bool has_scoring(IndexConfig cfg) { + return cfg == IndexConfig::kDocsPositionsScoring; +} + +// ---- DictEntry flags bit definitions ---- +namespace dict_flags { +inline constexpr uint8_t kKind = 1u << 0; // 0=pod_ref / 1=inline +inline constexpr uint8_t kEnc = 1u << 1; // 0=slim / 1=windowed +inline constexpr uint8_t kHasSb = 1u << 2; // posting prelude includes sub-block directory +inline constexpr uint8_t kHasChampion = 1u << 3; // v1 always 0 +inline constexpr uint8_t kOffsetsRef = 1u << 4; // v1 always 0 +// bit5-7 reserved +} // namespace dict_flags + +enum class DictEntryKind : uint8_t { kPodRef = 0, kInline = 1 }; +enum class DictEntryEnc : uint8_t { kSlim = 0, kWindowed = 1 }; + +// ---- .prx window codec (codec byte bit0-5) ---- +// kRaw : plaintext varint payload (doc_count, per-doc pos_count + position +// deltas). kZstd : zstd-compressed plaintext payload (legacy reader still +// supported). kPfor : doc_count + per-doc pos_count (varint), then position +// deltas bit-packed +// as PFOR runs (kFrqBaseUnit each). No entropy coding -> far cheaper +// build CPU than zstd while staying competitive on size for ascending +// deltas. +enum class PrxCodec : uint8_t { + kRaw = 0, + kZstd = 1, + kPfor = 2 /* bit7 cont-reserved */ +}; + +// ---- Build-time parameters (not format semantics; may be tuned against real +// metrics) ---- +inline constexpr uint32_t kFrqBaseUnit = 256; // window base unit +inline constexpr uint32_t kSlimDfThreshold = 512; // df < this → slim +inline constexpr uint32_t kDefaultInlineThreshold = 256; // slim encoded bytes ≤ this → inline +// Adaptive window sizing (design #4): high-df windowed terms use larger windows +// to cut prelude rows + per-window header/crc overhead. Windows remain a whole +// multiple of kFrqBaseUnit so .prx alignment and win_base/last_docid semantics +// are preserved. A term whose df >= kAdaptiveWindowDfThreshold splits into +// kAdaptiveWindowDocs-sized windows instead of kFrqBaseUnit-sized ones. +inline constexpr uint32_t kAdaptiveWindowDfThreshold = 8192; // df >= this -> larger windows +inline constexpr uint32_t kAdaptiveWindowDocs = 1024; // larger window size (4 * base unit) +// G16-e: prune-mode (docs-only) phrase-bigram terms split into much larger +// windows. The DOMINANT 2-term hit path reads the FULL docs-only prefix (no +// window narrowing), so fine windows bought nothing there while paying +// per-window prelude rows; larger regions also give the G16-i dd zstd a real +// compression context. NOTE: the 3+-term phrase chain DOES window-narrow +// bigram links via candidates -- those reads now fetch up to 16x larger dd +// regions per covered window (rare path, bounded by the candidate count). +// 64 * base unit; readers accept any whole multiple of kFrqBaseUnit up to the +// 16M-doc window cap. Legacy-mode bigrams (positions kept) stay on the +// adaptive sizing, preserving byte-identical legacy output. +inline constexpr uint32_t kBigramWindowDocs = 16384; +inline constexpr uint32_t kDefaultTargetDictBlockBytes = 64 * 1024; +// G13: embedded meta sub-sections (SampledTermIndex / DictBlockDirectory frames) +// at or above this raw size are emitted zstd-compressed (kSampledTermIndexZstd / +// kDictBlockDirectoryZstd); smaller ones stay raw -- compression overhead is not +// worth it below a few KB and the raw layout keeps small segments byte-identical +// to the pre-G13 format. A build-time parameter, not format semantics: readers +// accept both layouts regardless of the value. +inline constexpr size_t kMetaSectionCompressMinBytes = 4 * 1024; + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/frq_pod.cpp b/be/src/storage/index/snii/format/frq_pod.cpp new file mode 100644 index 00000000000000..ddea9e39bbf4f7 --- /dev/null +++ b/be/src/storage/index/snii/format/frq_pod.cpp @@ -0,0 +1,236 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/frq_pod.h" + +#include +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/common/uninitialized_buffer.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/encoding/pfor.h" +#include "storage/index/snii/encoding/zstd_codec.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::format { +namespace { + +// Auto-compression threshold: use raw when a region is smaller than this byte +// count (zstd gain is negligible and metadata overhead is relatively large). +inline constexpr size_t kAutoZstdMinBytes = 512; +// Default zstd level for auto mode. +inline constexpr int kDefaultZstdLevel = 3; +// Maximum decompressed byte size for a single region. Guards against a +// corrupted uncomp_len read from S3 that inflated to a huge value: sanity-check +// before allocating/decompressing to avoid GB-scale allocations. Windows are +// 256-doc aligned and normally far smaller than this. +inline constexpr uint32_t kMaxRegionUncompBytes = 256u * 1024 * 1024; +// Maximum doc count per .frq window (guards against a corrupted n). Window +// baseline is 256, practical combined cap is 2048, so this is a loose but +// astronomically-large-number-blocking upper bound. +inline constexpr uint32_t kMaxWindowDocs = 1u << 24; + +// Encode a uint32 array into multiple PFOR runs, each of 256 (kFrqBaseUnit) +// elements. n / run count is not written: the number of runs is derived from +// total length n and kFrqBaseUnit, and the decoder computes it the same way. +void encode_pfor_runs(std::span values, ByteSink* out) { + size_t n = values.size(); + for (size_t off = 0; off < n; off += kFrqBaseUnit) { + size_t run = (n - off < kFrqBaseUnit) ? (n - off) : kFrqBaseUnit; + pfor_encode(values.data() + off, run, out); + } +} + +// Decode n uint32 values from source (multiple PFOR runs of 256 each). +Status decode_pfor_runs(ByteSource* src, size_t n, std::vector* out) { + // Sized then fully overwritten by pfor_decode below; no zero-fill needed. + resize_uninitialized(*out, n); + for (size_t off = 0; off < n; off += kFrqBaseUnit) { + size_t run = (n - off < kFrqBaseUnit) ? (n - off) : kFrqBaseUnit; + RETURN_IF_ERROR(pfor_decode(src, run, out->data() + off)); + } + return Status::OK(); +} + +// Verifies docids are ascending and the first entry is not below win_base. +Status validate_docs(std::span docs, uint64_t win_base) { + if (docs.empty()) return Status::OK(); + if (static_cast(docs.front()) < win_base) { + return Status::Error("frq: first docid below win_base"); + } + for (size_t i = 1; i < docs.size(); ++i) { + if (docs[i] < docs[i - 1]) { + return Status::Error( + "frq: docids must be ascending"); + } + } + return Status::OK(); +} + +// Decision: given level and plaintext length, determine whether to compress. +bool should_compress(int level, size_t plain_len) { + if (level == 0) return false; // force raw + if (level > 0) return true; // force zstd + return plain_len >= kAutoZstdMinBytes; // auto +} + +// Encodes one region's plaintext into raw or zstd, appends the on-disk bytes to +// out, and fills meta (mode/uncomp_len/disk_len/crc). The region carries no +// header. +Status emit_region(Slice plain, int level, ByteSink* out, FrqRegionMeta* meta) { + if (out == nullptr || meta == nullptr) { + return Status::Error("frq: null region out"); + } + meta->uncomp_len = plain.size(); + if (should_compress(level, plain.size())) { + // zstd needs its own buffer: the compressed bytes differ from `plain`. + std::vector disk; + meta->zstd = true; + RETURN_IF_ERROR(zstd_compress(plain, level > 0 ? level : kDefaultZstdLevel, &disk)); + meta->disk_len = static_cast(disk.size()); + meta->crc = crc32c(Slice(disk)); + out->put_bytes(Slice(disk)); + return Status::OK(); + } + // Raw: the on-disk bytes ARE `plain` (a view over the caller's contiguous + // ByteSink), so crc and emit straight from it -- no temp `disk` alloc/copy. + // disk_len MUST stay == plain.size(): open_region enforces uncomp_len == + // disk_len for raw regions. Byte-identical to the former disk.assign() path + // (disk == plain, so crc32c(disk) == crc32c(plain), put_bytes(disk) == same + // bytes). + meta->zstd = false; + meta->disk_len = static_cast(plain.size()); + meta->crc = crc32c(plain); + out->put_bytes(plain); + return Status::OK(); +} + +// Materializes a region's plaintext (raw borrows the view; zstd decompresses) +// and verifies its crc + slice length against meta. +Status open_region(Slice disk, const FrqRegionMeta& meta, std::vector* holder, + Slice* plain) { + if (disk.size() != static_cast(meta.disk_len)) { + return Status::Error( + "frq: region slice length mismatch"); + } + if (meta.uncomp_len > kMaxRegionUncompBytes) { + return Status::Error( + "frq: region uncomp_len exceeds sane cap"); + } + // Inline entries (verify_crc=false) carry no per-region crc: their on-disk + // bytes are covered by the enclosing dict block's block-level crc32c, so the + // region crc would be redundant. POD-ref regions keep their own crc check. + if (meta.verify_crc && crc32c(disk) != meta.crc) { + return Status::Error( + "frq: region crc mismatch"); + } + if (!meta.zstd) { + if (meta.uncomp_len != meta.disk_len) { + return Status::Error( + "frq: raw region length inconsistent"); + } + *plain = disk; + return Status::OK(); + } + RETURN_IF_ERROR(zstd_decompress(disk, static_cast(meta.uncomp_len), holder)); + *plain = Slice(*holder); + return Status::OK(); +} + +} // namespace + +Status build_dd_region(std::span docids_ascending, uint64_t win_base, + int zstd_level_or_neg_for_auto, ByteSink* out, FrqRegionMeta* meta) { + if (out == nullptr || meta == nullptr) { + return Status::Error("frq: null dd region out"); + } + RETURN_IF_ERROR(validate_docs(docids_ascending, win_base)); + ByteSink plain; // VInt n ++ PFOR_runs(doc_delta) + std::vector dd(docids_ascending.size()); + uint64_t prev = win_base; + for (size_t i = 0; i < docids_ascending.size(); ++i) { + dd[i] = static_cast(static_cast(docids_ascending[i]) - prev); + prev = docids_ascending[i]; + } + plain.put_varint32(static_cast(docids_ascending.size())); + encode_pfor_runs(dd, &plain); + return emit_region(plain.view(), zstd_level_or_neg_for_auto, out, meta); +} + +Status build_freq_region(std::span freqs, int zstd_level_or_neg_for_auto, + ByteSink* out, FrqRegionMeta* meta) { + if (out == nullptr || meta == nullptr) { + return Status::Error("frq: null freq region out"); + } + ByteSink plain; + encode_pfor_runs(freqs, &plain); + return emit_region(plain.view(), zstd_level_or_neg_for_auto, out, meta); +} + +Status decode_dd_region(Slice dd_disk, const FrqRegionMeta& meta, uint64_t win_base, + std::vector* docids) { + if (docids == nullptr) + return Status::Error("frq: null docids out"); + std::vector holder; + Slice plain; + RETURN_IF_ERROR(open_region(dd_disk, meta, &holder, &plain)); + ByteSource src(plain); + uint32_t n = 0; + RETURN_IF_ERROR(src.get_varint32(&n)); + if (n > kMaxWindowDocs) + return Status::Error( + "frq: doc count exceeds sane cap"); + RETURN_IF_ERROR(decode_pfor_runs(&src, n, docids)); + if (!src.eof()) { + return Status::Error( + "frq: trailing bytes after dd region payload"); + } + uint64_t cur = win_base; + for (uint32_t i = 0; i < n; ++i) { + cur += (*docids)[i]; + (*docids)[i] = static_cast(cur); + } + return Status::OK(); +} + +Status decode_freq_region(Slice freq_disk, const FrqRegionMeta& meta, size_t doc_count, + std::vector* freqs) { + if (freqs == nullptr) + return Status::Error("frq: null freqs out"); + std::vector holder; + Slice plain; + RETURN_IF_ERROR(open_region(freq_disk, meta, &holder, &plain)); + if (doc_count == 0) { + if (meta.uncomp_len != 0) { + return Status::Error( + "frq: empty freq region expected"); + } + freqs->clear(); + return Status::OK(); + } + ByteSource src(plain); + RETURN_IF_ERROR(decode_pfor_runs(&src, doc_count, freqs)); + if (!src.eof()) { + return Status::Error( + "frq: trailing bytes after freq region payload"); + } + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/frq_pod.h b/be/src/storage/index/snii/format/frq_pod.h new file mode 100644 index 00000000000000..56499bec1955db --- /dev/null +++ b/be/src/storage/index/snii/format/frq_pod.h @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +// .frq region codec (FrqPod): doc-delta (dd) and freq postings, columnar + PFOR +// (see docs/design SNII "frq design" and the read-byte-optimizations +// design 1.6). +// +// PHASE D (posting-level dd/freq grouping): windows are NO LONGER +// self-describing. A windowed .frq payload is laid out as +// [prelude][dd-block][freq-block] +// where the dd-block concatenates every window's dd_region and the freq-block +// concatenates every window's freq_region. Each region is independently encoded +// (raw or zstd, chosen by size) and the per-window codec metadata (mode, +// lengths, crc, offsets) is hoisted into the frq_prelude rows -- the region +// bytes carry NO header. This makes the docs-only prefix ([prelude][dd-block]) +// ONE contiguous run a docid-only / phrase reader can fetch in a single range, +// skipping the freq-block entirely. +// +// dd_region plaintext = VInt n ++ PFOR_runs(doc_delta) # n = doc count +// dd[0] = first_docid - win_base; dd[i] = docid[i] - docid[i-1]; win_base is +// the previous window's last docid (first window = 0). +// freq_region plaintext = PFOR_runs(freq) # present iff +// has_freq PFOR runs are segmented at 256 docs (kFrqBaseUnit); a partial +// segment writes the remainder. Variable-length integers reuse +// snii/encoding/varint; PFOR reuses snii/encoding/pfor; crc32c covers each +// region's ON-DISK bytes. +namespace doris::snii::format { + +// Codec metadata for ONE encoded region (dd or freq), hoisted into the prelude. +// The region's on-disk bytes are pure payload (no header); these fields drive +// the decode. crc covers the on-disk (disk_len) bytes. +struct FrqRegionMeta { + bool zstd = false; // true => disk bytes are zstd(plaintext); false => raw + uint64_t uncomp_len = 0; // plaintext byte length (== disk_len when raw) + uint64_t disk_len = 0; // on-disk byte length of this region + uint32_t crc = 0; // crc32c of the on-disk (disk_len) bytes + // When false, decode_*_region SKIPS the per-region crc check (and the writer + // omits the 4-byte crc from the dict entry). Set false for INLINE entries: + // their region bytes live inside the dict block, whose own block-level crc32c + // already covers them, so a per-region crc is fully redundant. POD-ref + // regions (slim/windowed) live in the separately-fetched .frq POD -- their + // crc stays. + bool verify_crc = true; +}; + +// Encodes a window's dd_region plaintext (VInt n ++ PFOR_runs(doc_delta)) into +// raw or zstd (per zstd_level_or_neg_for_auto), APPENDS the on-disk bytes to +// out, and fills meta (mode/uncomp_len/disk_len/crc). The region carries no +// header. docids_ascending: ascending docids in this window (single doc or +// empty allowed). win_base: previous window's last docid (first window = 0); +// requires docids[0] >= win_base. zstd_level_or_neg_for_auto: <0 auto (zstd +// when large enough, else raw); 0 force +// raw; >0 force zstd at that level. +// Non-ascending docids / first_docid < win_base / null out returns +// InvalidArgument. +Status build_dd_region(std::span docids_ascending, uint64_t win_base, + int zstd_level_or_neg_for_auto, ByteSink* out, FrqRegionMeta* meta); + +// Vector convenience overload (forwards a span view; no copy of the elements). +inline Status build_dd_region(const std::vector& docids_ascending, uint64_t win_base, + int zstd_level_or_neg_for_auto, ByteSink* out, FrqRegionMeta* meta) { + return build_dd_region(std::span(docids_ascending), win_base, + zstd_level_or_neg_for_auto, out, meta); +} + +// Encodes a window's freq_region plaintext (PFOR_runs(freq)) into raw or zstd, +// APPENDS the on-disk bytes to out, and fills meta. Empty freqs yields a +// zero-length region. Null out returns InvalidArgument. +Status build_freq_region(std::span freqs, int zstd_level_or_neg_for_auto, + ByteSink* out, FrqRegionMeta* meta); + +// Vector convenience overload (forwards a span view; no copy of the elements). +inline Status build_freq_region(const std::vector& freqs, int zstd_level_or_neg_for_auto, + ByteSink* out, FrqRegionMeta* meta) { + return build_freq_region(std::span(freqs), zstd_level_or_neg_for_auto, out, + meta); +} + +// Decodes a dd_region from its on-disk slice (exactly disk_len bytes) + meta + +// win_base, reconstructing ascending docids. Verifies meta.crc against the +// slice. crc mismatch / wrong slice length / truncation / decompression / +// oversized count all return a non-OK Status. The freq region is irrelevant +// here (docs-only path). +Status decode_dd_region(Slice dd_disk, const FrqRegionMeta& meta, uint64_t win_base, + std::vector* docids); + +// Decodes a freq_region from its on-disk slice (exactly disk_len bytes) + meta, +// producing doc_count freqs. Verifies meta.crc. doc_count == 0 yields empty +// freqs (and requires a zero-length region). crc mismatch / wrong slice length +// / etc. return a non-OK Status. +Status decode_freq_region(Slice freq_disk, const FrqRegionMeta& meta, size_t doc_count, + std::vector* freqs); + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/frq_prelude.cpp b/be/src/storage/index/snii/format/frq_prelude.cpp new file mode 100644 index 00000000000000..cbd67ee6feefdc --- /dev/null +++ b/be/src/storage/index/snii/format/frq_prelude.cpp @@ -0,0 +1,642 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/frq_prelude.h" + +#include +#include +#include + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" + +namespace doris::snii::format { + +namespace { + +// Anti-DoS: a segment holds at most ~15M docs (>=1 doc/window), so 1<<24 +// windows is a generous ceiling that still prevents multi-GB allocations from a +// crafted N. (crc32c is not a MAC and cannot defend a re-stamped inflated count.) +constexpr uint64_t kMaxWindows = 1ull << 24; + +uint64_t ceil_div(uint64_t a, uint64_t b) { + return (a + b - 1) / b; +} + +uint8_t make_flags(const FrqPreludeColumns& cols) { + uint8_t flags = 0; + if (cols.has_freq) flags |= frq_prelude_flags::kHasFreq; + if (cols.has_prx) flags |= frq_prelude_flags::kHasPrx; + return flags; +} + +uint8_t make_win_mode(const WindowMeta& m, bool has_freq) { + uint8_t mode = 0; + if (m.dd_zstd) mode |= frq_win_mode::kDdZstd; + if (has_freq && m.freq_zstd) mode |= frq_win_mode::kFreqZstd; + return mode; +} + +Status checked_add_u64(uint64_t lhs, uint64_t rhs, const char* message, uint64_t* out) { + if (rhs > std::numeric_limits::max() - lhs) { + return Status::Error(message); + } + *out = lhs + rhs; + return Status::OK(); +} + +Status checked_u32(uint64_t value, const char* message, uint32_t* out) { + if (value > std::numeric_limits::max()) { + return Status::Error(message); + } + *out = static_cast(value); + return Status::OK(); +} + +Status validate_window_doc_count(bool first_window, uint64_t win_base, uint64_t last_docid, + uint64_t doc_count) { + uint64_t first_docid = 0; + if (!first_window) { + RETURN_IF_ERROR(checked_add_u64(win_base, 1, "frq_prelude: window base exceeds docid range", + &first_docid)); + } + if (last_docid < first_docid) { + return Status::Error( + "frq_prelude: invalid window docid range"); + } + const uint64_t width = last_docid - first_docid + 1; + if (doc_count > width) { + return Status::Error( + "frq_prelude: doc_count exceeds window width"); + } + return Status::OK(); +} + +// Validates builder input: non-null sink, group_size>=1, sane count, and +// non-decreasing absolute last_docid across windows. +Status validate_input(const FrqPreludeColumns& cols, ByteSink* out) { + if (out == nullptr) + return Status::Error("frq_prelude: null sink"); + if (cols.group_size == 0) { + return Status::Error( + "frq_prelude: group_size must be >= 1"); + } + if (cols.windows.size() > kMaxWindows) { + return Status::Error( + "frq_prelude: window count exceeds cap"); + } + for (size_t w = 1; w < cols.windows.size(); ++w) { + if (cols.windows[w].last_docid < cols.windows[w - 1].last_docid) { + return Status::Error( + "frq_prelude: last_docid not monotonic"); + } + } + return Status::OK(); +} + +// Encodes one window row into a per-block sink. last_docid_delta is the row's +// absolute last_docid minus prev_last (the previous window's absolute last). +// +// dd_off/freq_off/prx_off are NOT serialized: each was a pure running prefix sum +// of the per-window disk/prx lengths, which the reader reconstructs (see +// decode_window_row / RunningOffsets). dd_uncomp_len/freq_uncomp_len are written +// ONLY when the region's zstd win_mode bit is set; a raw region's uncomp_len is +// defined to equal its disk_len (open_region enforces uncomp_len == disk_len on +// raw region bytes), so the reader derives it without a stored field. +void encode_window_row(const WindowMeta& m, bool has_freq, bool has_prx, uint64_t prev_last, + ByteSink* block) { + const uint8_t win_mode = make_win_mode(m, has_freq); + block->put_varint64(static_cast(m.last_docid) - prev_last); + block->put_varint64(m.doc_count); + block->put_u8(win_mode); + block->put_varint64(m.dd_disk_len); + if ((win_mode & frq_win_mode::kDdZstd) != 0) { + block->put_varint64(m.dd_uncomp_len); + } + block->put_fixed32(m.crc_dd); + if (has_freq) { + block->put_varint64(m.freq_disk_len); + if ((win_mode & frq_win_mode::kFreqZstd) != 0) { + block->put_varint64(m.freq_uncomp_len); + } + block->put_fixed32(m.crc_freq); + } + if (has_prx) { + block->put_varint64(m.prx_len); + } + block->put_varint64(m.max_freq); + block->put_u8(m.max_norm); +} + +// One super-block's serialized window block plus its directory fields. +struct SuperBlock { + ByteSink block; + uint64_t last_docid = 0; // absolute last docid of this super-block's last window +}; + +// Builds every super-block's window block (row-encoded) and records the running +// absolute last docid at each super-block boundary. +std::vector encode_super_blocks(const FrqPreludeColumns& cols) { + const uint32_t g = cols.group_size; + const size_t n = cols.windows.size(); + std::vector blocks; + blocks.reserve(static_cast(ceil_div(n, g))); + uint64_t prev_last = 0; // previous window's absolute last docid (chains across blocks) + for (size_t start = 0; start < n; start += g) { + const size_t end = std::min(n, start + g); + SuperBlock sb; + for (size_t w = start; w < end; ++w) { + encode_window_row(cols.windows[w], cols.has_freq, cols.has_prx, prev_last, &sb.block); + prev_last = cols.windows[w].last_docid; + } + sb.last_docid = prev_last; + blocks.push_back(std::move(sb)); + } + return blocks; +} + +// Serializes the super_block_dir (one row per super-block) into dir_sink, using +// each block's byte length to compute its offset within the window_dir region. +void encode_super_block_dir(const std::vector& blocks, ByteSink* dir_sink) { + uint64_t prev_last = 0; + uint64_t block_off = 0; + for (const SuperBlock& sb : blocks) { + dir_sink->put_varint64(sb.last_docid - prev_last); + dir_sink->put_varint64(block_off); + dir_sink->put_varint64(sb.block.size()); + prev_last = sb.last_docid; + block_off += sb.block.size(); + } +} + +} // namespace + +Status build_frq_prelude(const FrqPreludeColumns& cols, ByteSink* out) { + RETURN_IF_ERROR(validate_input(cols, out)); + + const std::vector blocks = encode_super_blocks(cols); + ByteSink dir_sink; + encode_super_block_dir(blocks, &dir_sink); + + // covered = header + super_block_dir (the crc covers exactly this region). + ByteSink covered; + covered.put_u8(make_flags(cols)); + covered.put_varint64(cols.windows.size()); + covered.put_varint64(cols.group_size); + covered.put_varint64(blocks.size()); + covered.put_varint64(dir_sink.size()); + covered.put_bytes(dir_sink.view()); + + out->put_bytes(covered.view()); + out->put_fixed32(crc32c(covered.view())); + for (const SuperBlock& sb : blocks) out->put_bytes(sb.block.view()); + return Status::OK(); +} + +namespace { + +// Decoded header fields shared between parse phases. +struct Header { + bool has_freq = false; + bool has_prx = false; + uint64_t n = 0; + uint64_t group_size = 0; + uint64_t n_super = 0; + uint64_t sbdir_len = 0; +}; + +// Verifies the trailing crc covers [start of buffer .. end of super_block_dir]. +// covered_len = header bytes (up to and including sbdir_len) + sbdir_len. +Status verify_covered_crc(Slice prelude, size_t header_end, uint64_t sbdir_len) { + const size_t covered = header_end + static_cast(sbdir_len); + if (covered + sizeof(uint32_t) > prelude.size()) { + return Status::Error( + "frq_prelude: buffer too short for crc region"); + } + uint32_t stored = 0; + ByteSource crc_src(prelude.subslice(covered, sizeof(uint32_t))); + RETURN_IF_ERROR(crc_src.get_fixed32(&stored)); + if (crc32c(prelude.subslice(0, covered)) != stored) { + return Status::Error( + "frq_prelude: crc32c mismatch"); + } + return Status::OK(); +} + +// Parses + validates the header (counts capped before any later reserve). +Status parse_header(ByteSource* src, Header* h) { + uint8_t flags = 0; + RETURN_IF_ERROR(src->get_u8(&flags)); + h->has_freq = (flags & frq_prelude_flags::kHasFreq) != 0; + h->has_prx = (flags & frq_prelude_flags::kHasPrx) != 0; + RETURN_IF_ERROR(src->get_varint64(&h->n)); + RETURN_IF_ERROR(src->get_varint64(&h->group_size)); + RETURN_IF_ERROR(src->get_varint64(&h->n_super)); + RETURN_IF_ERROR(src->get_varint64(&h->sbdir_len)); + if (h->n > kMaxWindows || h->n_super > kMaxWindows) { + return Status::Error( + "frq_prelude: window count exceeds sane cap"); + } + if (h->group_size == 0) { + return Status::Error( + "frq_prelude: group_size is zero"); + } + if (h->n_super != ceil_div(h->n, h->group_size)) { + return Status::Error( + "frq_prelude: n_super inconsistent with N/G"); + } + return Status::OK(); +} + +// One super-block directory row. +struct SbDirRow { + uint64_t last_docid = 0; + uint64_t block_off = 0; + uint64_t block_len = 0; +}; + +// Decodes the super_block_dir region into absolute-last-docid rows, validating +// monotonic last docids and contiguous, in-bounds block offsets. +Status decode_super_block_dir(Slice dir, const Header& h, std::vector* rows, + uint64_t* window_region_len) { + ByteSource src(dir); + rows->clear(); + rows->reserve(static_cast(h.n_super)); + uint64_t prev_last = 0; + uint64_t expect_off = 0; + for (uint64_t s = 0; s < h.n_super; ++s) { + SbDirRow r; + uint64_t ldd = 0; + RETURN_IF_ERROR(src.get_varint64(&ldd)); + RETURN_IF_ERROR(src.get_varint64(&r.block_off)); + RETURN_IF_ERROR(src.get_varint64(&r.block_len)); + RETURN_IF_ERROR(checked_add_u64( + prev_last, ldd, "frq_prelude: super-block last_docid overflow", &r.last_docid)); + uint32_t checked_last = 0; + RETURN_IF_ERROR(checked_u32(r.last_docid, "frq_prelude: super-block last_docid exceeds u32", + &checked_last)); + if (r.last_docid < prev_last || r.block_off != expect_off) { + return Status::Error( + "frq_prelude: super-block dir inconsistent"); + } + expect_off += r.block_len; + prev_last = r.last_docid; + rows->push_back(r); + } + if (!src.eof()) { + return Status::Error( + "frq_prelude: super-block dir has trailing bytes"); + } + *window_region_len = expect_off; + return Status::OK(); +} + +// Validates a per-window codec mode byte against the known bits. +Status check_win_mode(uint8_t mode, bool has_freq) { + if ((mode & ~frq_win_mode::kKnownBits) != 0) { + return Status::Error( + "frq_prelude: unknown win_mode bits"); + } + if (!has_freq && (mode & frq_win_mode::kFreqZstd) != 0) { + return Status::Error( + "frq_prelude: freq mode set without has_freq"); + } + return Status::OK(); +} + +// Running per-block byte offsets, chained across windows AND across super-blocks +// with the same lifetime as prev_last: dd/freq are prefix sums of the per-window +// on-disk region lengths, prx the prefix sum of prx lengths. They replace the +// three offset columns the row used to serialize (each was exactly this sum), so +// the reader reproduces dd_off/freq_off/prx_off bit-identically to the old +// explicit fields. +struct RunningOffsets { + uint64_t dd = 0; + uint64_t freq = 0; + uint64_t prx = 0; +}; + +// Decodes one window row, advancing prev_last to this window's absolute last and +// the running offsets past this window's dd/freq/prx regions. +Status decode_window_row(ByteSource* src, bool has_freq, bool has_prx, bool first_window, + uint64_t* prev_last, RunningOffsets* run, WindowMeta* m) { + uint64_t ldd = 0, doc_count = 0; + RETURN_IF_ERROR(src->get_varint64(&ldd)); + RETURN_IF_ERROR(src->get_varint64(&doc_count)); + uint8_t mode = 0; + RETURN_IF_ERROR(src->get_u8(&mode)); + RETURN_IF_ERROR(check_win_mode(mode, has_freq)); + m->dd_zstd = (mode & frq_win_mode::kDdZstd) != 0; + m->freq_zstd = has_freq && (mode & frq_win_mode::kFreqZstd) != 0; + + // dd region: read disk_len, derive dd_off from the running dd-block offset, + // then advance it (overflow-guarded). uncomp_len is stored only for a zstd + // region; a raw region's uncomp_len == disk_len by contract. + RETURN_IF_ERROR(src->get_varint64(&m->dd_disk_len)); + m->dd_off = run->dd; + RETURN_IF_ERROR(checked_add_u64(run->dd, m->dd_disk_len, + "frq_prelude: dd-block offset overflow", &run->dd)); + if (m->dd_zstd) { + RETURN_IF_ERROR(src->get_varint64(&m->dd_uncomp_len)); + } else { + m->dd_uncomp_len = m->dd_disk_len; + } + RETURN_IF_ERROR(src->get_fixed32(&m->crc_dd)); + if (has_freq) { + RETURN_IF_ERROR(src->get_varint64(&m->freq_disk_len)); + m->freq_off = run->freq; + RETURN_IF_ERROR(checked_add_u64(run->freq, m->freq_disk_len, + "frq_prelude: freq-block offset overflow", &run->freq)); + if (m->freq_zstd) { + RETURN_IF_ERROR(src->get_varint64(&m->freq_uncomp_len)); + } else { + m->freq_uncomp_len = m->freq_disk_len; + } + RETURN_IF_ERROR(src->get_fixed32(&m->crc_freq)); + } + if (has_prx) { + RETURN_IF_ERROR(src->get_varint64(&m->prx_len)); + m->prx_off = run->prx; + RETURN_IF_ERROR(checked_add_u64(run->prx, m->prx_len, "frq_prelude: prx offset overflow", + &run->prx)); + } + uint64_t max_freq = 0; + RETURN_IF_ERROR(src->get_varint64(&max_freq)); + RETURN_IF_ERROR(src->get_u8(&m->max_norm)); + uint64_t last_docid = 0; + RETURN_IF_ERROR(checked_add_u64(*prev_last, ldd, "frq_prelude: window last_docid overflow", + &last_docid)); + RETURN_IF_ERROR(validate_window_doc_count(first_window, *prev_last, last_docid, doc_count)); + m->win_base = *prev_last; + RETURN_IF_ERROR( + checked_u32(last_docid, "frq_prelude: window last_docid exceeds u32", &m->last_docid)); + RETURN_IF_ERROR( + checked_u32(doc_count, "frq_prelude: window doc_count exceeds u32", &m->doc_count)); + RETURN_IF_ERROR( + checked_u32(max_freq, "frq_prelude: window max_freq exceeds u32", &m->max_freq)); + *prev_last = last_docid; + return Status::OK(); +} + +// Decodes one super-block's window block (<=G rows) into the global window list, +// seeding win_base from prev_last and re-checking the recorded sb last docid. +Status decode_one_block(Slice block, const Header& h, uint64_t sb_last_docid, size_t row_count, + uint64_t* prev_last, RunningOffsets* run, + std::vector* windows) { + ByteSource src(block); + for (size_t i = 0; i < row_count; ++i) { + WindowMeta m; + RETURN_IF_ERROR(decode_window_row(&src, h.has_freq, h.has_prx, windows->empty(), prev_last, + run, &m)); + windows->push_back(m); + } + if (!src.eof()) { + return Status::Error( + "frq_prelude: window block has trailing bytes"); + } + if (*prev_last != sb_last_docid) { + return Status::Error( + "frq_prelude: window block last docid mismatch"); + } + return Status::OK(); +} + +// Decodes all window blocks pointed to by the super_block_dir. +Status decode_all_blocks(Slice window_region, const Header& h, const std::vector& dir, + std::vector* windows) { + windows->clear(); + windows->reserve(static_cast(h.n)); + uint64_t prev_last = 0; + // dd/freq/prx running offsets chain across ALL super-blocks (not reset per + // block), the same lifetime as prev_last, so the derived per-window offsets are + // continuous over the whole dd-block / freq-block / prx span. + RunningOffsets run; + for (size_t s = 0; s < dir.size(); ++s) { + const SbDirRow& r = dir[s]; + if (r.block_off + r.block_len > window_region.size() || + r.block_off + r.block_len < r.block_off) { + return Status::Error( + "frq_prelude: window block out of region"); + } + const uint64_t already = static_cast(windows->size()); + const uint64_t rows = std::min(h.group_size, h.n - already); + Slice block = window_region.subslice(static_cast(r.block_off), + static_cast(r.block_len)); + RETURN_IF_ERROR(decode_one_block(block, h, r.last_docid, static_cast(rows), + &prev_last, &run, windows)); + } + if (windows->size() != h.n) { + return Status::Error( + "frq_prelude: decoded window count mismatch"); + } + return Status::OK(); +} + +// Sums the per-window dd/freq on-disk lengths into the dd-block / freq-block +// lengths, guarding each running sum against u64 overflow. The dd_off/freq_off +// contiguity cross-checks the old prelude ran here are now tautological -- the +// reader DERIVES dd_off/freq_off as these very prefix sums (decode_window_row), +// so `m.dd_off == running-sum` holds by construction and is dropped. The +// length-overflow guards and the returned block lengths are retained: they still +// bound the dd-block/freq-block range the callers' InBounds checks fetch against, +// and mirror the same checked_add_u64 the offset derivation uses. +Status validate_region_layout(const Header& h, const std::vector& windows, + uint64_t* dd_block_len, uint64_t* freq_block_len) { + uint64_t dd_expect = 0; + uint64_t freq_expect = 0; + for (const WindowMeta& m : windows) { + // Raw regions carry uncomp_len == disk_len (derived, not stored); this + // guard stays as defensive documentation of that invariant. + if (m.dd_disk_len > m.dd_uncomp_len && !m.dd_zstd) { + return Status::Error( + "frq_prelude: raw dd region length inconsistent"); + } + if (dd_expect + m.dd_disk_len < dd_expect) { + return Status::Error( + "frq_prelude: dd block length overflow"); + } + dd_expect += m.dd_disk_len; + if (h.has_freq) { + if (freq_expect + m.freq_disk_len < freq_expect) { + return Status::Error( + "frq_prelude: freq block length overflow"); + } + freq_expect += m.freq_disk_len; + } + } + *dd_block_len = dd_expect; + *freq_block_len = freq_expect; + return Status::OK(); +} + +} // namespace + +namespace { +// TEST-ONLY seam backing testing::window_probe_count(): one increment per window +// last_docid comparison, in select_covering_windows_cursor and in locate_window's +// level-2 scan. thread_local => race-free under the shared const reader and free of +// atomic cost in the production cursor's hot loop; tests reset/read on their own thread. +thread_local uint64_t g_window_probes = 0; +inline void note_window_probe() { + ++g_window_probes; +} +} // namespace + +Status FrqPreludeReader::open(Slice prelude, FrqPreludeReader* out) { + ByteSource src(prelude); + Header h; + RETURN_IF_ERROR(parse_header(&src, &h)); + const size_t header_end = src.position(); + RETURN_IF_ERROR(verify_covered_crc(prelude, header_end, h.sbdir_len)); + + if (header_end + static_cast(h.sbdir_len) > prelude.size()) { + return Status::Error( + "frq_prelude: sbdir_len past buffer"); + } + Slice dir = prelude.subslice(header_end, static_cast(h.sbdir_len)); + std::vector rows; + uint64_t window_region_len = 0; + RETURN_IF_ERROR(decode_super_block_dir(dir, h, &rows, &window_region_len)); + + const size_t region_start = header_end + static_cast(h.sbdir_len) + sizeof(uint32_t); + if (region_start + static_cast(window_region_len) > prelude.size()) { + return Status::Error( + "frq_prelude: window region past buffer"); + } + Slice window_region = prelude.subslice(region_start, static_cast(window_region_len)); + + out->has_freq_ = h.has_freq; + out->has_prx_ = h.has_prx; + out->group_size_ = static_cast(h.group_size); + out->n_super_ = static_cast(h.n_super); + out->sb_last_docid_.clear(); + out->sb_last_docid_.reserve(rows.size()); + for (const SbDirRow& r : rows) out->sb_last_docid_.push_back(r.last_docid); + RETURN_IF_ERROR(decode_all_blocks(window_region, h, rows, &out->windows_)); + // Packed last_docid catalogue for the covering-window cursor (in-memory only; + // byte-identical to each windows_[w].last_docid, never serialized). + out->win_last_docid_.clear(); + out->win_last_docid_.reserve(out->windows_.size()); + for (const WindowMeta& m : out->windows_) { + out->win_last_docid_.push_back(m.last_docid); + } + return validate_region_layout(h, out->windows_, &out->dd_block_len_, &out->freq_block_len_); +} + +Status FrqPreludeReader::window(uint32_t w, WindowMeta* out) const { + if (out == nullptr) + return Status::Error("frq_prelude: null window out"); + if (w >= windows_.size()) { + return Status::Error( + "frq_prelude: window index out of range"); + } + *out = windows_[w]; + return Status::OK(); +} + +Status FrqPreludeReader::locate_window(uint32_t docid, bool* found, uint32_t* w) const { + if (found == nullptr || w == nullptr) { + return Status::Error("frq_prelude: null locate out"); + } + *found = false; + if (windows_.empty()) return Status::OK(); + if (docid > windows_.back().last_docid) return Status::OK(); + + // Level 1: first super-block whose absolute last docid >= docid. + const auto sb_it = std::lower_bound(sb_last_docid_.begin(), sb_last_docid_.end(), + static_cast(docid)); + const size_t sb = static_cast(sb_it - sb_last_docid_.begin()); + // Level 2: window binary search within [sb*G, min((sb+1)*G, N)). + const size_t lo = sb * group_size_; + const size_t hi = std::min(lo + group_size_, windows_.size()); + for (size_t i = lo; i < hi; ++i) { + note_window_probe(); + if (docid <= windows_[i].last_docid) { + *found = true; + *w = static_cast(i); + return Status::OK(); + } + } + return Status::OK(); // unreachable when invariants hold; defensive miss. +} + +void select_covering_windows_cursor(const uint32_t* win_last_docid, uint32_t n_windows, + const uint64_t* sb_last_docid, uint32_t n_super, + uint32_t group_size, const std::vector& candidates, + std::vector* windows) { + windows->clear(); + if (n_windows == 0) { + return; // empty-windows guard (mirrors locate_window's windows_.empty() early-out). + } + uint32_t sb = 0; // monotonic super-block cursor + uint32_t w = 0; // monotonic window cursor + uint32_t last_emitted = UINT32_MAX; // last window pushed (run-collapse dedup) + for (uint32_t d : candidates) { + const uint64_t target = d; // widen once; keeps the comparisons template-bracket free + // Level 1: first super-block whose absolute last docid >= d. sb only advances + // forward, so across the whole call it steps at most n_super times. + while (sb < n_super && sb_last_docid[sb] < target) { + ++sb; + } + if (sb == n_super) { + break; // d past the term's last docid -> this and every later candidate miss. + } + // Boundary jump: never scan windows below the current super-block's first window. + if (w < sb * group_size) { + w = sb * group_size; + } + // Level 2: first window whose absolute last docid >= d. w only advances forward, + // so across the whole call the comparisons below total at most n_windows misses + // plus one hit per candidate => probe_count <= candidates + n_windows. + while (w < n_windows) { + note_window_probe(); + if (d <= win_last_docid[w]) { + break; + } + ++w; + } + if (w == n_windows) { + break; // defensive: invariants guarantee a hit once sb < n_super. + } + if (w != last_emitted) { + windows->push_back(w); + last_emitted = w; + } + } +} + +void FrqPreludeReader::select_covering_windows(const std::vector& candidates, + std::vector* windows) const { + select_covering_windows_cursor( + win_last_docid_.data(), static_cast(win_last_docid_.size()), + sb_last_docid_.data(), static_cast(sb_last_docid_.size()), group_size_, + candidates, windows); +} + +} // namespace doris::snii::format + +namespace doris::snii::format::testing { + +uint64_t window_probe_count() { + return g_window_probes; +} + +void reset_window_probe_count() { + g_window_probes = 0; +} + +} // namespace doris::snii::format::testing diff --git a/be/src/storage/index/snii/format/frq_prelude.h b/be/src/storage/index/snii/format/frq_prelude.h new file mode 100644 index 00000000000000..642f3aed96b71c --- /dev/null +++ b/be/src/storage/index/snii/format/frq_prelude.h @@ -0,0 +1,256 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +// FrqPrelude: a TWO-LEVEL (super-block -> window) skippable directory that +// precedes a windowed .frq posting whose payload is laid out (PHASE D, design +// 1.6) with dd and freq regions GROUPED at posting level: +// windowed .frq payload = [prelude][dd-block][freq-block] +// dd-block = dd_region_0 ++ dd_region_1 ++ ... ++ dd_region_{N-1} +// freq-block = freq_region_0 ++ ... ++ freq_region_{N-1} (iff has_freq) +// Windows are NOT self-describing: each window's full codec metadata (region +// offsets, on-disk/uncompressed lengths, modes, crcs) lives in the prelude rows. +// The docs-only prefix [prelude][dd-block] is therefore ONE contiguous run a +// docid-only / phrase reader fetches in a single range, skipping the freq-block. +// +// DictEntry records prelude_len, frq_len (whole payload) and frq_docs_len +// (= prelude_len + dd_block_len) so a reader can range-fetch the prelude first, +// then fetch either the contiguous dd-block (docs-only) or both blocks (scoring). +// +// On-disk layout (strict; all multi-byte fixed fields little-endian, VInt = +// LEB128 via snii/encoding): +// header: +// u8 flags # bit0 has_freq, bit1 has_prx +// VInt N # number of .frq windows +// VInt G # windows per super-block (group_size; >=1) +// VInt n_super # = ceil(N / G); 0 when N==0 +// VInt sbdir_len # byte length of the super_block_dir region +// u32 crc32c # covers header + super_block_dir (NOT the window blocks) +// super_block_dir[n_super]: # small, resident: one row per super-block +// VInt sb_last_docid_delta # cumulative across super-blocks => absolute last +// # docid of the super-block's last window +// VInt sb_block_off # byte offset of this super-block's window block, +// # measured from the start of the window_dir region +// VInt sb_block_len # byte length of this super-block's window block +// window_dir: n_super self-contained blocks, each holding <=G window rows. +// per window row (T18 slim layout -- dd_off/freq_off/prx_off are NOT stored; +// the reader derives them as running prefix sums of the disk/prx lengths): +// VInt last_docid_delta # cumulative WITHIN the block => absolute last docid +// # (previous window's absolute last docid = win_base; +// # first window of first block: win_base = 0) +// VInt doc_count # number of docs in the window (frq_pod needs it) +// u8 win_mode # bit0 dd_zstd, bit1 freq_zstd +// VInt dd_disk_len # dd_region on-disk byte length +// [VInt dd_uncomp_len] # dd_region plaintext length; present ONLY when +// # win_mode & kDdZstd. A raw region's uncomp_len +// # == dd_disk_len (derived, not stored). +// u32 crc_dd # crc32c of the dd_region on-disk bytes +// VInt freq_disk_len # freq_region on-disk byte length (has_freq) +// [VInt freq_uncomp_len] # freq_region plaintext length; present ONLY when +// # has_freq && win_mode & kFreqZstd (raw: derived +// # == freq_disk_len). +// u32 crc_freq # crc32c of the freq_region on-disk bytes (has_freq) +// VInt prx_len # .prx payload byte length (present iff has_prx) +// VInt max_freq # window max term frequency (WAND block-max) +// u8 max_norm # window score-max norm (WAND); 0 acceptable +// +// The reader reconstructs each window's dd_off / freq_off (byte offset within the +// dd-block / freq-block) and prx_off (offset within the entry's .prx span) as the +// running prefix sums of dd_disk_len / freq_disk_len / prx_len over all windows, +// chained across super-blocks; WindowMeta still exposes those offsets, now derived. +// +// Reconstructing win_base / absolute last_docid (READER CONTRACT) is unchanged: +// the writer chains absolute last docids across windows; each row stores the delta +// of its absolute last docid from the previous window, and sb_last_docid seeds +// each block, so super-block binary search then in-block window binary search +// locate the window covering any docid without decoding the .frq blocks. +// +// The trailing crc32c covers only header + super_block_dir; every region carries +// its own crc (crc_dd / crc_freq) in the row. +namespace doris::snii::format { + +namespace frq_prelude_flags { +inline constexpr uint8_t kHasFreq = 1u << 0; +inline constexpr uint8_t kHasPrx = 1u << 1; +// Reserved extension point (T18): kSlimRows = 1u << 2 would gate the trimmed +// window-row layout (no stored dd_off/freq_off/prx_off, conditional uncomp_len) +// as a distinct on-disk path. It is NOT emitted today: the trim folds into the +// single pre-launch v1 encoding (writer/reader symmetric, no dual decode path). +// If a `lifecycle: launched` index appears before this lands, set this bit on the +// slim writer and branch the reader on it instead of unconditionally decoding slim. +} // namespace frq_prelude_flags + +// Per-window codec mode bits (win_mode byte). +namespace frq_win_mode { +inline constexpr uint8_t kDdZstd = 1u << 0; +inline constexpr uint8_t kFreqZstd = 1u << 1; +inline constexpr uint8_t kKnownBits = kDdZstd | kFreqZstd; +} // namespace frq_win_mode + +// Absolute, decoded metadata for one window (as the reader exposes it). The dd / +// freq region locators are offsets WITHIN the dd-block / freq-block respectively +// (both blocks follow the prelude). dd_off/freq_off/prx_off are DERIVED by the +// reader as running prefix sums of the disk/prx lengths (they are no longer stored +// per row; see the header layout note) -- these public members are unchanged and +// still populated, just by derivation. The reader derives the dd-block length from +// the last window's dd_off + dd_disk_len. +struct WindowMeta { + uint32_t last_docid = 0; // absolute last docid in the window + uint64_t win_base = 0; // absolute last docid of the previous window (0 for w==0) + uint32_t doc_count = 0; + + // dd_region locator (within the dd-block). + bool dd_zstd = false; + uint64_t dd_off = 0; // DERIVED: running sum of prior windows' dd_disk_len + uint64_t dd_disk_len = 0; + uint64_t dd_uncomp_len = 0; // DERIVED == dd_disk_len for raw; stored only when dd_zstd + uint32_t crc_dd = 0; + + // freq_region locator (within the freq-block); valid only when has_freq. + bool freq_zstd = false; + uint64_t freq_off = 0; // DERIVED: running sum of prior windows' freq_disk_len + uint64_t freq_disk_len = 0; + uint64_t freq_uncomp_len = 0; // DERIVED == freq_disk_len for raw; stored only when freq_zstd + uint32_t crc_freq = 0; + + uint64_t prx_off = 0; // valid only when has_prx; DERIVED: running sum of prior prx_len + uint64_t prx_len = 0; // valid only when has_prx + uint32_t max_freq = 0; + uint8_t max_norm = 0; + + // In-memory only (NOT serialized in the prelude row). When false, the dd/freq + // region decode skips crc verification -- used when these region bytes are + // covered by an enclosing crc (e.g. an INLINE entry inside its dict block). + // Windowed/slim POD-ref rows leave this true (their regions carry a crc). + bool verify_crc = true; +}; + +// Builder input: one fully-computed WindowMeta per window, in term order, plus the +// super-block grouping factor. The writer fills last_docid (absolute), doc_count, +// the region locators/crcs, prx locator, max_freq and max_norm; win_base is derived +// during build (so callers may leave it 0). group_size must be >= 1. +struct FrqPreludeColumns { + bool has_freq = true; + bool has_prx = false; + uint32_t group_size = 64; // windows per super-block (G) + std::vector windows; +}; + +// Builds the prelude bytes and appends them to out. +// Returns InvalidArgument when out is null, group_size is 0, or the windows are +// not in non-decreasing last_docid order (a window's absolute last docid must be +// >= the previous window's). +Status build_frq_prelude(const FrqPreludeColumns& cols, ByteSink* out); + +// Reads and verifies a prelude buffer, exposing two-level skip access. The reader +// parses the header + super_block_dir on open (verifying the trailing crc) and +// eagerly decodes every window block into owned WindowMeta rows (the prelude is +// small relative to the postings). It does not retain the input. +class FrqPreludeReader { +public: + // Parses + verifies the prelude. crc mismatch / truncation / inconsistent + // offsets-or-lengths / oversized counts => kCorruption. + static Status open(Slice prelude, FrqPreludeReader* out); + + uint32_t n_windows() const { return static_cast(windows_.size()); } + uint32_t n_super_blocks() const { return n_super_; } + bool has_freq() const { return has_freq_; } + bool has_prx() const { return has_prx_; } + + // Total on-disk byte length of the dd-block (== sum of dd_disk_len; the docs-only + // prefix after the prelude). 0 when there are no windows. + uint64_t dd_block_len() const { return dd_block_len_; } + // Total on-disk byte length of the freq-block (== sum of freq_disk_len). 0 when + // !has_freq or no windows. + uint64_t freq_block_len() const { return freq_block_len_; } + + // Returns the absolute WindowMeta for window w. Out-of-range => InvalidArgument. + Status window(uint32_t w, WindowMeta* out) const; + + // Locates the window covering docid via super-block binary search then window + // binary search. *found=false (with OK) when docid is past the term's last + // docid; otherwise *w is the index of the covering window (the first window + // whose absolute last_docid >= docid). + Status locate_window(uint32_t docid, bool* found, uint32_t* w) const; + + // Selects, as a monotonic two-pointer cursor, the ascending de-duplicated set of + // windows covering the ascending `candidates` (each window covering its + // (win_base, last_docid] span). Writes them to *windows (cleared first). The + // result is element-for-element identical to calling locate_window per candidate + // and collapsing equal runs, but uses O(C + N) window last_docid comparisons + // (C = candidates, N = windows) instead of O(C * group_size). Pure in-memory over + // the decoded directory; never fails. + void select_covering_windows(const std::vector& candidates, + std::vector* windows) const; + + // Packed absolute last_docid of window w (byte-identical to window(w).last_docid), + // exposed for the covering-window cursor's contiguous scan and equivalence tests. + uint32_t window_last_docid(uint32_t w) const { + DCHECK_LT(w, win_last_docid_.size()); + return win_last_docid_[w]; + } + +private: + bool has_freq_ = false; + bool has_prx_ = false; + uint32_t group_size_ = 1; + uint32_t n_super_ = 0; + uint64_t dd_block_len_ = 0; + uint64_t freq_block_len_ = 0; + // Absolute last docid at each super-block boundary (size n_super_). + std::vector sb_last_docid_; + // All windows decoded with absolute fields, in term order (size N). + std::vector windows_; + // Packed copy of each window's absolute last_docid (size N; win_last_docid_[w] == + // windows_[w].last_docid). Built in open() so the covering-window cursor scans a + // contiguous 4B/window array rather than the ~104B WindowMeta rows. In-memory only: + // never serialized; immutable after open() (same lifetime as windows_). + std::vector win_last_docid_; +}; + +// Pure cursor core (no FrqPreludeReader / IO): selects into *windows the ascending, +// de-duplicated indices of the windows covering the ascending `candidates`, given the +// packed window last_docid array (size n_windows), the super-block last_docid directory +// (size n_super) and group_size. A super-block cursor does boundary jumps while a window +// cursor advances forward only => O(C + N) window comparisons, element-for-element equal +// to per-candidate locate_window + run collapse. *windows is cleared first; n_windows == 0 +// yields an empty result. Exposed for isolated equivalence / complexity tests. +void select_covering_windows_cursor(const uint32_t* win_last_docid, uint32_t n_windows, + const uint64_t* sb_last_docid, uint32_t n_super, + uint32_t group_size, const std::vector& candidates, + std::vector* windows); + +// TEST-ONLY observability seam (mirrors the format dict-block decode counter). Counts the +// window last_docid comparisons performed by select_covering_windows_cursor and by +// locate_window's level-2 scan, so tests can assert the cursor stays O(C + N) and bounded +// by C + N regardless of group_size, while the legacy per-candidate scan grows with G. The +// counter is thread-local: race-free under the shared const reader and free of atomic cost +// in the production cursor loop; reset and read on the thread that ran the cursor. +namespace testing { +uint64_t window_probe_count(); +void reset_window_probe_count(); +} // namespace testing + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/logical_index_directory.cpp b/be/src/storage/index/snii/format/logical_index_directory.cpp new file mode 100644 index 00000000000000..77842a0bbc69eb --- /dev/null +++ b/be/src/storage/index/snii/format/logical_index_directory.cpp @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/logical_index_directory.h" + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/section_framer.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::format { + +namespace { + +// Minimum payload bytes any entry can occupy: index_id (>=1) + suffix_len (>=1, value 0) + +// meta_off (>=1) + meta_len (>=1). Used as an anti-DoS lower bound before reserving. +constexpr size_t kMinEntryBytes = 4; + +// Encode one directory entry. Fixed field order; reuse ByteSink varint/bytes primitives. +void encode_entry(const LogicalIndexRef& ref, ByteSink* payload) { + payload->put_varint64(ref.index_id); + payload->put_varint32(static_cast(ref.index_suffix.size())); + payload->put_bytes(Slice(std::string_view(ref.index_suffix))); + payload->put_varint64(ref.meta_off); + payload->put_varint64(ref.meta_len); +} + +// Decode one directory entry, validating suffix_len against the remaining payload before copying. +Status decode_entry(ByteSource* ps, LogicalIndexRef* ref) { + RETURN_IF_ERROR(ps->get_varint64(&ref->index_id)); + uint32_t suffix_len = 0; + RETURN_IF_ERROR(ps->get_varint32(&suffix_len)); + // Anti-DoS: reject a suffix_len that cannot fit in the remaining bytes before allocating. + if (suffix_len > ps->remaining()) { + return Status::Error( + "logical_index_directory: suffix_len exceeds payload"); + } + Slice suffix; + RETURN_IF_ERROR(ps->get_bytes(suffix_len, &suffix)); + ref->index_suffix.assign(reinterpret_cast(suffix.data()), suffix.size()); + RETURN_IF_ERROR(ps->get_varint64(&ref->meta_off)); + RETURN_IF_ERROR(ps->get_varint64(&ref->meta_len)); + return Status::OK(); +} + +Status decode_payload(Slice payload, std::vector* refs) { + ByteSource ps(payload); + uint32_t n_entries = 0; + RETURN_IF_ERROR(ps.get_varint32(&n_entries)); + // Anti-DoS: cap n_entries against the remaining payload before reserving, so a corrupted + // inflated count cannot trigger a huge allocation. + if (n_entries > ps.remaining() / kMinEntryBytes) { + return Status::Error( + "logical_index_directory: n_entries exceeds payload capacity"); + } + refs->clear(); + refs->reserve(n_entries); + for (uint32_t i = 0; i < n_entries; ++i) { + LogicalIndexRef ref {}; + RETURN_IF_ERROR(decode_entry(&ps, &ref)); + refs->push_back(std::move(ref)); + } + if (!ps.eof()) { + return Status::Error( + "logical_index_directory: trailing bytes in payload"); + } + return Status::OK(); +} + +} // namespace + +void LogicalIndexDirectoryBuilder::finish(ByteSink* sink) const { + ByteSink payload; + payload.put_varint32(static_cast(refs_.size())); + for (const auto& ref : refs_) { + encode_entry(ref, &payload); + } + SectionFramer::write(*sink, static_cast(SectionType::kLogicalIndexDirectory), + payload.view()); +} + +Status LogicalIndexDirectoryReader::open(Slice framed, LogicalIndexDirectoryReader* out) { + if (out == nullptr) { + return Status::Error( + "logical_index_directory: out is null"); + } + ByteSource src(framed); + FramedSection sec; + RETURN_IF_ERROR(SectionFramer::read(src, &sec)); + if (sec.type != static_cast(SectionType::kLogicalIndexDirectory)) { + return Status::Error( + "logical_index_directory: unexpected section type"); + } + return decode_payload(sec.payload, &out->refs_); +} + +Status LogicalIndexDirectoryReader::get(uint32_t i, LogicalIndexRef* out) const { + if (out == nullptr) { + return Status::Error( + "logical_index_directory: out is null"); + } + if (i >= refs_.size()) { + return Status::Error( + "logical_index_directory: index out of range"); + } + *out = refs_[i]; + return Status::OK(); +} + +Status LogicalIndexDirectoryReader::find(uint64_t index_id, std::string_view suffix, bool* found, + LogicalIndexRef* out) const { + if (found == nullptr || out == nullptr) { + return Status::Error( + "logical_index_directory: output pointer is null"); + } + *found = false; + for (const auto& ref : refs_) { + if (ref.index_id != index_id || std::string_view(ref.index_suffix) != suffix) { + continue; + } + *out = ref; + *found = true; + return Status::OK(); + } + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/logical_index_directory.h b/be/src/storage/index/snii/format/logical_index_directory.h new file mode 100644 index 00000000000000..f65143244b0d93 --- /dev/null +++ b/be/src/storage/index/snii/format/logical_index_directory.h @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +namespace doris::snii::format { + +// Container-level directory entry: maps a logical index identity (index_id, index_suffix) +// to the physical location of its per-index meta block. Aligned with Doris key system +// (see design spec "footer meta region" logical index directory). The reader issues a +// single range read over [meta_off, meta_off + meta_len) to load that per-index meta. +struct LogicalIndexRef { + uint64_t index_id = 0; // logical index id (matches Doris InvertedIndexDescriptor key) + std::string index_suffix; // UTF-8 sub-index suffix; may be empty for the primary index + uint64_t meta_off = 0; // absolute byte offset of the per-index meta block in the container + uint64_t meta_len = 0; // byte length of the per-index meta block +}; + +// Logical index directory: (index_id, index_suffix) -> per-index meta block reference. +// +// on-disk layout (framed by SectionFramer with a unified type+len+crc32c wrapper): +// [u8 type=kLogicalIndexDirectory][varint64 payload_len][payload][fixed32 crc32c] +// payload = varint32 n_entries +// then n_entries x { +// varint64 index_id, +// varint32 suffix_len, suffix_bytes, +// varint64 per_index_meta_off, +// varint64 per_index_meta_len } +// The section-level crc covers the whole directory, so no per-entry crc is stored +// (the spec lists a per-entry crc32c as optional; it is folded into the framer crc here). +class LogicalIndexDirectoryBuilder { +public: + void add(const LogicalIndexRef& ref) { refs_.push_back(ref); } + + // Encodes as a kLogicalIndexDirectory framed section (with embedded crc32c) and appends to sink. + void finish(ByteSink* sink) const; + +private: + std::vector refs_; +}; + +// Reads and verifies a kLogicalIndexDirectory framed section; provides ordinal access and +// (index_id, suffix) lookup. After parsing, all entries reside in the reader (entering the +// searcher cache along with the rest of the tail meta region). +class LogicalIndexDirectoryReader { +public: + // Verifies the section crc and deserializes all entries. + // crc mismatch / truncation / trailing bytes / oversized counts -> kCorruption; + // wrong section type -> kInvalidArgument; null out -> kInvalidArgument. + static Status open(Slice framed, LogicalIndexDirectoryReader* out); + + uint32_t size() const { return static_cast(refs_.size()); } + + // Returns the i-th entry in encounter order; i >= size -> kNotFound. + Status get(uint32_t i, LogicalIndexRef* out) const; + + // Looks up the entry for (index_id, suffix). On match, *found=true and *out is populated; + // when absent, *found=false and *out is left untouched. Returns kInvalidArgument on null + // output pointers. The pair (index_id, suffix) is the unique key. + Status find(uint64_t index_id, std::string_view suffix, bool* found, + LogicalIndexRef* out) const; + +private: + std::vector refs_; +}; + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/norms_pod.cpp b/be/src/storage/index/snii/format/norms_pod.cpp new file mode 100644 index 00000000000000..77d7558903ba72 --- /dev/null +++ b/be/src/storage/index/snii/format/norms_pod.cpp @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/norms_pod.h" + +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/section_framer.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::format { + +void NormsPodWriter::finish(ByteSink* sink) const { + // Build inner payload: [varint64 doc_count][raw norm bytes]. + ByteSink payload; + payload.put_varint64(norms_.size()); + payload.put_bytes(Slice(norms_)); + // Delegate outer framing to SectionFramer to append type+len+crc32c, avoiding manual checksum assembly. + SectionFramer::write(*sink, static_cast(SectionType::kStatsBlock), payload.view()); +} + +Status NormsPodReader::open(Slice framed, NormsPodReader* out) { + // framer handles CRC verify, truncation detection, and payload slicing. + ByteSource src(framed); + FramedSection sec; + RETURN_IF_ERROR(SectionFramer::read(src, &sec)); + + // Parse inner payload: [varint64 doc_count][bytes]. + ByteSource payload(sec.payload); + uint64_t doc_count = 0; + RETURN_IF_ERROR(payload.get_varint64(&doc_count)); + if (doc_count > std::numeric_limits::max()) { + return Status::Error( + "norms POD doc_count overflows uint32"); + } + // doc_count must exactly equal the remaining byte count (1 byte per doc). + if (payload.remaining() != doc_count) { + return Status::Error( + "norms POD length mismatch"); + } + + Slice bytes; + RETURN_IF_ERROR(payload.get_bytes(static_cast(doc_count), &bytes)); + out->doc_count_ = static_cast(doc_count); + out->norms_ = bytes.data(); + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/norms_pod.h b/be/src/storage/index/snii/format/norms_pod.h new file mode 100644 index 00000000000000..5fd7cd92d16f24 --- /dev/null +++ b/be/src/storage/index/snii/format/norms_pod.h @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +namespace doris::snii::format { + +// norms POD: per logical index / field stores 1-byte encoded doc length per doc, +// used by BM25 length normalization (SniiStatsProvider::encoded_norm) for per-docid lookup. +// +// On-disk layout (the whole section is framed by SectionFramer, which adds a type+len+crc32c envelope): +// framer payload = [varint64 doc_count][bytes encoded_norm[doc_count]] +// framer envelope = [u8 type][varint64 payload_len][payload][fixed32 crc32c] +// The encoding of encoded_norm (length -> 1B) is out of scope for this module; here we only handle raw byte storage and retrieval. +class NormsPodWriter { +public: + // Appends the encoded_norm for the next docid (docid is implicit, assigned in append order starting from 0). + void add(uint8_t encoded_norm) { norms_.push_back(encoded_norm); } + + // Number of docs accumulated so far (i.e., the next docid to be assigned). + size_t count() const { return norms_.size(); } + + // Writes [doc_count][bytes] framed by SectionFramer into sink (appends; does not clear sink). + void finish(ByteSink* sink) const; + +private: + std::vector norms_; +}; + +// Read-only view: on open, verifies the framer CRC and checks that doc_count/payload length are consistent, +// afterwards encoded_norm(docid) is O(1) direct indexing (zero-copy, borrows the underlying buffer). +class NormsPodReader { +public: + NormsPodReader() = default; + + // Parses the entire section (including the framer envelope). Returns Corruption on CRC mismatch, truncation, or length inconsistency. + // On success, *out borrows the memory pointed to by framer_payload; the caller must ensure its lifetime. + static Status open(Slice framed, NormsPodReader* out); + + uint32_t doc_count() const { return doc_count_; } + + // Precondition (hard contract): docid < doc_count(). Semantics match std::vector::operator[]: + // the caller is responsible for guaranteeing this (docid comes from trusted postings decoded internally by SNII). Asserts in debug builds; + // no check in Release (NDEBUG). Use try_encoded_norm when the docid is untrusted and needs validation. + uint8_t encoded_norm(uint32_t docid) const { + assert(docid < doc_count_); + return norms_[docid]; + } + + // Checked access: returns InvalidArgument if docid is out of range; never reads out-of-range memory. + Status try_encoded_norm(uint32_t docid, uint8_t* out) const { + if (docid >= doc_count_) + return Status::Error("norms: docid out of range"); + *out = norms_[docid]; + return Status::OK(); + } + +private: + const uint8_t* norms_ = nullptr; + uint32_t doc_count_ = 0; +}; + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/null_bitmap.cpp b/be/src/storage/index/snii/format/null_bitmap.cpp new file mode 100644 index 00000000000000..427bceb3f6bdd6 --- /dev/null +++ b/be/src/storage/index/snii/format/null_bitmap.cpp @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/null_bitmap.h" + +#include +#include + +#include "roaring/roaring.h" +#include "roaring/roaring.hh" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/section_framer.h" + +namespace doris::snii::format { + +NullBitmapWriter:: + NullBitmapWriter() // NOLINT(modernize-use-equals-default): roaring type is incomplete in the header. + : bitmap_(std::make_unique()) {} + +NullBitmapWriter::~NullBitmapWriter() = default; + +void NullBitmapWriter::add_null(uint32_t docid) { + bitmap_->add(docid); +} + +uint32_t NullBitmapWriter::null_count() const { + return static_cast(bitmap_->cardinality()); +} + +void NullBitmapWriter::finish(uint32_t doc_count, ByteSink* sink) const { + // Serialize the Roaring bitmap to its portable on-disk form. + const size_t roaring_size = bitmap_->getSizeInBytes(); + std::vector roaring_buf(roaring_size); + bitmap_->write(roaring_buf.data()); + + // Build inner payload: [varint64 doc_count][varint64 roaring_size][bytes]. + ByteSink payload; + payload.put_varint64(doc_count); + payload.put_varint64(roaring_size); + payload.put_bytes(Slice(reinterpret_cast(roaring_buf.data()), roaring_size)); + + // Delegate the type + len + crc32c envelope to SectionFramer. + SectionFramer::write(*sink, kNullBitmapSectionType, payload.view()); +} + +NullBitmapReader:: + NullBitmapReader() // NOLINT(modernize-use-equals-default): roaring type is incomplete in the header. + : bitmap_(std::make_unique()) {} + +NullBitmapReader::~NullBitmapReader() = default; + +NullBitmapReader::NullBitmapReader(NullBitmapReader&&) noexcept = default; +NullBitmapReader& NullBitmapReader::operator=(NullBitmapReader&&) noexcept = default; + +Status NullBitmapReader::open(Slice framed, NullBitmapReader* out) { + // SectionFramer handles CRC verification, truncation detection, and payload + // slicing. + ByteSource src(framed); + FramedSection sec; + RETURN_IF_ERROR(SectionFramer::read(src, &sec)); + + // Parse inner payload: [varint64 doc_count][varint64 roaring_size][bytes]. + ByteSource payload(sec.payload); + uint64_t doc_count = 0; + RETURN_IF_ERROR(payload.get_varint64(&doc_count)); + if (doc_count > std::numeric_limits::max()) { + return Status::Error( + "null bitmap doc_count overflows uint32"); + } + + uint64_t roaring_size = 0; + RETURN_IF_ERROR(payload.get_varint64(&roaring_size)); + // Anti-DoS: the declared roaring_size must not exceed the bytes actually + // present, otherwise readSafe could be told to walk past the payload. + if (roaring_size > payload.remaining()) { + return Status::Error( + "null bitmap roaring_size exceeds payload"); + } + + Slice roaring_bytes; + RETURN_IF_ERROR(payload.get_bytes(static_cast(roaring_size), &roaring_bytes)); + + // Validate the Roaring container BEFORE deserializing. A CRC-valid frame can + // still carry malformed roaring bytes; Roaring::readSafe / read would then hit + // CRoaring's terminate-or-throw path (NULL -> ROARING_TERMINATE). The safe, + // non-throwing C probe returns the exact byte count a valid container would + // consume, or 0 on malformed/insufficient input. + const char* rb = reinterpret_cast(roaring_bytes.data()); + const size_t probed = + roaring_bitmap_portable_deserialize_size(rb, static_cast(roaring_size)); + if (probed == 0 || probed != static_cast(roaring_size)) { + return Status::Error( + "null bitmap: malformed roaring container"); + } + *out->bitmap_ = roaring::Roaring::readSafe(rb, static_cast(roaring_size)); + out->doc_count_ = static_cast(doc_count); + return Status::OK(); +} + +bool NullBitmapReader::is_null(uint32_t docid) const { + return bitmap_->contains(docid); +} + +uint32_t NullBitmapReader::null_count() const { + return static_cast(bitmap_->cardinality()); +} + +void NullBitmapReader::copy_to(roaring::Roaring* out) const { + *out = *bitmap_; +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/null_bitmap.h b/be/src/storage/index/snii/format/null_bitmap.h new file mode 100644 index 00000000000000..70af063ecdcd71 --- /dev/null +++ b/be/src/storage/index/snii/format/null_bitmap.h @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +// Forward-declare the CRoaring C++ bitmap so this header stays free of the +// (large) roaring include; the concrete type is only needed in the .cpp. +namespace roaring { +class Roaring; +} // namespace roaring + +namespace doris::snii::format { + +// SectionFramer type byte for the null-bitmap POD. There is no dedicated +// SectionType enum value yet, so we use a documented literal (0x20) outside the +// currently allocated enum range (1..9) to avoid colliding with existing types. +inline constexpr uint8_t kNullBitmapSectionType = 0x20; + +// NullBitmap POD: per logical index, a Roaring bitmap of null docids (docs whose +// value is NULL / not indexed). It decouples per-doc NULL information from the +// per-term dictionary / postings so NULL handling can pull only this side POD. +// +// On-disk layout (the whole section is framed by SectionFramer, which adds a +// type + varint64 len + payload + fixed32 crc32c envelope): +// framer payload = [varint64 doc_count][varint64 roaring_size][roaring_bytes] +// roaring_bytes is the portable CRoaring serialization (Roaring::write). +class NullBitmapWriter { +public: + NullBitmapWriter(); + ~NullBitmapWriter(); + + NullBitmapWriter(const NullBitmapWriter&) = delete; + NullBitmapWriter& operator=(const NullBitmapWriter&) = delete; + + // Marks docid as NULL (adding the same docid twice is idempotent). + void add_null(uint32_t docid); + + // Number of distinct null docids accumulated so far. + uint32_t null_count() const; + + // Serializes [doc_count][roaring_size][roaring_bytes] framed by SectionFramer + // and appends it to sink (does not clear sink). doc_count is the total number + // of docs in the logical index (recorded so the reader can round-trip it). + void finish(uint32_t doc_count, ByteSink* sink) const; + +private: + std::unique_ptr bitmap_; +}; + +// Read-only view: on open, SectionFramer verifies the CRC and truncation; this +// class then guards roaring_size against the remaining payload bytes before +// deserializing the Roaring bitmap (anti-DoS), so a corrupt size cannot trigger +// an oversized allocation/read. is_null() is then an O(1) membership test. +class NullBitmapReader { +public: + NullBitmapReader(); + ~NullBitmapReader(); + + NullBitmapReader(const NullBitmapReader&) = delete; + NullBitmapReader& operator=(const NullBitmapReader&) = delete; + NullBitmapReader(NullBitmapReader&&) noexcept; + NullBitmapReader& operator=(NullBitmapReader&&) noexcept; + + // Parses the entire section (framer envelope + payload). Returns Corruption on + // CRC mismatch, truncation, doc_count overflow, or an oversized roaring_size. + static Status open(Slice framed, NullBitmapReader* out); + + // True iff docid was marked NULL. docids outside the null set (including those + // >= doc_count) return false. + bool is_null(uint32_t docid) const; + + // Number of distinct null docids in the bitmap. + uint32_t null_count() const; + + // Copies the decoded bitmap into the caller-owned Roaring object. + void copy_to(roaring::Roaring* out) const; + + // Total doc count of the logical index, as recorded by the writer. + uint32_t doc_count() const { return doc_count_; } + +private: + std::unique_ptr bitmap_; + uint32_t doc_count_ = 0; +}; + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/per_index_meta.cpp b/be/src/storage/index/snii/format/per_index_meta.cpp new file mode 100644 index 00000000000000..2fd8baef1fa3eb --- /dev/null +++ b/be/src/storage/index/snii/format/per_index_meta.cpp @@ -0,0 +1,352 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/per_index_meta.h" + +#include + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/encoding/section_framer.h" +#include "storage/index/snii/encoding/zstd_codec.h" + +namespace doris::snii::format { + +namespace { + +// Upper bound on index_suffix length read from untrusted bytes, capped before +// allocation to avoid a DoS-inflated reserve. A logical index suffix is a short +// column/field name; 64 KiB is far beyond any legitimate value. +constexpr uint32_t kMaxSuffixLen = 64u * 1024u; + +// G13 zstd-compressed embedded sub-sections. Level matches the other SNII zstd +// producers (dict blocks / .prx windows); these sorted tables compress several +// fold at level 3 already. +constexpr int kMetaSectionZstdLevel = 3; +// Upper bound on a declared uncomp_len read from untrusted bytes, capped before +// the decompress allocation (same bound as dict blocks). A meta sub-section is a +// per-segment sampled-term/offset table -- far below this in practice. +constexpr uint64_t kMaxMetaSectionUncompBytes = 256ULL * 1024 * 1024; + +// Raw-frame size at/above which finish() emits the zstd carrier. The env +// SNII_META_COMPRESS_MIN overrides the default for tuning and for building +// uncompressed control segments in tests without a huge corpus. Read per finish. +size_t meta_compress_min_bytes() { + const char* s = std::getenv("SNII_META_COMPRESS_MIN"); + if (s != nullptr) { + char* end = nullptr; + const unsigned long long v = std::strtoull(s, &end, 10); + if (end != s) { + return v; + } + } + return kMetaSectionCompressMinBytes; +} + +// Emits one embedded sub-section: the raw already-framed bytes verbatim (legacy +// layout), or -- when the raw frame reaches the compress threshold AND zstd +// actually shrinks it -- a `zstd_type` carrier frame whose payload is +// varint64 uncomp_len + zstd(raw frame). An empty `framed` (absent section) +// emits nothing, exactly as before. zstd failure falls back to the raw layout: +// compression is an encoding optimization, never a correctness dependency. +void emit_embedded_section(const std::vector& framed, SectionType zstd_type, + ByteSink* sink) { + if (framed.empty()) { + return; + } + if (framed.size() >= meta_compress_min_bytes()) { + std::vector compressed; + if (zstd_compress(Slice(framed), kMetaSectionZstdLevel, &compressed).ok()) { + ByteSink payload; + payload.put_varint64(framed.size()); + payload.put_bytes(Slice(compressed)); + // ~9B frame overhead (type+len+crc); require a real win over raw. + if (payload.size() + 16 < framed.size()) { + SectionFramer::write(*sink, static_cast(zstd_type), payload.view()); + return; + } + } + } + sink->put_bytes(Slice(framed)); +} + +// Materializes an embedded sub-section frame captured by open(): a raw frame is +// returned as-is (view into the block; *scratch untouched); a zstd carrier frame +// is unwrapped and decompressed into *scratch. The decompressed bytes are the +// byte-exact original frame, whose own crc32c the sub-module reader re-verifies. +Status materialize_embedded_frame(Slice frame, bool is_zstd, const char* what, + std::vector* scratch, Slice* out) { + if (scratch == nullptr || out == nullptr) { + return Status::Error("per_index_meta: null frame out"); + } + if (!is_zstd) { + *out = frame; + return Status::OK(); + } + // Re-read the carrier frame (crc already verified during open's walk). + ByteSource src(frame); + FramedSection sec; + RETURN_IF_ERROR(SectionFramer::read(src, &sec)); + ByteSource ps(sec.payload); + uint64_t uncomp_len = 0; + RETURN_IF_ERROR(ps.get_varint64(&uncomp_len)); + if (uncomp_len == 0 || uncomp_len > kMaxMetaSectionUncompBytes) { + return Status::Error( + "per_index_meta: {} zstd uncomp_len out of range", what); + } + Slice comp; + RETURN_IF_ERROR(ps.get_bytes(ps.remaining(), &comp)); + RETURN_IF_ERROR(zstd_decompress(comp, static_cast(uncomp_len), scratch)); + *out = Slice(*scratch); + return Status::OK(); +} + +void encode_region(const RegionRef& r, ByteSink* payload) { + payload->put_varint64(r.offset); + payload->put_varint64(r.length); +} + +Status decode_region(ByteSource* ps, RegionRef* r) { + RETURN_IF_ERROR(ps->get_varint64(&r->offset)); + RETURN_IF_ERROR(ps->get_varint64(&r->length)); + return Status::OK(); +} + +// SectionRefs payload: five RegionRefs in fixed order, each as varint64 pair. +// Order: dict_region, posting_region, norms, null_bitmap, bsbf. +void encode_section_refs(const SectionRefs& refs, ByteSink* sink) { + ByteSink payload; + encode_region(refs.dict_region, &payload); + encode_region(refs.posting_region, &payload); + encode_region(refs.norms, &payload); + encode_region(refs.null_bitmap, &payload); + encode_region(refs.bsbf, &payload); + SectionFramer::write(*sink, static_cast(SectionType::kSectionRefs), payload.view()); +} + +Status decode_section_refs(Slice payload, SectionRefs* out) { + ByteSource ps(payload); + RETURN_IF_ERROR(decode_region(&ps, &out->dict_region)); + RETURN_IF_ERROR(decode_region(&ps, &out->posting_region)); + RETURN_IF_ERROR(decode_region(&ps, &out->norms)); + RETURN_IF_ERROR(decode_region(&ps, &out->null_bitmap)); + RETURN_IF_ERROR(decode_region(&ps, &out->bsbf)); + if (!ps.eof()) { + return Status::Error( + "per_index_meta: trailing bytes in section_refs"); + } + return Status::OK(); +} + +// kBigramPruneInfo payload: varint64 effective bigram_prune_min_df, then +// (G15) varint64 effective bigram_prune_max_df -- both written whenever the +// section is emitted (either gate active), 0 marking the inactive gate. The +// section is OPTIONAL (emitted only when the writer pruned) and +// forward-extensible: pre-G15 writers emitted only the min varint (decode +// defaults max to 0), and trailing payload bytes past the known fields are +// IGNORED so a future writer can append more without a new section type. +void encode_bigram_prune_info(uint64_t min_df, uint64_t max_df, ByteSink* sink) { + ByteSink payload; + payload.put_varint64(min_df); + payload.put_varint64(max_df); + SectionFramer::write(*sink, static_cast(SectionType::kBigramPruneInfo), + payload.view()); +} + +Status decode_bigram_prune_info(Slice payload, uint64_t* min_df, uint64_t* max_df) { + ByteSource ps(payload); + RETURN_IF_ERROR(ps.get_varint64(min_df)); + *max_df = 0; + if (!ps.eof()) { + RETURN_IF_ERROR(ps.get_varint64(max_df)); + } + return Status::OK(); +} + +// Writes the self-checksummed header prefix. Layout matches the class comment. +void encode_header(uint64_t index_id, const std::string& suffix, uint32_t flags, ByteSink* sink) { + ByteSink head; + head.put_fixed16(kMetaFormatVersion); + head.put_varint64(index_id); + head.put_varint32(static_cast(suffix.size())); + head.put_bytes(Slice(suffix)); + head.put_fixed32(flags); + uint32_t crc = crc32c(head.view()); + sink->put_bytes(head.view()); + sink->put_fixed32(crc); +} + +// Parses and crc-verifies the header prefix, advancing src past the crc field. +Status decode_header(Slice block, ByteSource* src, uint64_t* index_id, std::string* suffix, + uint32_t* flags) { + size_t start = src->position(); + uint16_t version = 0; + RETURN_IF_ERROR(src->get_fixed16(&version)); + if (version != kMetaFormatVersion) { + return Status::Error( + "per_index_meta: unsupported meta_format_version"); + } + RETURN_IF_ERROR(src->get_varint64(index_id)); + uint32_t suffix_len = 0; + RETURN_IF_ERROR(src->get_varint32(&suffix_len)); + if (suffix_len > kMaxSuffixLen || suffix_len > src->remaining()) { + return Status::Error( + "per_index_meta: suffix_len exceeds bounds"); + } + Slice suffix_view; + RETURN_IF_ERROR(src->get_bytes(suffix_len, &suffix_view)); + RETURN_IF_ERROR(src->get_fixed32(flags)); + size_t covered = src->position() - start; + uint32_t stored = 0; + RETURN_IF_ERROR(src->get_fixed32(&stored)); + if (crc32c(block.subslice(start, covered)) != stored) { + return Status::Error( + "per_index_meta: header crc mismatch"); + } + suffix->assign(reinterpret_cast(suffix_view.data()), suffix_view.size()); + return Status::OK(); +} + +// Reads one framed section, returning both its type and the FULL frame Slice +// (type+len+payload+crc) so it can be re-opened by a sub-module reader. The +// framer itself crc-verifies the frame. +Status read_frame(Slice block, ByteSource* src, uint8_t* type, Slice* frame) { + size_t start = src->position(); + FramedSection sec; + RETURN_IF_ERROR(SectionFramer::read(*src, &sec)); + *type = sec.type; + *frame = block.subslice(start, src->position() - start); + return Status::OK(); +} + +// Routes an embedded sub-section frame (raw or its G13 zstd carrier) to its +// slot, remembering which layout was captured. Unknown section types are +// intentionally ignored (forward compatibility: skip unknown optional sections). +void dispatch_frame(uint8_t type, Slice frame, Slice* sampled, bool* sampled_zstd, Slice* dict, + bool* dict_zstd) { + if (type == static_cast(SectionType::kSampledTermIndex)) { + *sampled = frame; + *sampled_zstd = false; + } else if (type == static_cast(SectionType::kSampledTermIndexZstd)) { + *sampled = frame; + *sampled_zstd = true; + } else if (type == static_cast(SectionType::kDictBlockDirectory)) { + *dict = frame; + *dict_zstd = false; + } else if (type == static_cast(SectionType::kDictBlockDirectoryZstd)) { + *dict = frame; + *dict_zstd = true; + } +} + +} // namespace + +PerIndexMetaBuilder::PerIndexMetaBuilder(uint64_t index_id, std::string index_suffix, + uint32_t flags) + : index_id_(index_id), index_suffix_(std::move(index_suffix)), flags_(flags) {} + +void PerIndexMetaBuilder::set_stats(const StatsBlock& stats) { + stats_ = stats; +} + +void PerIndexMetaBuilder::set_sampled_term_index(Slice framed_bytes) { + sampled_term_index_.assign(framed_bytes.data(), framed_bytes.data() + framed_bytes.size()); +} + +void PerIndexMetaBuilder::set_dict_block_directory(Slice framed_bytes) { + dict_block_directory_.assign(framed_bytes.data(), framed_bytes.data() + framed_bytes.size()); +} + +void PerIndexMetaBuilder::set_section_refs(const SectionRefs& refs) { + section_refs_ = refs; +} + +void PerIndexMetaBuilder::add_raw_section(Slice framed_bytes) { + extra_sections_.emplace_back(framed_bytes.data(), framed_bytes.data() + framed_bytes.size()); +} + +Status PerIndexMetaBuilder::finish(ByteSink* sink) const { + if (sink == nullptr) { + return Status::Error("per_index_meta: null sink"); + } + encode_header(index_id_, index_suffix_, flags_, sink); + encode_stats_block(stats_, sink); + emit_embedded_section(sampled_term_index_, SectionType::kSampledTermIndexZstd, sink); + emit_embedded_section(dict_block_directory_, SectionType::kDictBlockDirectoryZstd, sink); + encode_section_refs(section_refs_, sink); + if (bigram_prune_min_df_ != 0 || bigram_prune_max_df_ != 0) { + encode_bigram_prune_info(bigram_prune_min_df_, bigram_prune_max_df_, sink); + } + for (const auto& extra : extra_sections_) { + sink->put_bytes(Slice(extra)); + } + return Status::OK(); +} + +Status PerIndexMetaReader::open(Slice block, PerIndexMetaReader* out) { + if (out == nullptr) { + return Status::Error("per_index_meta: null reader"); + } + ByteSource src(block); + RETURN_IF_ERROR(decode_header(block, &src, &out->index_id_, &out->index_suffix_, &out->flags_)); + bool have_stats = false; + bool have_refs = false; + while (!src.eof()) { + uint8_t type = 0; + Slice frame; + RETURN_IF_ERROR(read_frame(block, &src, &type, &frame)); + if (type == static_cast(SectionType::kStatsBlock)) { + ByteSource fs(frame); + RETURN_IF_ERROR(decode_stats_block(&fs, &out->stats_)); + have_stats = true; + } else if (type == static_cast(SectionType::kSectionRefs)) { + FramedSection sec; + ByteSource fs(frame); + RETURN_IF_ERROR(SectionFramer::read(fs, &sec)); + RETURN_IF_ERROR(decode_section_refs(sec.payload, &out->section_refs_)); + have_refs = true; + } else if (type == static_cast(SectionType::kBigramPruneInfo)) { + FramedSection sec; + ByteSource fs(frame); + RETURN_IF_ERROR(SectionFramer::read(fs, &sec)); + RETURN_IF_ERROR(decode_bigram_prune_info(sec.payload, &out->bigram_prune_min_df_, + &out->bigram_prune_max_df_)); + } else { + dispatch_frame(type, frame, &out->sampled_term_index_, &out->sampled_term_index_zstd_, + &out->dict_block_directory_, &out->dict_block_directory_zstd_); + } + } + if (!have_stats || !have_refs) { + return Status::Error( + "per_index_meta: missing required sub-section"); + } + return Status::OK(); +} + +Status PerIndexMetaReader::sampled_term_index_frame(std::vector* scratch, + Slice* frame) const { + return materialize_embedded_frame(sampled_term_index_, sampled_term_index_zstd_, + "sampled_term_index", scratch, frame); +} + +Status PerIndexMetaReader::dict_block_directory_frame(std::vector* scratch, + Slice* frame) const { + return materialize_embedded_frame(dict_block_directory_, dict_block_directory_zstd_, + "dict_block_directory", scratch, frame); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/per_index_meta.h b/be/src/storage/index/snii/format/per_index_meta.h new file mode 100644 index 00000000000000..c2f5dbc63e5884 --- /dev/null +++ b/be/src/storage/index/snii/format/per_index_meta.h @@ -0,0 +1,221 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/stats_block.h" + +// PerIndexMeta -- the per-logical-index metadata block that enters the searcher +// cache. It COMPOSES already-built sub-sections (StatsBlock, SampledTermIndex, +// DICT block directory, optional XFilter) plus the physical SectionRefs into a +// single contiguous block. See design spec "Per-index meta block". +// +// On-disk layout: +// PerIndexMetaHeader (fixed prefix, self-checksummed): +// u16 meta_format_version (== kMetaFormatVersion), little-endian +// varint64 index_id +// varint32 suffix_len +// u8[] suffix_bytes +// u32 flags (fixed32, little-endian) # feature bits, e.g. kHasBsbf +// u32 crc32c (fixed32) over all preceding header bytes +// then framed sub-sections (each via SectionFramer, type+len+payload+crc32c): +// StatsBlock (kStatsBlock, built here) +// SampledTermIndex (kSampledTermIndex, embedded already-framed bytes) +// DICT block directory (kDictBlockDirectory,embedded already-framed bytes) +// SectionRefs (kSectionRefs, built here; carries the bsbf ref) +// (+ any extra raw framed sections appended by add_raw_section) +// +// Design choice: the SampledTermIndex / DICT block directory / XFilter +// sub-sections are EMBEDDED as their producers' already-framed output (the raw +// SectionFramer frame), not re-framed. This lets the reader hand the exact frame +// Slice straight back to each sub-module's open() (which expects a full frame), +// and reuses the framer instead of re-implementing sub-section parsing. +// +// G13: when an embedded SampledTermIndex / DICT block directory frame reaches +// kMetaSectionCompressMinBytes, finish() wraps it in a kSampledTermIndexZstd / +// kDictBlockDirectoryZstd frame instead (payload = varint64 uncomp_len + +// zstd(original frame)) -- these sorted string/offset tables compress several +// fold and dominate the serial per-segment meta fetch at open. Smaller frames +// (and any frame zstd fails to shrink) are emitted raw, byte-identical to the +// pre-G13 layout, so old segments keep reading through the same path. The reader +// captures either layout and materializes the original frame on demand via +// sampled_term_index_frame() / dict_block_directory_frame(). +namespace doris::snii::format { + +// Physical reference to a contiguous region within the container. (0, 0) means +// the region is absent (e.g. no norms POD for a non-scoring index). A present- +// but-empty region (e.g. an all-INLINE index's posting_region) is (off, 0). +struct RegionRef { + uint64_t offset = 0; + uint64_t length = 0; +}; + +// Physical references to the data sections / side PODs of one logical index. +// Each RegionRef is encoded as varint64 offset followed by varint64 length, in +// the field order below. +// +// posting_region is the single interleaved [prx][frq] posting region (it replaced +// the former two separate frq_pod + prx_pod refs). Each pod_ref term writes its +// prx span first then its frq span, contiguously, in term order; both +// frq_off_delta and prx_off_delta now index into this one region. NO positions +// capability is inferred from posting_region.length -- it is non-zero for any +// docs-only index with a pod_ref term, and zero for an all-INLINE positional +// index; capability lives in the header kHasPositions flag instead. +struct SectionRefs { + RegionRef dict_region; + RegionRef posting_region; // interleaved [prx][frq] per term; was frq_pod + prx_pod + RegionRef norms; + RegionRef null_bitmap; + // Block-split bloom XFilter section ([28B header][bitset]); {0,0} when absent. + // A PHYSICAL section (not embedded in the resident meta) so a single 32-byte block + // can be probed on demand without loading the whole filter at open. + RegionRef bsbf; +}; + +// Builds a per-index meta block by composing already-built sub-sections. +class PerIndexMetaBuilder { +public: + // Header flags / feature bits. + static constexpr uint32_t kHasPositions = 1u << 0; // index is positions-capable (tier>=T2) + static constexpr uint32_t kHasBsbf = 1u << 1; // block-split bloom XFilter (section ref) + static constexpr uint32_t kPhraseBigramsDeferred = 1u << 2; // no hidden pair postings/sentinel + + PerIndexMetaBuilder(uint64_t index_id, std::string index_suffix, uint32_t flags); + + void set_stats(const StatsBlock& stats); + + // Raw output of SampledTermIndexBuilder::finish (a full kSampledTermIndex frame). + void set_sampled_term_index(Slice framed_bytes); + + // Raw output of DictBlockDirectoryBuilder::finish (a full kDictBlockDirectory frame). + void set_dict_block_directory(Slice framed_bytes); + + void set_section_refs(const SectionRefs& refs); + + // Effective phrase-bigram df-prune thresholds applied by the writer (G01 + // min / G15 max, absolute doc counts). Either non-zero emits an OPTIONAL + // kBigramPruneInfo framed section carrying BOTH values (varint64 min then + // varint64 max); both 0 (the default) emits nothing -- legacy segments + // carry no section and old readers skip the new one (unknown optional + // type). + void set_bigram_prune_min_df(uint64_t min_df) { bigram_prune_min_df_ = min_df; } + void set_bigram_prune_max_df(uint64_t max_df) { bigram_prune_max_df_ = max_df; } + + // Appends an arbitrary already-framed section verbatim. Used for forward-compat + // optional sections; the reader skips unrecognized types. + void add_raw_section(Slice framed_bytes); + + // Serializes the header and all sub-sections into sink. + // sink == nullptr -> kInvalidArgument. + Status finish(ByteSink* sink) const; + +private: + uint64_t index_id_; + std::string index_suffix_; + uint32_t flags_; + StatsBlock stats_; + std::vector sampled_term_index_; + std::vector dict_block_directory_; + SectionRefs section_refs_; + uint64_t bigram_prune_min_df_ = 0; + uint64_t bigram_prune_max_df_ = 0; + std::vector> extra_sections_; +}; + +// Parses a per-index meta block: verifies the header crc, then walks the framed +// sub-sections (each crc-verified by the framer), capturing the full frame Slice +// of each known sub-section so callers can re-open it with the sub-module reader. +// Unrecognized optional section types are skipped. +class PerIndexMetaReader { +public: + PerIndexMetaReader() = default; + + // block == the full per-index meta block bytes; out must be non-null. + // Header crc mismatch / truncation / a sub-section crc mismatch -> kCorruption; + // missing a required sub-section -> kCorruption; out == nullptr -> kInvalidArgument. + static Status open(Slice block, PerIndexMetaReader* out); + + uint64_t index_id() const { return index_id_; } + const std::string& index_suffix() const { return index_suffix_; } + uint32_t flags() const { return flags_; } + + const StatsBlock& stats() const { return stats_; } + const SectionRefs& section_refs() const { return section_refs_; } + + // Materializes the full kSampledTermIndex frame, ready for + // SampledTermIndexReader::open. A raw (uncompressed) section returns a view + // into the opened block (*scratch untouched); a kSampledTermIndexZstd section + // decompresses into *scratch and returns a view of it (G13). Decompression is + // deliberately deferred to this call: several open() callers only need the + // header/refs and must not pay a decompress. Corrupt/truncated zstd payload + // or an out-of-range uncomp_len -> kCorruption. *scratch must be non-null and + // must outlive the returned *frame; the sub-module readers fully materialize + // on open, so a stack-local scratch is sufficient. + Status sampled_term_index_frame(std::vector* scratch, Slice* frame) const; + // Same contract for the kDictBlockDirectory frame / DictBlockDirectoryReader. + Status dict_block_directory_frame(std::vector* scratch, Slice* frame) const; + + // Block-split bloom XFilter: present iff a non-empty bsbf section ref exists. + bool has_bsbf() const { return section_refs_.bsbf.length > 0; } + + // Positions capability, read from the persisted header flag (NOT from any region + // length). True iff the index was built as docs-positions(+scoring) (tier>=T2). + bool has_positions() const { return (flags_ & PerIndexMetaBuilder::kHasPositions) != 0; } + + // The writer deliberately skipped every hidden phrase-bigram token and the + // sentinel for this fresh direct-load segment. The flag is resident metadata, + // so phrase readers can enter positions verification without probing a pair + // term that cannot exist. Absent on all older segments. + bool phrase_bigrams_deferred() const { + return (flags_ & PerIndexMetaBuilder::kPhraseBigramsDeferred) != 0; + } + + // Effective phrase-bigram df-prune thresholds the writer applied (G01 min / + // G15 max), from the OPTIONAL kBigramPruneInfo section. Both 0 == section + // absent == legacy semantics (every adjacent pair was materialized; a + // bigram dict miss means "no adjacency"). Either non-zero declares this + // index bigram-df-pruned: a bigram dict miss must fall back to generic + // positions verification. max is 0 on pre-G15 sections (single-varint + // payload) -- only the min gate was applied there. + uint64_t bigram_prune_min_df() const { return bigram_prune_min_df_; } + uint64_t bigram_prune_max_df() const { return bigram_prune_max_df_; } + +private: + uint64_t index_id_ = 0; + std::string index_suffix_; + uint32_t flags_ = 0; + StatsBlock stats_; + SectionRefs section_refs_; + uint64_t bigram_prune_min_df_ = 0; + uint64_t bigram_prune_max_df_ = 0; + // Captured frame Slices into the opened block: the raw sub-section frame, or + // its kXxxZstd carrier frame when *_zstd_ is set (G13). + Slice sampled_term_index_; + bool sampled_term_index_zstd_ = false; + Slice dict_block_directory_; + bool dict_block_directory_zstd_ = false; +}; + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/phrase_bigram.h b/be/src/storage/index/snii/format/phrase_bigram.h new file mode 100644 index 00000000000000..327ff18780180d --- /dev/null +++ b/be/src/storage/index/snii/format/phrase_bigram.h @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +namespace doris::snii::format { + +inline constexpr std::string_view kPhraseBigramTermMarker = + "\x1F" + "SNII_PHRASE_BIGRAM" + "\x1F"; + +// Encodes `value` as LEB128 into `buf` (>= 5 bytes), returning the byte count. +// Shared by the string-appending encoder below and by piecewise (no-string) +// consumers such as the SPIMI zero-alloc bigram probe. +inline size_t encode_phrase_bigram_varint32(uint32_t value, char* buf) { + size_t n = 0; + while (value >= 0x80) { + buf[n++] = static_cast((value & 0x7F) | 0x80); + value >>= 7; + } + buf[n++] = static_cast(value); + return n; +} + +inline void append_phrase_bigram_varint32(uint32_t value, std::string* out) { + char buf[5]; + out->append(buf, encode_phrase_bigram_varint32(value, buf)); +} + +inline std::string make_phrase_bigram_term(std::string_view left, std::string_view right) { + std::string out; + out.reserve(kPhraseBigramTermMarker.size() + 5 + left.size() + right.size()); + out.append(kPhraseBigramTermMarker); + append_phrase_bigram_varint32(static_cast(left.size()), &out); + out.append(left); + out.append(right); + return out; +} + +inline std::string make_phrase_bigram_sentinel_term() { + std::string out(kPhraseBigramTermMarker); + out.push_back('\0'); + return out; +} + +inline bool is_phrase_bigram_term(std::string_view term) { + return term.starts_with(kPhraseBigramTermMarker); +} + +// The sentinel is marker + '\0'. A REAL bigram term is marker + varint(len(left)) +// + left + right with len(left) >= 1, so its first post-marker byte is never 0 -- +// the sentinel is unambiguous. The sentinel marks "this index was built with the +// hidden-bigram feature" and must NEVER be df-pruned (it gates reader semantics). +inline bool is_phrase_bigram_sentinel_term(std::string_view term) { + return term.size() == kPhraseBigramTermMarker.size() + 1 && + term.starts_with(kPhraseBigramTermMarker) && term.back() == '\0'; +} + +// Piecewise equality: does `term` equal make_phrase_bigram_term(left, right)? +// Compares marker / varint(len(left)) / left / right fragment by fragment so the +// caller never composes the synthetic string (the SPIMI zero-alloc intern probe). +inline bool phrase_bigram_term_equals(std::string_view term, std::string_view left, + std::string_view right) { + if (!term.starts_with(kPhraseBigramTermMarker)) { + return false; + } + char varint_buf[5]; + const size_t varint_len = + encode_phrase_bigram_varint32(static_cast(left.size()), varint_buf); + const size_t expected_size = + kPhraseBigramTermMarker.size() + varint_len + left.size() + right.size(); + if (term.size() != expected_size) { + return false; + } + size_t off = kPhraseBigramTermMarker.size(); + if (term.substr(off, varint_len) != std::string_view(varint_buf, varint_len)) { + return false; + } + off += varint_len; + if (term.substr(off, left.size()) != left) { + return false; + } + off += left.size(); + return term.substr(off) == right; +} + +// Default phrase-bigram df-prune threshold for a segment of `segment_doc_count` +// docs: max(64, doc_count / 10000) (0.01%). Pure policy (not format semantics); +// the EFFECTIVE threshold a writer applied is recorded in the per-index meta +// (SectionType::kBigramPruneInfo), so readers never re-derive it from this +// formula. +inline uint32_t default_phrase_bigram_prune_min_df(uint64_t segment_doc_count) { + return static_cast( + std::max(64, std::min(segment_doc_count / 10000, UINT32_MAX))); +} + +inline bool is_ascii_alpha_phrase_bigram_char(char c) { + return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z'); +} + +inline bool is_phrase_bigram_indexable_term(std::string_view term) { + constexpr size_t kMinPhraseBigramTermLength = 2; + constexpr size_t kMaxPhraseBigramTermLength = 32; + if (term.size() < kMinPhraseBigramTermLength || term.size() > kMaxPhraseBigramTermLength) { + return false; + } + for (const char c : term) { + if (!is_ascii_alpha_phrase_bigram_char(c)) { + return false; + } + } + return true; +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/prx_pod.cpp b/be/src/storage/index/snii/format/prx_pod.cpp new file mode 100644 index 00000000000000..e3d917eb78cf32 --- /dev/null +++ b/be/src/storage/index/snii/format/prx_pod.cpp @@ -0,0 +1,920 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/prx_pod.h" + +#include +#include +#include +#include +#include +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/common/uninitialized_buffer.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/encoding/pfor.h" +#include "storage/index/snii/encoding/zstd_codec.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::format { +namespace { + +// Auto-compression threshold: use raw when payload is smaller than this (zstd +// gain is negligible and metadata overhead is relatively large). +inline constexpr size_t kAutoZstdMinBytes = 512; +// Default zstd level in auto mode. +inline constexpr int kDefaultZstdLevel = 3; +// Maximum decompressed byte size for a single .prx window. Guards against a +// corrupted uncomp_len read from S3 inflated to a huge value: sanity-check +// before allocating/decompressing to avoid GB-scale allocations. Windows are +// 256-doc aligned and normally far below this limit. +inline constexpr uint32_t kMaxWindowUncompBytes = 256u * 1024 * 1024; +// Anti-DoS cap on position count decoded from a single window before +// allocation. +inline constexpr uint32_t kMaxWindowPositions = 1u << 26; // 64M positions/window +// Anti-DoS cap on doc count decoded from a single window before allocation. A +// corrupt doc_count is otherwise fed straight to assign()/reserve() -> +// bad_alloc. +inline constexpr uint32_t kMaxWindowDocs = 1u << 24; // 16M docs/window + +// Writer-side precondition for the FLAT builders: the per-doc partition `freqs` +// must address exactly the positions present in `flat`. If sum(freqs) overruns +// flat.size() a (positions_flat, freqs) mismatch would index flat[off+i] past +// the span end -- an out-of-bounds read on caller-supplied data. Reject it as +// InvalidArgument BEFORE any indexing so the bug surfaces as a clean Status, +// never UB. (sum < size leaves trailing positions unused, which is also a +// writer bug, so we require exact equality.) Uint64 accumulation cannot +// overflow for uint32 freqs. +Status check_flat_partition(std::span flat, std::span freqs) { + uint64_t sum = 0; + for (uint32_t fc : freqs) sum += fc; + if (sum != flat.size()) { + return Status::Error( + "prx: sum(freqs) does not match positions_flat size"); + } + return Status::OK(); +} + +// Encode per-doc position lists into a self-describing plain payload (doc_count +// + per-doc delta stream). +Status encode_payload(std::span> per_doc, ByteSink* out) { + out->put_varint32(static_cast(per_doc.size())); + for (const auto& doc : per_doc) { + out->put_varint32(static_cast(doc.size())); + uint32_t prev = 0; + for (size_t i = 0; i < doc.size(); ++i) { + uint32_t pos = doc[i]; + if (i > 0 && pos < prev) { + return Status::Error( + "prx: positions within a doc must be ascending"); + } + out->put_varint32(i == 0 ? pos : pos - prev); + prev = pos; + } + } + return Status::OK(); +} + +// FLAT-positions encoder: identical wire output to encode_payload above, but +// reads positions from a single flat span partitioned per-doc by `freqs` (doc d +// owns the next freqs[d] entries). This avoids materializing a +// vector-of-vectors for the window; freqs.size() is the doc count and +// sum(freqs) == flat.size(). +Status encode_payload_flat(std::span flat, std::span freqs, + ByteSink* out) { + RETURN_IF_ERROR(check_flat_partition(flat, freqs)); + out->put_varint32(static_cast(freqs.size())); + size_t off = 0; + for (uint32_t fc : freqs) { + out->put_varint32(fc); + uint32_t prev = 0; + for (uint32_t i = 0; i < fc; ++i) { + const uint32_t pos = flat[off + i]; + if (i > 0 && pos < prev) { + return Status::Error( + "prx: positions within a doc must be ascending"); + } + out->put_varint32(i == 0 ? pos : pos - prev); + prev = pos; + } + off += fc; + } + return Status::OK(); +} + +// Encode a uint32 array into PFOR runs of kFrqBaseUnit (256) elements each. The +// run count is derived by the decoder from the total length, so it is not +// stored. +void encode_pfor_runs(std::span values, ByteSink* out) { + const size_t n = values.size(); + for (size_t off = 0; off < n; off += kFrqBaseUnit) { + const size_t run = (n - off < kFrqBaseUnit) ? (n - off) : kFrqBaseUnit; + pfor_encode(values.data() + off, run, out); + } +} + +// Decode n uint32 values (multiple PFOR runs of kFrqBaseUnit each) into out. +Status decode_pfor_runs(ByteSource* src, size_t n, std::vector* out) { + // Sized then fully overwritten by pfor_decode below (every [0, n) slot is + // written); no zero-fill needed beyond what std::vector mandates. + resize_uninitialized(*out, n); + for (size_t off = 0; off < n; off += kFrqBaseUnit) { + const size_t run = (n - off < kFrqBaseUnit) ? (n - off) : kFrqBaseUnit; + RETURN_IF_ERROR(pfor_decode(src, run, out->data() + off)); + } + return Status::OK(); +} + +// Derive the per-doc position deltas ONCE into `deltas` (flat, in doc order: the +// first position of each doc is absolute, the rest are deltas within the doc), +// enforcing the writer-side preconditions BOTH payload encoders relied on: the +// (flat, freqs) partition must be exact (check_flat_partition) and positions +// within each doc must be ascending. The loop is identical to the delta +// derivation the old encode_pfor_payload_flat ran inline, lifted out so the auto +// path can feed BOTH the PFOR payload and (only when needed) the raw plaintext +// payload from one buffer instead of walking `flat` twice. +Status compute_flat_deltas(std::span flat, std::span freqs, + std::vector* deltas) { + RETURN_IF_ERROR(check_flat_partition(flat, freqs)); + deltas->clear(); + deltas->reserve(flat.size()); + size_t off = 0; + for (uint32_t fc : freqs) { + uint32_t prev = 0; + for (uint32_t i = 0; i < fc; ++i) { + const uint32_t pos = flat[off + i]; + if (i > 0 && pos < prev) { + return Status::Error( + "prx: positions within a doc must be ascending"); + } + deltas->push_back(i == 0 ? pos : pos - prev); + prev = pos; + } + off += fc; + } + return Status::OK(); +} + +// PFOR window payload (self-describing; no entropy coding): +// VInt doc_count +// VInt total_pos # sum of all pos_counts +// PFOR_runs(pos_counts) # doc_count values (bit-packed; mostly 1 -> ~1 +// bit) PFOR_runs(position_deltas) # total_pos deltas, flat across docs (first +// per +// # doc absolute, rest delta-within-doc) +// Bit-packing the per-doc pos_counts (vs one varint each) is the size win: in a +// uniform corpus most docs have freq 1, so the count column packs to ~1 bit/doc. +// Emits byte-for-byte the same payload the old encode_pfor_payload_flat produced +// (doc_count == freqs.size(), total_pos == deltas.size() == sum(freqs)), but +// reads the already-derived `deltas` instead of re-walking the positions. +void encode_pfor_payload_from_deltas(std::span freqs, + std::span deltas, ByteSink* out) { + out->put_varint32(static_cast(freqs.size())); + out->put_varint32(static_cast(deltas.size())); + encode_pfor_runs(freqs, out); + encode_pfor_runs(deltas, out); +} + +// Raw plaintext payload (self-describing per-doc boundaries): +// VInt doc_count +// per doc: VInt pos_count, then pos_count position deltas (VInt) +// Emits byte-for-byte the same payload the old encode_payload_flat produced, but +// reads the already-derived `deltas` instead of re-walking the positions and +// re-running the partition/ascending checks. +void encode_payload_from_deltas(std::span freqs, std::span deltas, + ByteSink* out) { + out->put_varint32(static_cast(freqs.size())); + size_t off = 0; + for (uint32_t fc : freqs) { + out->put_varint32(fc); + for (uint32_t i = 0; i < fc; ++i) { + out->put_varint32(deltas[off + i]); + } + off += fc; + } +} + +// Decode per-doc position lists from a PFOR payload. +Status decode_pfor_payload(Slice plain, std::vector>* out) { + ByteSource src(plain); + uint32_t doc_count = 0, total_pos = 0; + RETURN_IF_ERROR(src.get_varint32(&doc_count)); + RETURN_IF_ERROR(src.get_varint32(&total_pos)); + if (total_pos > kMaxWindowPositions) { + return Status::Error( + "prx: position count exceeds sane cap"); + } + if (doc_count > kMaxWindowDocs) { + return Status::Error( + "prx: doc count exceeds sane cap"); + } + std::vector pos_counts; + RETURN_IF_ERROR(decode_pfor_runs(&src, doc_count, &pos_counts)); + uint64_t sum = 0; + for (uint32_t d = 0; d < doc_count; ++d) sum += pos_counts[d]; + if (sum != total_pos) { + return Status::Error( + "prx: pos_count sum mismatch"); + } + std::vector deltas; + RETURN_IF_ERROR(decode_pfor_runs(&src, total_pos, &deltas)); + out->clear(); + out->reserve(doc_count); + size_t off = 0; + for (uint32_t d = 0; d < doc_count; ++d) { + std::vector doc; + doc.reserve(pos_counts[d]); + uint32_t prev = 0; + for (uint32_t i = 0; i < pos_counts[d]; ++i) { + prev = (i == 0) ? deltas[off + i] : prev + deltas[off + i]; + doc.push_back(prev); + } + off += pos_counts[d]; + out->push_back(std::move(doc)); + } + if (!src.eof()) + return Status::Error( + "prx: trailing bytes after pfor payload"); + return Status::OK(); +} + +// Writes a PFOR window: codec=pfor, payload, crc(header+payload). +void write_pfor(Slice payload, ByteSink* sink) { + // Single-copy framing: write [codec][varint len][payload] straight into the + // caller's sink, then crc exactly those bytes. view() is taken AFTER the + // payload and BEFORE the crc, so subslice([start, framed_len)) is over a + // settled, contiguous buffer with no pending realloc/aliasing. Byte-identical + // to the former temp-ByteSink assembly, minus one heap alloc + one payload copy. + const size_t start = sink->size(); + sink->put_u8(static_cast(PrxCodec::kPfor)); + sink->put_varint32(static_cast(payload.size())); + sink->put_bytes(payload); + const size_t framed_len = sink->size() - start; + const uint32_t crc = crc32c(sink->view().subslice(start, framed_len)); + sink->put_fixed32(crc); +} + +size_t varint32_size(uint32_t value) { + size_t bytes = 1; + while (value >= 128) { + value >>= 7; + ++bytes; + } + return bytes; +} + +// Exact byte length of the raw plaintext payload encode_payload_from_deltas would +// emit, computed WITHOUT materializing it. Mirrors that encoder field for field: +// varint32_size(doc_count) + Sum varint32_size(pos_count) + Sum varint32_size(delta) +// (order is irrelevant to the total). The auto path uses this to decide whether +// the raw plaintext is large enough (>= kAutoZstdMinBytes) to be worth +// materializing for the zstd-vs-pfor comparison, so it MUST equal +// encode_payload_from_deltas's output size EXACTLY -- otherwise the codec choice +// could drift across the 512 threshold and change the on-disk bytes. +size_t exact_plain_payload_size(std::span freqs, std::span deltas) { + size_t bytes = varint32_size(static_cast(freqs.size())); + for (uint32_t fc : freqs) { + bytes += varint32_size(fc); + } + for (uint32_t delta : deltas) { + bytes += varint32_size(delta); + } + return bytes; +} + +size_t pfor_frame_size(size_t payload_size) { + return 1 + varint32_size(static_cast(payload_size)) + payload_size + sizeof(uint32_t); +} + +size_t zstd_frame_size(size_t plain_size, size_t compressed_size) { + return 1 + varint32_size(static_cast(plain_size)) + + varint32_size(static_cast(compressed_size)) + compressed_size + + sizeof(uint32_t); +} + +void write_zstd_compressed(Slice plain, Slice compressed, ByteSink* sink) { + // Single-copy framing (see write_pfor): assemble [codec][uncomp_len][comp_len] + // [compressed] in the caller's sink and crc that span before appending the crc. + const size_t start = sink->size(); + sink->put_u8(static_cast(PrxCodec::kZstd)); + sink->put_varint32(static_cast(plain.size())); + sink->put_varint32(static_cast(compressed.size())); + sink->put_bytes(compressed); + const size_t framed_len = sink->size() - start; + const uint32_t crc = crc32c(sink->view().subslice(start, framed_len)); + sink->put_fixed32(crc); +} + +Status write_auto_pfor_or_zstd(Slice pfor_payload, Slice plain_payload, int zstd_level, + ByteSink* sink) { + if (plain_payload.size() >= kAutoZstdMinBytes) { + std::vector compressed; + RETURN_IF_ERROR(zstd_compress(plain_payload, zstd_level, &compressed)); + if (zstd_frame_size(plain_payload.size(), compressed.size()) < + pfor_frame_size(pfor_payload.size())) { + write_zstd_compressed(plain_payload, Slice(compressed), sink); + return Status::OK(); + } + } + write_pfor(pfor_payload, sink); + return Status::OK(); +} + +// Shared auto-mode (kAutoZstd) single-encode path for BOTH .prx builders. The +// per-doc and flat builders both funnel through here on a flat (positions, freqs) +// view so their windows stay byte-identical. Derives the per-doc deltas once, +// encodes the PFOR payload from them, and only materializes the raw plaintext +// payload (to try zstd) when its EXACT size reaches kAutoZstdMinBytes; smaller +// windows -- the vast majority in a Zipfian corpus -- skip the throwaway +// plaintext ByteSink and the second delta walk entirely and emit PFOR directly. +// Byte-identical to the former "encode_pfor_payload(_flat) + encode_payload(_flat) +// + write_auto_pfor_or_zstd" sequence: identical PFOR bytes, identical plaintext +// bytes when materialized, and an exact (not estimated) size so the 512-threshold +// codec choice never drifts. +Status build_prx_window_auto_from_flat(std::span positions_flat, + std::span freqs, int zstd_level, + ByteSink* sink) { + std::vector deltas; + RETURN_IF_ERROR(compute_flat_deltas(positions_flat, freqs, &deltas)); + ByteSink payload; + encode_pfor_payload_from_deltas(freqs, deltas, &payload); + if (exact_plain_payload_size(freqs, deltas) >= kAutoZstdMinBytes) { + ByteSink plain; + encode_payload_from_deltas(freqs, deltas, &plain); + testing::note_prx_raw_build(); + return write_auto_pfor_or_zstd(payload.view(), plain.view(), zstd_level, sink); + } + write_pfor(payload.view(), sink); + return Status::OK(); +} + +// Decode per-doc position lists from a plain payload. +Status decode_payload(Slice plain, std::vector>* out) { + ByteSource src(plain); + uint32_t doc_count = 0; + RETURN_IF_ERROR(src.get_varint32(&doc_count)); + if (doc_count > kMaxWindowDocs) { + return Status::Error( + "prx: doc count exceeds sane cap"); + } + out->clear(); + out->reserve(doc_count); + for (uint32_t d = 0; d < doc_count; ++d) { + uint32_t pos_count = 0; + RETURN_IF_ERROR(src.get_varint32_fast(&pos_count)); + std::vector doc; + doc.reserve(pos_count); + uint32_t prev = 0; + for (uint32_t i = 0; i < pos_count; ++i) { + uint32_t delta = 0; + RETURN_IF_ERROR(src.get_varint32(&delta)); + prev = (i == 0) ? delta : prev + delta; + doc.push_back(prev); + } + out->push_back(std::move(doc)); + } + if (!src.eof()) + return Status::Error( + "prx: trailing bytes after payload"); + return Status::OK(); +} + +// CSR decode of a PFOR payload: all docs' positions into one flat buffer + +// per-doc offsets, with NO per-doc std::vector allocation. `pos_off` has +// doc_count+1 entries (pos_off[0]==0); doc d's positions are +// pos_flat[pos_off[d] .. pos_off[d+1]). +Status decode_pfor_payload_csr(Slice plain, std::vector* pos_flat, + std::vector* pos_off) { + ByteSource src(plain); + uint32_t doc_count = 0, total_pos = 0; + RETURN_IF_ERROR(src.get_varint32(&doc_count)); + RETURN_IF_ERROR(src.get_varint32(&total_pos)); + if (total_pos > kMaxWindowPositions) { + return Status::Error( + "prx: position count exceeds sane cap"); + } + if (doc_count > kMaxWindowDocs) { + return Status::Error( + "prx: doc count exceeds sane cap"); + } + pos_off->clear(); + pos_off->reserve(static_cast(doc_count) + 1); + RETURN_IF_ERROR(decode_pfor_runs(&src, doc_count, pos_off)); + uint64_t sum = 0; + for (uint32_t d = 0; d < doc_count; ++d) sum += (*pos_off)[d]; + if (sum != total_pos) + return Status::Error( + "prx: pos_count sum mismatch"); + // pos_flat is sized to total_pos by decode_pfor_runs (resize_uninitialized); + // a separate reserve is redundant. pos_off keeps its reserve (push_back below). + RETURN_IF_ERROR(decode_pfor_runs(&src, total_pos, pos_flat)); + size_t off = 0; + uint32_t next_off = 0; + for (uint32_t d = 0; d < doc_count; ++d) { + const uint32_t pos_count = (*pos_off)[d]; + (*pos_off)[d] = next_off; + uint32_t prev = 0; + for (uint32_t i = 0; i < pos_count; ++i) { + uint32_t& value = (*pos_flat)[off + i]; + prev = (i == 0) ? value : prev + value; + value = prev; + } + off += pos_count; + next_off += pos_count; + } + pos_off->push_back(next_off); + if (!src.eof()) + return Status::Error( + "prx: trailing bytes after pfor payload"); + return Status::OK(); +} + +Status validate_doc_ordinals(std::span doc_ordinals, uint32_t doc_count) { + uint32_t prev = 0; + for (size_t i = 0; i < doc_ordinals.size(); ++i) { + const uint32_t doc = doc_ordinals[i]; + if (doc >= doc_count) { + return Status::Error( + "prx: selected doc ordinal out of range"); + } + if (i != 0 && doc <= prev) { + return Status::Error( + "prx: selected doc ordinals must be strictly ascending"); + } + prev = doc; + } + return Status::OK(); +} + +struct SelectedRange { + SelectedRange(uint32_t begin_, uint32_t end_, uint32_t out_begin_) + : begin(begin_), end(end_), out_begin(out_begin_) {} + + uint32_t begin; + uint32_t end; + uint32_t out_begin; +}; + +uint32_t count_covered_pfor_runs(std::span selected, uint32_t total_pos) { + if (selected.empty() || total_pos == 0) { + return 0; + } + uint32_t runs = 0; + uint32_t next_run = 0; + for (const SelectedRange& range : selected) { + if (range.begin == range.end) { + continue; + } + const uint32_t first_run = range.begin / kFrqBaseUnit; + const uint32_t last_run = (range.end - 1) / kFrqBaseUnit; + const uint32_t counted_first = std::max(first_run, next_run); + if (counted_first <= last_run) { + runs += last_run - counted_first + 1; + next_run = last_run + 1; + } + } + return runs; +} + +bool should_decode_full_prx_positions(std::span selected, + uint32_t selected_pos_count, uint32_t total_pos) { + if (selected.empty() || total_pos == 0) { + return false; + } + if (selected_pos_count * 2 >= total_pos) { + return true; + } + const uint32_t total_runs = (total_pos + kFrqBaseUnit - 1) / kFrqBaseUnit; + const uint32_t covered_runs = count_covered_pfor_runs(selected, total_pos); + return covered_runs * 4 >= total_runs * 3; +} + +Status decode_selected_pfor_count_ranges(ByteSource* src, uint32_t doc_count, + std::span doc_ordinals, + std::vector& selected, + std::vector& pos_off, uint64_t* total_pos_count, + uint32_t* selected_pos_count) { + selected.clear(); + selected.reserve(doc_ordinals.size()); + pos_off.clear(); + pos_off.reserve(doc_ordinals.size() + 1); + pos_off.push_back(0); + + *selected_pos_count = 0; + uint32_t delta_begin = 0; + size_t next_doc = 0; + *total_pos_count = 0; + std::array run_buf {}; + for (uint32_t run_begin = 0; run_begin < doc_count; run_begin += kFrqBaseUnit) { + const uint32_t run_len = std::min(kFrqBaseUnit, doc_count - run_begin); + RETURN_IF_ERROR(pfor_decode(src, run_len, run_buf.data())); + for (uint32_t i = 0; i < run_len; ++i) { + const uint32_t d = run_begin + i; + const uint32_t count = run_buf[i]; + *total_pos_count += count; + if (*total_pos_count > kMaxWindowPositions) { + return Status::Error( + "prx: pos_count sum exceeds sane cap"); + } + if (next_doc < doc_ordinals.size() && doc_ordinals[next_doc] == d) { + selected.emplace_back(delta_begin, delta_begin + count, *selected_pos_count); + *selected_pos_count += count; + pos_off.push_back(*selected_pos_count); + ++next_doc; + } + delta_begin += count; + } + } + if (next_doc != doc_ordinals.size()) { + return Status::Error( + "prx: selected doc ordinal was not decoded"); + } + return Status::OK(); +} + +Status decode_selected_pfor_positions(ByteSource* src, uint32_t total_pos, + std::span selected, bool decode_all_runs, + std::span pos_flat) { + std::array run_buf {}; + size_t range_idx = 0; + uint32_t prev = 0; + for (uint32_t run_begin = 0; run_begin < total_pos; run_begin += kFrqBaseUnit) { + const uint32_t run_len = std::min(kFrqBaseUnit, total_pos - run_begin); + const uint32_t run_end = run_begin + run_len; + while (range_idx < selected.size() && selected[range_idx].end <= run_begin) { + ++range_idx; + prev = 0; + } + if (!decode_all_runs && + (range_idx == selected.size() || selected[range_idx].begin >= run_end)) { + RETURN_IF_ERROR(pfor_skip(src, run_len)); + continue; + } + + RETURN_IF_ERROR(pfor_decode(src, run_len, run_buf.data())); + while (range_idx < selected.size() && selected[range_idx].begin < run_end) { + const SelectedRange& range = selected[range_idx]; + const uint32_t copy_begin = std::max(range.begin, run_begin); + const uint32_t copy_end = std::min(range.end, run_end); + if (copy_begin == range.begin) { + prev = 0; + } + uint32_t dst = range.out_begin + copy_begin - range.begin; + for (uint32_t off = copy_begin; off < copy_end; ++off) { + const uint32_t delta = run_buf[off - run_begin]; + prev = (off == range.begin) ? delta : prev + delta; + pos_flat[dst++] = prev; + } + if (copy_end < range.end) { + break; + } + ++range_idx; + prev = 0; + } + } + return Status::OK(); +} + +Status decode_pfor_payload_csr_selective(Slice plain, std::span doc_ordinals, + std::vector* pos_flat, + std::vector* pos_off) { + ByteSource src(plain); + uint32_t doc_count = 0, total_pos = 0; + RETURN_IF_ERROR(src.get_varint32(&doc_count)); + RETURN_IF_ERROR(src.get_varint32(&total_pos)); + if (total_pos > kMaxWindowPositions) { + return Status::Error( + "prx: position count exceeds sane cap"); + } + if (doc_count > kMaxWindowDocs) { + return Status::Error( + "prx: doc count exceeds sane cap"); + } + RETURN_IF_ERROR(validate_doc_ordinals(doc_ordinals, doc_count)); + + pos_flat->clear(); + + std::vector selected; + uint64_t sum = 0; + uint32_t selected_pos_count = 0; + RETURN_IF_ERROR(decode_selected_pfor_count_ranges(&src, doc_count, doc_ordinals, selected, + *pos_off, &sum, &selected_pos_count)); + if (sum != total_pos) { + return Status::Error( + "prx: pos_count sum mismatch"); + } + + pos_flat->resize(selected_pos_count); + RETURN_IF_ERROR(decode_selected_pfor_positions( + &src, total_pos, selected, + should_decode_full_prx_positions(selected, selected_pos_count, total_pos), + std::span(pos_flat->data(), pos_flat->size()))); + if (!src.eof()) { + return Status::Error( + "prx: trailing bytes after pfor payload"); + } + return Status::OK(); +} + +// CSR decode of a plain (raw) payload. See decode_pfor_payload_csr. +Status decode_payload_csr(Slice plain, std::vector* pos_flat, + std::vector* pos_off) { + ByteSource src(plain); + uint32_t doc_count = 0; + RETURN_IF_ERROR(src.get_varint32(&doc_count)); + if (doc_count > kMaxWindowDocs) { + return Status::Error( + "prx: doc count exceeds sane cap"); + } + pos_flat->clear(); + pos_off->clear(); + pos_off->reserve(static_cast(doc_count) + 1); + pos_off->push_back(0); + uint64_t total_pos = 0; + for (uint32_t d = 0; d < doc_count; ++d) { + uint32_t pos_count = 0; + RETURN_IF_ERROR(src.get_varint32_fast(&pos_count)); + total_pos += pos_count; + if (total_pos > kMaxWindowPositions) { + return Status::Error( + "prx: position count exceeds sane cap"); + } + // Tight inline prefix-sum decode (single-byte fast path) -- see + // decode_delta_run. Shared with the selective reader below. + RETURN_IF_ERROR(src.decode_delta_run(pos_count, pos_flat)); + pos_off->push_back(static_cast(pos_flat->size())); + } + if (!src.eof()) + return Status::Error( + "prx: trailing bytes after payload"); + return Status::OK(); +} + +Status decode_payload_csr_selective(Slice plain, std::span doc_ordinals, + std::vector* pos_flat, + std::vector* pos_off) { + ByteSource src(plain); + uint32_t doc_count = 0; + RETURN_IF_ERROR(src.get_varint32(&doc_count)); + if (doc_count > kMaxWindowDocs) { + return Status::Error( + "prx: doc count exceeds sane cap"); + } + RETURN_IF_ERROR(validate_doc_ordinals(doc_ordinals, doc_count)); + pos_flat->clear(); + pos_off->clear(); + pos_off->reserve(doc_ordinals.size() + 1); + pos_off->push_back(0); + size_t next_doc = 0; + uint64_t total_pos = 0; + for (uint32_t d = 0; d < doc_count; ++d) { + uint32_t pos_count = 0; + RETURN_IF_ERROR(src.get_varint32_fast(&pos_count)); + total_pos += pos_count; + if (total_pos > kMaxWindowPositions) { + return Status::Error( + "prx: position count exceeds sane cap"); + } + const bool selected = next_doc < doc_ordinals.size() && doc_ordinals[next_doc] == d; + if (!selected) { + // Skip this doc's position deltas without decoding them -- the CSR + // layout is sequential so we must advance past them, but only the + // candidate (selected) docs' positions are ever used. With a sparse + // candidate set (the common phrase / phrase-prefix case after docid + // narrowing) most docs in a window are skipped, so this avoids the + // dominant varint-decode cost. + RETURN_IF_ERROR(src.skip_varints(pos_count)); + continue; + } + // Selected doc: decode its `pos_count` ascending position deltas with a + // tight inline prefix-sum decoder (single-byte fast path, no per-value + // get_varint32/Status call chain). This is the CPU hotspot for narrowed + // phrase/phrase-prefix candidate sets, where each selected doc's varint + // run dominates after non-selected docs are skipped. + RETURN_IF_ERROR(src.decode_delta_run(pos_count, pos_flat)); + pos_off->push_back(static_cast(pos_flat->size())); + ++next_doc; + } + if (!src.eof()) + return Status::Error( + "prx: trailing bytes after payload"); + return Status::OK(); +} + +// Decision: given level and plain length, determine whether to compress. +bool should_compress(int level, size_t plain_len) { + if (level == 0) return false; // force raw + if (level > 0) return true; // force zstd + return plain_len >= kAutoZstdMinBytes; // auto +} + +// Write a raw window: codec=raw, uncomp_len, crc(header+payload), payload. +void write_raw(Slice plain, ByteSink* sink) { + // Single-copy framing (see write_pfor): assemble [codec][uncomp_len][payload] + // in the caller's sink and crc that span before appending the crc. + const size_t start = sink->size(); + sink->put_u8(static_cast(PrxCodec::kRaw)); + sink->put_varint32(static_cast(plain.size())); + sink->put_bytes(plain); + const size_t framed_len = sink->size() - start; + const uint32_t crc = crc32c(sink->view().subslice(start, framed_len)); + sink->put_fixed32(crc); +} + +// Write a zstd window: codec=zstd, uncomp_len, comp_len, crc(header+payload), +// payload. +Status write_zstd(Slice plain, int level, ByteSink* sink) { + std::vector comp; + RETURN_IF_ERROR(zstd_compress(plain, level > 0 ? level : kDefaultZstdLevel, &comp)); + write_zstd_compressed(plain, Slice(comp), sink); + return Status::OK(); +} + +// Read header + payload, verify crc in retrospect, and return the payload view +// and uncomp_len to the caller. +Status read_framed(ByteSource* src, uint8_t* codec, uint32_t* uncomp_len, Slice* payload) { + size_t start = src->position(); + RETURN_IF_ERROR(src->get_u8(codec)); + if (*codec != static_cast(PrxCodec::kRaw) && + *codec != static_cast(PrxCodec::kZstd) && + *codec != static_cast(PrxCodec::kPfor)) { + return Status::Error("prx: unknown codec"); + } + RETURN_IF_ERROR(src->get_varint32(uncomp_len)); + if (*uncomp_len > kMaxWindowUncompBytes) { + return Status::Error( + "prx: uncomp_len exceeds sane window cap"); + } + size_t payload_len = *uncomp_len; + if (*codec == static_cast(PrxCodec::kZstd)) { + uint32_t comp_len = 0; + RETURN_IF_ERROR(src->get_varint32(&comp_len)); + payload_len = comp_len; + } + RETURN_IF_ERROR(src->get_bytes(payload_len, payload)); + size_t framed_len = src->position() - start; + uint32_t stored = 0; + RETURN_IF_ERROR(src->get_fixed32(&stored)); + if (crc32c(src->slice_from(start, framed_len)) != stored) { + return Status::Error( + "prx: window crc mismatch"); + } + return Status::OK(); +} + +} // namespace + +Status build_prx_window(std::span> per_doc_positions, + int zstd_level_or_negative_for_auto, ByteSink* sink) { + if (sink == nullptr) return Status::Error("prx: null sink"); + // Forced legacy codecs (level 0 = raw varint, level > 0 = zstd) are kept so + // the test/legacy paths still exercise them; the auto path (< 0) now emits + // PFOR bit-packed deltas -- no entropy coding, far cheaper build CPU than + // zstd-3. + if (zstd_level_or_negative_for_auto >= 0) { + ByteSink plain; + RETURN_IF_ERROR(encode_payload(per_doc_positions, &plain)); + Slice plain_view = plain.view(); + if (!should_compress(zstd_level_or_negative_for_auto, plain_view.size())) { + write_raw(plain_view, sink); + return Status::OK(); + } + return write_zstd(plain_view, zstd_level_or_negative_for_auto, sink); + } + // Auto mode: flatten the per-doc lists into (positions_flat, freqs) exactly as + // the former encode_pfor_payload did, then run the shared single-encode path so + // this builder stays byte-identical to build_prx_window_flat. + std::vector flat, freqs; + freqs.reserve(per_doc_positions.size()); + for (const auto& doc : per_doc_positions) { + freqs.push_back(static_cast(doc.size())); + flat.insert(flat.end(), doc.begin(), doc.end()); + } + // G16-h: level < -1 is auto mode at zstd level |level| (-1 stays the default). + const int auto_level = zstd_level_or_negative_for_auto == -1 ? kDefaultZstdLevel + : -zstd_level_or_negative_for_auto; + return build_prx_window_auto_from_flat(flat, freqs, auto_level, sink); +} + +Status build_prx_window_flat(std::span positions_flat, + std::span freqs, int zstd_level_or_negative_for_auto, + ByteSink* sink) { + if (sink == nullptr) return Status::Error("prx: null sink"); + if (zstd_level_or_negative_for_auto >= 0) { + ByteSink plain; + RETURN_IF_ERROR(encode_payload_flat(positions_flat, freqs, &plain)); + Slice plain_view = plain.view(); + if (!should_compress(zstd_level_or_negative_for_auto, plain_view.size())) { + write_raw(plain_view, sink); + return Status::OK(); + } + return write_zstd(plain_view, zstd_level_or_negative_for_auto, sink); + } + // Auto mode: single-encode path (shared with build_prx_window) -- derive deltas + // once, emit PFOR, and only materialize the raw plaintext when it is large + // enough to attempt zstd. Byte-identical to the prior double-encode. + // G16-h: level < -1 is auto mode at zstd level |level| (-1 stays the default). + const int auto_level = zstd_level_or_negative_for_auto == -1 ? kDefaultZstdLevel + : -zstd_level_or_negative_for_auto; + return build_prx_window_auto_from_flat(positions_flat, freqs, auto_level, sink); +} + +Status read_prx_window(ByteSource* source, std::vector>* per_doc_positions) { + if (source == nullptr || per_doc_positions == nullptr) { + return Status::Error("prx: null arg"); + } + uint8_t codec = 0; + uint32_t uncomp_len = 0; + Slice payload; + RETURN_IF_ERROR(read_framed(source, &codec, &uncomp_len, &payload)); + if (codec == static_cast(PrxCodec::kPfor)) { + return decode_pfor_payload(payload, per_doc_positions); + } + if (codec == static_cast(PrxCodec::kRaw)) { + return decode_payload(payload, per_doc_positions); + } + std::vector plain; + RETURN_IF_ERROR(zstd_decompress(payload, uncomp_len, &plain)); + return decode_payload(Slice(plain), per_doc_positions); +} + +Status read_prx_window_csr(ByteSource* source, std::vector* pos_flat, + std::vector* pos_off) { + if (source == nullptr || pos_flat == nullptr || pos_off == nullptr) { + return Status::Error("prx: null arg"); + } + uint8_t codec = 0; + uint32_t uncomp_len = 0; + Slice payload; + RETURN_IF_ERROR(read_framed(source, &codec, &uncomp_len, &payload)); + if (codec == static_cast(PrxCodec::kPfor)) { + return decode_pfor_payload_csr(payload, pos_flat, pos_off); + } + if (codec == static_cast(PrxCodec::kRaw)) { + return decode_payload_csr(payload, pos_flat, pos_off); + } + std::vector plain; + RETURN_IF_ERROR(zstd_decompress(payload, uncomp_len, &plain)); + return decode_payload_csr(Slice(plain), pos_flat, pos_off); +} + +Status read_prx_window_csr_selective(ByteSource* source, std::span doc_ordinals, + std::vector* pos_flat, + std::vector* pos_off) { + if (source == nullptr || pos_flat == nullptr || pos_off == nullptr) { + return Status::Error("prx: null arg"); + } + uint8_t codec = 0; + uint32_t uncomp_len = 0; + Slice payload; + RETURN_IF_ERROR(read_framed(source, &codec, &uncomp_len, &payload)); + if (codec == static_cast(PrxCodec::kPfor)) { + return decode_pfor_payload_csr_selective(payload, doc_ordinals, pos_flat, pos_off); + } + if (codec == static_cast(PrxCodec::kRaw)) { + return decode_payload_csr_selective(payload, doc_ordinals, pos_flat, pos_off); + } + std::vector plain; + RETURN_IF_ERROR(zstd_decompress(payload, uncomp_len, &plain)); + return decode_payload_csr_selective(Slice(plain), doc_ordinals, pos_flat, pos_off); +} + +} // namespace doris::snii::format + +namespace doris::snii::format::testing { +namespace { +std::atomic& prx_raw_build_atomic() { + static std::atomic counter {0}; + return counter; +} +} // namespace + +uint64_t prx_raw_build_count() { + return prx_raw_build_atomic().load(std::memory_order_relaxed); +} + +void reset_prx_raw_build_count() { + prx_raw_build_atomic().store(0, std::memory_order_relaxed); +} + +void note_prx_raw_build() { + prx_raw_build_atomic().fetch_add(1, std::memory_order_relaxed); +} + +} // namespace doris::snii::format::testing diff --git a/be/src/storage/index/snii/format/prx_pod.h b/be/src/storage/index/snii/format/prx_pod.h new file mode 100644 index 00000000000000..7c76b6ac7b2ebb --- /dev/null +++ b/be/src/storage/index/snii/format/prx_pod.h @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/byte_source.h" + +// .prx position window (PrxPod): stores term position information for several +// docs within one window. +// +// Single-window on-disk byte layout (see docs/design SNII "prx design"): +// u8 codec # PrxCodec: 0=raw / 1=zstd / 2=pfor (bit7 cont-reserved) +// VInt uncomp_len # payload length (raw/pfor: on-disk payload bytes; zstd: +// plaintext) VInt comp_len # present only when codec==zstd u32 crc32c # +// covers header (codec..comp_len) + payload bytes payload # raw: varint +// plaintext; zstd: compressed; pfor: bit-packed +// +// raw/zstd plaintext payload (self-describing per-doc boundaries): +// VInt doc_count +// per doc: VInt pos_count, followed by pos_count position deltas (VInt) +// positions within a doc are ascending, stored as deltas (first absolute). +// +// pfor payload (one auto-build candidate; no entropy coding): +// VInt doc_count +// VInt total_pos # sum of all pos_counts +// PFOR_runs(pos_counts) # doc_count values +// PFOR_runs(position_deltas) # total_pos deltas, kFrqBaseUnit per run, +// # flat doc order (first per doc +// absolute) +// +// Multi-byte fixed-length fields are little-endian; variable-length integers +// reuse snii/encoding/varint. crc32c checksum at window tail detects +// corruption. +namespace doris::snii::format { + +// Build a .prx window and append it to sink. +// per_doc_positions[d] is the position list for the d-th doc within this +// window; must be ascending (duplicates allowed). +// zstd_level_or_negative_for_auto: +// <0 → auto: use the smaller PFOR or ZSTD(default level) frame. +// 0 → force raw varint payload. +// >0 → force ZSTD with the given level. +// Non-ascending positions within a doc return InvalidArgument. +Status build_prx_window(std::span> per_doc_positions, + int zstd_level_or_negative_for_auto, ByteSink* sink); + +// Vector convenience overload (forwards a span view over the window's per-doc +// lists; the writer can pass a slice of its flat positions WITHOUT deep-copying +// the inner vectors into a fresh std::vector> per +// window). +inline Status build_prx_window(const std::vector>& per_doc_positions, + int zstd_level_or_negative_for_auto, ByteSink* sink) { + return build_prx_window(std::span>(per_doc_positions), + zstd_level_or_negative_for_auto, sink); +} + +// FLAT-positions builder: byte-identical output to build_prx_window above, but +// reads the window's positions from a single flat span partitioned per-doc by +// `freqs` (doc d owns the next freqs[d] entries; freqs.size() == doc count and +// sum(freqs) == positions_flat.size()). Lets the writer pass a subspan of the +// term's flat positions/freqs with NO vector-of-vectors materialization. +Status build_prx_window_flat(std::span positions_flat, + std::span freqs, int zstd_level_or_negative_for_auto, + ByteSink* sink); + +// Read and verify a .prx window from source, reconstructing the per-doc +// position list. CRC mismatch / invalid codec / truncation / decompression +// failure all return a non-OK Status. +Status read_prx_window(ByteSource* source, std::vector>* per_doc_positions); + +// CSR variant of read_prx_window: decodes ALL docs' positions into one flat +// buffer `pos_flat` with per-doc offsets `pos_off` (size doc_count+1, +// pos_off[0]==0), so doc d's positions are pos_flat[pos_off[d] .. +// pos_off[d+1]). Avoids the per-doc std::vector allocation of read_prx_window +// -- both output vectors are flat uint32 buffers whose capacity a caller can +// retain (clear()) across windows/queries. +Status read_prx_window_csr(ByteSource* source, std::vector* pos_flat, + std::vector* pos_off); + +// Selective CSR variant: decodes positions only for the requested local doc +// ordinals within this PRX window. `doc_ordinals` must be strictly ascending. +// The output uses the same CSR shape, but has doc_ordinals.size()+1 offsets. +Status read_prx_window_csr_selective(ByteSource* source, std::span doc_ordinals, + std::vector* pos_flat, + std::vector* pos_off); + +} // namespace doris::snii::format + +// Test-only instrumentation seam. prx_raw_build_count() returns a process-global +// count of raw plaintext payloads MATERIALIZED by the auto-mode (.prx) window +// builders -- i.e. the throwaway ByteSink the writer fills ONLY when the exact +// plaintext size reaches kAutoZstdMinBytes and a zstd attempt is therefore worth +// making. In auto mode the small windows that dominate a Zipfian corpus skip that +// materialization (and the second delta walk) entirely, so tests assert the count +// is 0 for sub-threshold windows and exactly 1 per materialized large window. +// Counters use relaxed atomics and are reset between tests; the writer's segment +// build is single-threaded, so the atomic adds introduce no data race. +namespace doris::snii::format::testing { + +uint64_t prx_raw_build_count(); +void reset_prx_raw_build_count(); +void note_prx_raw_build(); + +} // namespace doris::snii::format::testing diff --git a/be/src/storage/index/snii/format/sampled_term_index.cpp b/be/src/storage/index/snii/format/sampled_term_index.cpp new file mode 100644 index 00000000000000..3855d6d3a019c5 --- /dev/null +++ b/be/src/storage/index/snii/format/sampled_term_index.cpp @@ -0,0 +1,185 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/sampled_term_index.h" + +#include + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/section_framer.h" + +namespace doris::snii::format { + +namespace { + +// Longest common prefix length of term and prev (front coding primitive, consistent with dict_entry). +uint32_t common_prefix_len(std::string_view term, std::string_view prev) { + uint32_t n = 0; + const uint32_t lim = static_cast(std::min(term.size(), prev.size())); + while (n < lim && term[n] == prev[n]) ++n; + return n; +} + +// Write a front-coded term key (prefix_len + suffix_len + suffix). +void write_term_key(std::string_view term, std::string_view prev, ByteSink* sink) { + const uint32_t prefix = common_prefix_len(term, prev); + const std::string_view suffix = term.substr(prefix); + sink->put_varint32(prefix); + sink->put_varint32(static_cast(suffix.size())); + sink->put_bytes(Slice(suffix)); +} + +// Read a front-coded term key and reconstruct it into out from prev + suffix. +Status read_term_key(ByteSource* src, std::string_view prev, std::string* out) { + uint32_t prefix = 0; + uint32_t suffix_len = 0; + RETURN_IF_ERROR(src->get_varint32(&prefix)); + RETURN_IF_ERROR(src->get_varint32(&suffix_len)); + if (prefix > prev.size()) { + return Status::Error( + "sampled_term_index: prefix_len exceeds prev_term length"); + } + Slice suffix; + RETURN_IF_ERROR(src->get_bytes(suffix_len, &suffix)); + out->assign(prev.substr(0, prefix)); + out->append(reinterpret_cast(suffix.data()), suffix.size()); + return Status::OK(); +} + +} // namespace + +void SampledTermIndexBuilder::add_block_first_term(std::string_view first_term) { + first_terms_.emplace_back(first_term); +} + +void SampledTermIndexBuilder::finish(ByteSink* sink) { + ByteSink payload; + payload.put_varint32(static_cast(first_terms_.size())); + // min_term / max_term are written only when non-empty (== first/last sample_term). + if (!first_terms_.empty()) { + write_term_key(first_terms_.front(), std::string_view {}, &payload); + write_term_key(first_terms_.back(), std::string_view {}, &payload); + std::string_view prev {}; + for (const auto& t : first_terms_) { + write_term_key(t, prev, &payload); + prev = t; + } + } + SectionFramer::write(*sink, static_cast(SectionType::kSampledTermIndex), + payload.view()); +} + +namespace { + +// Parse n_blocks, min/max (not used directly; consumed for checksum alignment), and all sample_terms from payload. +Status parse_payload(Slice payload, std::vector* terms) { + ByteSource src(payload); + uint32_t n_blocks = 0; + RETURN_IF_ERROR(src.get_varint32(&n_blocks)); + if (n_blocks == 0) { + if (!src.eof()) { + return Status::Error( + "sampled_term_index: empty index contains trailing bytes"); + } + terms->clear(); + return Status::OK(); + } + + // min_term / max_term (do not drive binary search directly; must be consumed to verify structural alignment). + std::string min_term; + std::string max_term; + RETURN_IF_ERROR(read_term_key(&src, std::string_view {}, &min_term)); + RETURN_IF_ERROR(read_term_key(&src, std::string_view {}, &max_term)); + + std::vector out; + out.reserve(n_blocks); + std::string prev; + for (uint32_t i = 0; i < n_blocks; ++i) { + std::string term; + RETURN_IF_ERROR(read_term_key(&src, prev, &term)); + prev = term; + out.push_back(std::move(term)); + } + if (!src.eof()) { + return Status::Error( + "sampled_term_index: payload contains trailing bytes"); + } + if (out.front() != min_term || out.back() != max_term) { + return Status::Error( + "sampled_term_index: min/max inconsistent with sample_terms"); + } + *terms = std::move(out); + return Status::OK(); +} + +} // namespace + +Status SampledTermIndexReader::open(Slice section, SampledTermIndexReader* out) { + if (out == nullptr) { + return Status::Error("sampled_term_index: out is null"); + } + ByteSource src(section); + FramedSection sec; + RETURN_IF_ERROR(SectionFramer::read(src, &sec)); + if (sec.type != static_cast(SectionType::kSampledTermIndex)) { + return Status::Error( + "sampled_term_index: not a kSampledTermIndex section"); + } + *out = SampledTermIndexReader {}; + return parse_payload(sec.payload, &out->sample_terms_); +} + +size_t SampledTermIndexReader::heap_bytes() const { + size_t bytes = sample_terms_.capacity() * sizeof(std::string); + for (const auto& term : sample_terms_) { + bytes += std_string_heap_bytes(term); + } + return bytes; +} + +Status SampledTermIndexReader::locate(std::string_view target, bool* maybe_present, + uint32_t* block_ordinal) const { + if (maybe_present == nullptr || block_ordinal == nullptr) { + return Status::Error( + "sampled_term_index: output pointer is null"); + } + *maybe_present = false; + *block_ordinal = 0; + if (sample_terms_.empty()) { + return Status::OK(); // empty index: always out of range. + } + // target < min_term (first block's first term) -> before the first block, so it + // cannot exist in any block. NOTE: a target GREATER than the last sample term is + // NOT out of range -- sample_terms_ holds each block's FIRST term, so the LAST + // block can contain terms greater than its first term. Such a target routes to + // the last block (upper_bound -> end()), where find_term confirms presence. + if (target < std::string_view(sample_terms_.front())) { + return Status::OK(); + } + // Last sample_term <= target: step back one position after upper_bound. For a + // target past every sample term, upper_bound returns end() and idx = n-1 (the + // last block), which is correct. + auto it = std::upper_bound( + sample_terms_.begin(), sample_terms_.end(), target, + [](std::string_view t, const std::string& s) { return t < std::string_view(s); }); + const auto idx = (it - sample_terms_.begin()) - 1; // it > begin (< min excluded). + *maybe_present = true; + *block_ordinal = static_cast(idx); + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/sampled_term_index.h b/be/src/storage/index/snii/format/sampled_term_index.h new file mode 100644 index 00000000000000..f2d63af88334e8 --- /dev/null +++ b/be/src/storage/index/snii/format/sampled_term_index.h @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/format_constants.h" + +// SampledTermIndex -- resident metadata for locating a query term to a candidate DICT block. +// +// Sampling granularity is per DICT block (not a fixed term count): each time the writer produces a DICT block, +// it writes the block's first_term into this index. Size grows proportionally to block count. At read time it is +// loaded into the searcher cache together with SniiLogicalIndexReader. See design spec "Sampled Term Index". +// +// On-disk layout (framed by SectionFramer, uniform type+len+crc32c): +// [u8 type=kSampledTermIndex][varint64 payload_len][payload][fixed32 crc32c] +// payload = +// n_blocks varint32 +// min_term len(varint32) + bytes # == sample_terms[0], omitted when n_blocks=0 +// max_term len(varint32) + bytes # == sample_terms[n-1], omitted when n_blocks=0 +// sample_terms[n_blocks]: # first_term of each block, in ascending order +// prefix_len varint32 # shared prefix length with the previous sample_term +// suffix_len varint32 +// suffix u8[suffix_len] +// +// Term bytes are compared as unsigned byte order (UTF-8 friendly, binary-safe). Front coding reuses +// the same prefix/suffix primitives as DictEntry; do not reimplement. +namespace doris::snii::format { + +// SSO-aware heap-byte accounting for a std::string. libstdc++ keeps up to 15 +// chars inline (SSO), so only capacity() > 15 implies a separate heap buffer of +// capacity()+1 bytes (the +1 is the NUL terminator); an SSO string owns no heap +// and contributes 0. Shared by the resident format readers' heap_bytes() charge +// helpers, which back LogicalIndexReader::memory_usage() (the searcher-cache +// charge). NOTE: the threshold 15 is libstdc++-specific; a different standard +// library needs a different SSO bound here. +inline size_t std_string_heap_bytes(const std::string& s) { + return s.capacity() > 15 ? s.capacity() + 1 : 0; +} + +// Builder: appends the first_term of each DICT block in block ordinal order (must be strictly ascending), +// and serializes the entire set into a single kSampledTermIndex framed section on finish. +class SampledTermIndexBuilder { +public: + // Appends the first_term of the next DICT block. Call order determines block ordinal order. + void add_block_first_term(std::string_view first_term); + + // Serializes and appends to sink. An empty collection (no blocks) is valid; n_blocks=0. + void finish(ByteSink* sink); + +private: + std::vector first_terms_; +}; + +// Reader: verifies the checksum and materializes all sample_terms on open; subsequent locate calls are pure in-memory binary search. +class SampledTermIndexReader { +public: + SampledTermIndexReader() = default; + + // Parses a kSampledTermIndex framed section. + // CRC mismatch / truncation / field overrun → kCorruption; type != kSampledTermIndex → kInvalidArgument. + static Status open(Slice section, SampledTermIndexReader* out); + + // Binary-search locate: returns the block ordinal of the last sample_term <= target. + // target < min_term or target > max_term (including empty index) → *maybe_present=false (out of range, term is definitely absent). + // Otherwise *maybe_present=true and *block_ordinal is the ordinal of the matching block. + Status locate(std::string_view target, bool* maybe_present, uint32_t* block_ordinal) const; + + uint32_t n_blocks() const { return static_cast(sample_terms_.size()); } + + // Resident heap held beyond sizeof(*this): the sample_terms_ vector buffer + // plus each non-SSO term's heap allocation. Summed into + // LogicalIndexReader::memory_usage() so the searcher-cache charge reflects the + // decoded sampled index (previously omitted -> under-charge -> over-commit). + size_t heap_bytes() const; + +private: + std::vector sample_terms_; +}; + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/stats_block.cpp b/be/src/storage/index/snii/format/stats_block.cpp new file mode 100644 index 00000000000000..0acd1c02c83b5c --- /dev/null +++ b/be/src/storage/index/snii/format/stats_block.cpp @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/stats_block.h" + +namespace doris::snii::format { + +namespace { + +// Field order within payload is fixed; reuse ByteSink varint primitives — do not hand-assemble bytes. +void encode_payload(const StatsBlock& sb, ByteSink* payload) { + payload->put_varint64(sb.doc_count); + payload->put_varint64(sb.indexed_doc_count); + payload->put_varint64(sb.term_count); + payload->put_varint64(sb.sum_total_term_freq); + payload->put_varint64(sb.null_count); +} + +Status decode_payload(Slice payload, StatsBlock* out) { + ByteSource ps(payload); + RETURN_IF_ERROR(ps.get_varint64(&out->doc_count)); + RETURN_IF_ERROR(ps.get_varint64(&out->indexed_doc_count)); + RETURN_IF_ERROR(ps.get_varint64(&out->term_count)); + RETURN_IF_ERROR(ps.get_varint64(&out->sum_total_term_freq)); + RETURN_IF_ERROR(ps.get_varint64(&out->null_count)); + if (!ps.eof()) { + return Status::Error( + "stats_block: trailing bytes in payload"); + } + return Status::OK(); +} + +} // namespace + +void encode_stats_block(const StatsBlock& sb, ByteSink* sink) { + ByteSink payload; + encode_payload(sb, &payload); + SectionFramer::write(*sink, static_cast(SectionType::kStatsBlock), payload.view()); +} + +Status decode_stats_block(ByteSource* src, StatsBlock* out) { + FramedSection sec; + RETURN_IF_ERROR(SectionFramer::read(*src, &sec)); + if (sec.type != static_cast(SectionType::kStatsBlock)) { + return Status::Error( + "stats_block: unexpected section type"); + } + return decode_payload(sec.payload, out); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/stats_block.h b/be/src/storage/index/snii/format/stats_block.h new file mode 100644 index 00000000000000..06fd34425e4d88 --- /dev/null +++ b/be/src/storage/index/snii/format/stats_block.h @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/section_framer.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::format { + +// Statistics block within the per-index meta block. Carries only the counting stats +// needed for query planning and BM25; section location info is stored separately in SectionRefs (see design spec "Per-index meta block"). +// +// On-disk layout (framed by SectionFramer with unified type+len+crc32c): +// [u8 type=kStatsBlock][varint64 payload_len][payload][fixed32 crc32c] +// payload = varint64{ doc_count, indexed_doc_count, term_count, +// sum_total_term_freq, null_count } +// For field semantics see design spec "Scoring statistics design". +struct StatsBlock { + uint64_t doc_count = 0; // total doc count at segment level (including unindexed/NULL) + uint64_t indexed_doc_count = 0; // number of docs actually indexed (denominator for avgdl) + uint64_t term_count = 0; // number of unique terms in this index + uint64_t sum_total_term_freq = 0; // total token count across all indexed docs + uint64_t null_count = 0; // number of NULL / not-indexed docs +}; + +// Encodes into a kStatsBlock framed section (with built-in crc32c checksum) and appends to sink. +void encode_stats_block(const StatsBlock& sb, ByteSink* sink); + +// Reads and verifies a kStatsBlock framed section from src, populates out. +// CRC mismatch / truncation → kCorruption; type is not kStatsBlock → kInvalidArgument. +Status decode_stats_block(ByteSource* src, StatsBlock* out); + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/tail_meta_region.cpp b/be/src/storage/index/snii/format/tail_meta_region.cpp new file mode 100644 index 00000000000000..ba23c204a1cdff --- /dev/null +++ b/be/src/storage/index/snii/format/tail_meta_region.cpp @@ -0,0 +1,213 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/tail_meta_region.h" + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::format { +namespace { + +// Header field bytes (before header_crc): u32 ver + u32 flags + u64 meta_region_len +// + u32 n + u64 directory_offset + u64 directory_length. +constexpr size_t kHeaderFields = 4 + 4 + 8 + 4 + 8 + 8; // 36 +constexpr size_t kHeaderSize = kHeaderFields + 4; // + header_crc32c +constexpr size_t kRegionChecksumSize = 4; + +} // namespace + +size_t tail_meta_header_size() { + return kHeaderSize; +} + +void TailMetaRegionBuilder::add_index(uint64_t index_id, std::string index_suffix, + Slice per_index_meta_bytes) { + Entry e; + e.index_id = index_id; + e.suffix = std::move(index_suffix); + e.bytes.assign(per_index_meta_bytes.data(), + per_index_meta_bytes.data() + per_index_meta_bytes.size()); + entries_.push_back(std::move(e)); +} + +void TailMetaRegionBuilder::finish(ByteSink* sink) const { + // Lay out per-index meta blocks right after the header; build the directory + // with each block's in-region offset/length. + LogicalIndexDirectoryBuilder dir; + uint64_t offset = kHeaderSize; + for (const Entry& e : entries_) { + LogicalIndexRef ref; + ref.index_id = e.index_id; + ref.index_suffix = e.suffix; + ref.meta_off = offset; + ref.meta_len = e.bytes.size(); + dir.add(ref); + offset += e.bytes.size(); + } + const uint64_t directory_offset = offset; + ByteSink dir_bytes; + dir.finish(&dir_bytes); + const uint64_t directory_length = dir_bytes.size(); + const uint64_t meta_region_len = directory_offset + directory_length + kRegionChecksumSize; + + ByteSink fields; + fields.put_fixed32(kMetaFormatVersion); + fields.put_fixed32(0); // flags + fields.put_fixed64(meta_region_len); + fields.put_fixed32(static_cast(entries_.size())); + fields.put_fixed64(directory_offset); + fields.put_fixed64(directory_length); + + ByteSink region; + region.put_bytes(fields.view()); + region.put_fixed32(crc32c(fields.view())); // header_crc32c + for (const Entry& e : entries_) region.put_bytes(Slice(e.bytes)); + region.put_bytes(dir_bytes.view()); + region.put_fixed32(crc32c(region.view())); // meta_region_checksum + + sink->put_bytes(region.view()); +} + +Status TailMetaRegionReader::parse_header(Slice header, TailMetaRegionHeader* const out) { + if (out == nullptr) { + return Status::Error( + "tail_meta_region: null header out"); + } + if (header.size() != kHeaderSize) { + return Status::Error( + "tail_meta_region: header size mismatch"); + } + ByteSource hs(header.subslice(0, kHeaderFields)); + uint32_t ver = 0, flags = 0, n = 0; + uint64_t meta_region_len = 0, directory_offset = 0, directory_length = 0; + RETURN_IF_ERROR(hs.get_fixed32(&ver)); + RETURN_IF_ERROR(hs.get_fixed32(&flags)); + RETURN_IF_ERROR(hs.get_fixed64(&meta_region_len)); + RETURN_IF_ERROR(hs.get_fixed32(&n)); + RETURN_IF_ERROR(hs.get_fixed64(&directory_offset)); + RETURN_IF_ERROR(hs.get_fixed64(&directory_length)); + ByteSource hc(header.subslice(kHeaderFields, 4)); + uint32_t header_crc = 0; + RETURN_IF_ERROR(hc.get_fixed32(&header_crc)); + if (crc32c(header.subslice(0, kHeaderFields)) != header_crc) { + return Status::Error( + "tail_meta_region: header crc mismatch"); + } + if (ver != kMetaFormatVersion) { + return Status::Error( + "tail_meta_region: unsupported meta_format_version"); + } + if (flags != 0) { + return Status::Error( + "tail_meta_region: unsupported flags"); + } + if (meta_region_len < kHeaderSize + kRegionChecksumSize) { + return Status::Error( + "tail_meta_region: declared length too small"); + } + if (directory_offset < kHeaderSize || directory_offset > meta_region_len || + directory_length > meta_region_len - directory_offset) { + return Status::Error( + "tail_meta_region: directory out of range"); + } + out->meta_region_len = meta_region_len; + out->directory_offset = directory_offset; + out->directory_length = directory_length; + out->n_logical_indexes = n; + return Status::OK(); +} + +Status TailMetaRegionReader::open_directory(const TailMetaRegionHeader& header, Slice directory, + TailMetaRegionReader* const out) { + if (out == nullptr) { + return Status::Error("tail_meta_region: null out"); + } + if (directory.size() != header.directory_length) { + return Status::Error( + "tail_meta_region: directory length mismatch"); + } + + RETURN_IF_ERROR(LogicalIndexDirectoryReader::open(directory, &out->dir_)); + if (out->dir_.size() != header.n_logical_indexes) { + return Status::Error( + "tail_meta_region: directory size mismatch"); + } + out->region_ = Slice {}; + out->meta_region_len_ = header.meta_region_len; + out->n_ = header.n_logical_indexes; + return Status::OK(); +} + +Status TailMetaRegionReader::open(Slice region, TailMetaRegionReader* const out) { + if (out == nullptr) { + return Status::Error("tail_meta_region: null out"); + } + if (region.size() < kHeaderSize + kRegionChecksumSize) { + return Status::Error( + "tail_meta_region: region too short"); + } + + // Verify the trailing region checksum. + const size_t covered = region.size() - kRegionChecksumSize; + ByteSource cs(region.subslice(covered, kRegionChecksumSize)); + uint32_t region_crc = 0; + RETURN_IF_ERROR(cs.get_fixed32(®ion_crc)); + if (crc32c(region.subslice(0, covered)) != region_crc) { + return Status::Error( + "tail_meta_region: meta_region_checksum mismatch"); + } + + TailMetaRegionHeader header; + RETURN_IF_ERROR(parse_header(region.subslice(0, kHeaderSize), &header)); + if (header.meta_region_len != region.size()) { + return Status::Error( + "tail_meta_region: declared length mismatch"); + } + RETURN_IF_ERROR(open_directory( + header, region.subslice(header.directory_offset, header.directory_length), out)); + out->region_ = region; + return Status::OK(); +} + +Status TailMetaRegionReader::find_ref(uint64_t index_id, std::string_view suffix, bool* const found, + LogicalIndexRef* const ref) const { + return dir_.find(index_id, suffix, found, ref); +} + +Status TailMetaRegionReader::find(uint64_t index_id, std::string_view suffix, bool* const found, + Slice* const per_index_meta_bytes) const { + LogicalIndexRef ref; + RETURN_IF_ERROR(find_ref(index_id, suffix, found, &ref)); + if (!*found) { + return Status::OK(); + } + if (region_.empty()) { + return Status::Error( + "tail_meta_region: region bytes not resident"); + } + if (ref.meta_off > region_.size() || ref.meta_len > region_.size() - ref.meta_off || + ref.meta_off > meta_region_len_ || ref.meta_len > meta_region_len_ - ref.meta_off) { + return Status::Error( + "tail_meta_region: meta block out of range"); + } + *per_index_meta_bytes = region_.subslice(ref.meta_off, ref.meta_len); + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/tail_meta_region.h b/be/src/storage/index/snii/format/tail_meta_region.h new file mode 100644 index 00000000000000..524ef55aff1c86 --- /dev/null +++ b/be/src/storage/index/snii/format/tail_meta_region.h @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/logical_index_directory.h" + +namespace doris::snii::format { + +struct TailMetaRegionHeader { + uint64_t meta_region_len = 0; + uint64_t directory_offset = 0; + uint64_t directory_length = 0; + uint32_t n_logical_indexes = 0; +}; + +size_t tail_meta_header_size(); + +// TailMetaRegion: the container's tail metadata region, located via the fixed +// tail pointer and read in one range. It bundles the per-logical-index meta +// blocks and the logical index directory so a reader can, after a single read, +// map (index_id, index_suffix) -> per-index meta block. See spec "footer meta +// region". +// +// On-disk layout (offsets are relative to the region start; the region is read +// whole into memory, so internal refs need not be file-absolute): +// TailMetaHeader: +// u32 meta_format_version (== kMetaFormatVersion) +// u32 flags +// u64 meta_region_len (== total region byte length) +// u32 n_logical_indexes +// u64 directory_offset (offset of the logical index directory in-region) +// u64 directory_length +// u32 header_crc32c (covers the header fields above) +// [per-index meta block #0][per-index meta block #1]... (opaque payloads) +// [logical index directory] (framed via LogicalIndexDirectory) +// u32 meta_region_checksum (crc32c over everything before it) +class TailMetaRegionBuilder { +public: + // Adds a per-index meta block (already serialized by PerIndexMetaBuilder) keyed + // by (index_id, index_suffix). Bytes are copied. + void add_index(uint64_t index_id, std::string index_suffix, Slice per_index_meta_bytes); + + // Serializes the whole region and appends it to sink. + void finish(ByteSink* sink) const; + +private: + struct Entry { + uint64_t index_id; + std::string suffix; + std::vector bytes; + }; + std::vector entries_; +}; + +class TailMetaRegionReader { +public: + TailMetaRegionReader() = default; + + // Parses and validates only the fixed tail-meta header. This is used by the + // lazy segment open path to locate the directory without reading every + // per-index meta block. + static Status parse_header(Slice header, TailMetaRegionHeader* const out); + + // Parses and validates the region (header crc + region checksum + directory). + // region must outlive this reader (find() returns sub-views of it). + static Status open(Slice region, TailMetaRegionReader* const out); + + // Opens a reader from an already parsed header and a standalone framed + // logical-index directory. This intentionally does not verify the whole-region + // checksum because callers have not read the whole region; each per-index meta + // block still carries its own framed-section CRCs. + static Status open_directory(const TailMetaRegionHeader& header, Slice directory, + TailMetaRegionReader* const out); + + uint32_t n_logical_indexes() const { return n_; } + const LogicalIndexDirectoryReader& directory() const { return dir_; } + + Status find_ref(uint64_t index_id, std::string_view suffix, bool* const found, + LogicalIndexRef* const ref) const; + + // Locates the per-index meta block bytes for (index_id, suffix). On match, + // *found=true and *per_index_meta_bytes views into the region; else *found=false. + Status find(uint64_t index_id, std::string_view suffix, bool* const found, + Slice* const per_index_meta_bytes) const; + +private: + Slice region_; + LogicalIndexDirectoryReader dir_; + uint64_t meta_region_len_ = 0; + uint32_t n_ = 0; +}; + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/tail_pointer.cpp b/be/src/storage/index/snii/format/tail_pointer.cpp new file mode 100644 index 00000000000000..2c180e799ce23b --- /dev/null +++ b/be/src/storage/index/snii/format/tail_pointer.cpp @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/tail_pointer.h" + +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::format { + +namespace { + +// Byte widths of every fixed field, used to derive the constant on-disk size: +// u32 magic + u16 version + 3*u64 + 2*u32 + u8 size + u32 tail_checksum. +constexpr size_t kMagicBytes = 4; +constexpr size_t kVersionBytes = 2; +constexpr size_t kU64Bytes = 8; +constexpr size_t kU32Bytes = 4; +constexpr size_t kSizeByteBytes = 1; + +constexpr size_t kFixedSize = + kMagicBytes + kVersionBytes + 3 * kU64Bytes + 2 * kU32Bytes + kSizeByteBytes + kU32Bytes; +// tail_checksum is the trailing u32 and covers every byte before it. +constexpr size_t kChecksumCoverage = kFixedSize - kU32Bytes; + +// Serializes the checksum-covered region in fixed field order into covered. +void serialize_covered(const TailPointer& tp, ByteSink* covered) { + covered->put_fixed32(kTailMagic); + covered->put_fixed16(kFormatVersion); + covered->put_fixed64(tp.meta_region_offset); + covered->put_fixed64(tp.meta_region_length); + covered->put_fixed64(tp.hot_off); + covered->put_fixed32(tp.meta_region_checksum); + covered->put_fixed32(tp.bootstrap_header_checksum); + covered->put_u8(static_cast(kFixedSize)); +} + +} // namespace + +size_t tail_pointer_size() { + return kFixedSize; +} + +Status encode_tail_pointer(const TailPointer& tp, ByteSink* sink) { + ByteSink covered; + serialize_covered(tp, &covered); + if (covered.size() != kChecksumCoverage) { + return Status::Error( + "tail_pointer: covered size mismatch"); + } + const uint32_t tail_checksum = crc32c(covered.view()); + sink->put_bytes(covered.view()); + sink->put_fixed32(tail_checksum); + return Status::OK(); +} + +Status decode_tail_pointer(Slice last_bytes, TailPointer* out) { + // Anti-DoS / framing: the tail pointer is a fixed-size footer, so reject any + // input that is not exactly the fixed size before touching its contents. + if (last_bytes.size() != kFixedSize) { + return Status::Error( + "tail_pointer: input is not the fixed size"); + } + // Verify the trailing tail_checksum over the covered region first; a mismatch + // means any parsed field would be untrustworthy. + const Slice covered = last_bytes.subslice(0, kChecksumCoverage); + ByteSource src(last_bytes); + + uint32_t magic = 0; + RETURN_IF_ERROR(src.get_fixed32(&magic)); + if (magic != kTailMagic) { + return Status::Error( + "tail_pointer: bad magic"); + } + + uint16_t tail_format_version = 0; + RETURN_IF_ERROR(src.get_fixed16(&tail_format_version)); + if (tail_format_version != kFormatVersion) { + return Status::Error( + "tail_pointer: unsupported container format_version"); + } + RETURN_IF_ERROR(src.get_fixed64(&out->meta_region_offset)); + RETURN_IF_ERROR(src.get_fixed64(&out->meta_region_length)); + RETURN_IF_ERROR(src.get_fixed64(&out->hot_off)); + RETURN_IF_ERROR(src.get_fixed32(&out->meta_region_checksum)); + RETURN_IF_ERROR(src.get_fixed32(&out->bootstrap_header_checksum)); + + uint8_t on_disk_size = 0; + RETURN_IF_ERROR(src.get_u8(&on_disk_size)); + if (on_disk_size != kFixedSize) { + return Status::Error( + "tail_pointer: embedded size mismatch"); + } + + uint32_t tail_checksum = 0; + RETURN_IF_ERROR(src.get_fixed32(&tail_checksum)); + if (tail_checksum != crc32c(covered)) { + return Status::Error( + "tail_pointer: tail_checksum mismatch"); + } + return Status::OK(); +} + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/tail_pointer.h b/be/src/storage/index/snii/format/tail_pointer.h new file mode 100644 index 00000000000000..d72ac415c7600c --- /dev/null +++ b/be/src/storage/index/snii/format/tail_pointer.h @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +namespace doris::snii::format { + +// Fixed-size entry written at the very end of a segment's .idx file. It lets a +// reader locate the tail meta region with a single read of the trailing +// tail_pointer_size() bytes (see design spec "fixed tail pointer"). +// +// On-disk layout (all multi-byte fields little-endian, FIXED total size so the +// reader can read exactly the last tail_pointer_size() bytes): +// [u32 magic = kTailMagic] +// [u16 format_version = kFormatVersion] +// [u64 meta_region_offset] +// [u64 meta_region_length] +// [u64 hot_off] (offset of the hot region [hot_off, EOF); +// 0 if absent) +// [u32 meta_region_checksum] +// [u32 bootstrap_header_checksum] +// [u8 tail_pointer_size] (== tail_pointer_size()) +// [u32 tail_checksum] (crc32c over all preceding tail-pointer bytes) +// +// The fixed layout deliberately does NOT use the SectionFramer (which is +// variable-length): a footer needs a constant trailing size the reader knows up +// front. +struct TailPointer { + uint64_t meta_region_offset = 0; + uint64_t meta_region_length = 0; + uint64_t hot_off = 0; + uint32_t meta_region_checksum = 0; + uint32_t bootstrap_header_checksum = 0; +}; + +// Constant on-disk size of the tail pointer, so the reader knows how many +// trailing bytes to read. +size_t tail_pointer_size(); + +// Appends the fixed-layout tail-pointer bytes (magic / version / fields / size / +// tail_checksum) to sink. Returns Internal if the encoded size would not fit the +// fixed-size contract (a programming error, never expected at runtime). +Status encode_tail_pointer(const TailPointer& tp, ByteSink* sink); + +// Parses the trailing tail-pointer bytes. last_bytes must be exactly +// tail_pointer_size() bytes long. Verifies magic and tail_checksum, then fills +// out with the parsed fields. Wrong magic / checksum mismatch / wrong length -> +// Corruption. +Status decode_tail_pointer(Slice last_bytes, TailPointer* out); + +} // namespace doris::snii::format diff --git a/be/src/storage/index/snii/io/batch_range_fetcher.cpp b/be/src/storage/index/snii/io/batch_range_fetcher.cpp new file mode 100644 index 00000000000000..762c01d1c78024 --- /dev/null +++ b/be/src/storage/index/snii/io/batch_range_fetcher.cpp @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/io/batch_range_fetcher.h" + +#include +#include + +namespace doris::snii::io { +namespace { + +Status checked_end(uint64_t offset, uint64_t len, uint64_t* out) { + if (len > std::numeric_limits::max() - offset) { + return Status::Error( + "batch_range_fetcher: range end overflow"); + } + *out = offset + len; + return Status::OK(); +} + +Status checked_size(uint64_t len, size_t* out) { + if (len > static_cast(std::numeric_limits::max())) { + return Status::Error( + "batch_range_fetcher: physical range too large"); + } + *out = static_cast(len); + return Status::OK(); +} + +} // namespace + +BatchRangeFetcher::BatchRangeFetcher(FileReader* reader, uint64_t coalesce_gap) + : reader_(reader), coalesce_gap_(coalesce_gap) {} + +size_t BatchRangeFetcher::add(uint64_t offset, uint64_t len) { + reqs_.push_back(Req {offset, len}); + return reqs_.size() - 1; +} + +void BatchRangeFetcher::clear() { + reqs_.clear(); + phys_.clear(); +} + +Status BatchRangeFetcher::fetch() { + if (reader_ == nullptr) + return Status::Error( + "batch_range_fetcher: null reader"); + phys_.clear(); + if (reqs_.empty()) return Status::OK(); + + std::vector order(reqs_.size()); + for (size_t i = 0; i < order.size(); ++i) order[i] = i; + std::sort(order.begin(), order.end(), + [&](size_t a, size_t b) { return reqs_[a].offset < reqs_[b].offset; }); + + // Sweep in offset order, merging requests into physical segments. + std::vector segs; + uint64_t cur_start = 0; + uint64_t cur_end = 0; + for (size_t k = 0; k < order.size(); ++k) { + Req& r = reqs_[order[k]]; + uint64_t r_end = 0; + RETURN_IF_ERROR(checked_end(r.offset, r.len, &r_end)); + RETURN_IF_ERROR(checked_size(r.len, &r.len_size)); + const bool disjoint = r.offset > cur_end && r.offset - cur_end > coalesce_gap_; + if (segs.empty() || disjoint) { + segs.push_back(Range {r.offset, 0}); // length finalized below + cur_start = r.offset; + cur_end = r_end; + } else { + cur_end = std::max(cur_end, r_end); + } + r.phys_idx = segs.size() - 1; + RETURN_IF_ERROR(checked_size(r.offset - cur_start, &r.sub_offset)); + RETURN_IF_ERROR(checked_size(cur_end - cur_start, &segs.back().len)); + } + + return reader_->read_batch(segs, &phys_); +} + +Slice BatchRangeFetcher::get(size_t h) const { + const Req& r = reqs_[h]; + const std::vector& buf = phys_[r.phys_idx]; + return Slice(buf.data() + r.sub_offset, r.len_size); +} + +} // namespace doris::snii::io diff --git a/be/src/storage/index/snii/io/batch_range_fetcher.h b/be/src/storage/index/snii/io/batch_range_fetcher.h new file mode 100644 index 00000000000000..31218d1eb32e08 --- /dev/null +++ b/be/src/storage/index/snii/io/batch_range_fetcher.h @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/io/file_reader.h" + +namespace doris::snii::io { + +// Collects the byte ranges a query plan needs, coalesces overlapping/adjacent +// ranges into physical reads, and fetches them in a single batch (one serial +// I/O round on a MeteredFileReader). Callers retrieve each requested range by +// the handle returned from add(). This is the SNII read path's batching layer: +// it front-loads range planning so reads are issued concurrently rather than +// cursor-by-cursor. +class BatchRangeFetcher { +public: + // coalesce_gap: requests separated by a gap <= this many bytes are merged into + // one physical read (reads a few extra bytes to save a request). 0 merges only + // overlapping/adjacent ranges. + explicit BatchRangeFetcher(FileReader* reader, uint64_t coalesce_gap = 0); + + // Registers a desired range; returns a handle usable with get() after fetch(). + size_t add(uint64_t offset, uint64_t len); + + // Coalesces and issues one batched read; fills internal buffers. + Status fetch(); + + // Bytes for handle h (valid only after a successful fetch(), until clear()). + Slice get(size_t h) const; + + size_t pending() const { return reqs_.size(); } + void clear(); + +private: + struct Req { + uint64_t offset; + uint64_t len; + size_t len_size = 0; // validated size_t length after successful fetch() + size_t phys_idx = 0; // index into phys_ after fetch + size_t sub_offset = 0; // byte offset of this req within its physical read + }; + + FileReader* reader_; + uint64_t coalesce_gap_; + std::vector reqs_; + std::vector> phys_; // physical read buffers after fetch +}; + +} // namespace doris::snii::io diff --git a/be/src/storage/index/snii/io/file_reader.h b/be/src/storage/index/snii/io/file_reader.h new file mode 100644 index 00000000000000..2e1c0177afaa9f --- /dev/null +++ b/be/src/storage/index/snii/io/file_reader.h @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/io/io_metrics.h" + +namespace doris::snii::io { + +// One logical read request (offset, length). +struct Range { + uint64_t offset = 0; + size_t len = 0; +}; + +// The single physical-read primitive (a BE-internal read_at). All higher layers +// route reads through this so I/O can be accounted and backed by local files or +// object storage interchangeably. +class FileReader { +public: + virtual ~FileReader() = default; + + // Reads exactly len bytes starting at offset into *out (which is resized to + // len). Reading past EOF is an error (Corruption/IoError). + virtual Status read_at(uint64_t offset, size_t len, std::vector* out) = 0; + + // Reads a batch of ranges that may be served concurrently. The default is a + // sequential loop; backends that model concurrency (MeteredFileReader) or + // perform real parallel fetches (object storage) override this. + virtual Status read_batch(const std::vector& ranges, + std::vector>* outs) { + outs->resize(ranges.size()); + for (size_t i = 0; i < ranges.size(); ++i) { + RETURN_IF_ERROR(read_at(ranges[i].offset, ranges[i].len, &(*outs)[i])); + } + return Status::OK(); + } + + // Total size of the underlying object in bytes. + virtual uint64_t size() const = 0; + + // Optional live metrics. Readers that do not account I/O return nullptr. + virtual const IoMetrics* io_metrics() const { return nullptr; } +}; + +} // namespace doris::snii::io diff --git a/be/src/storage/index/snii/io/file_writer.h b/be/src/storage/index/snii/io/file_writer.h new file mode 100644 index 00000000000000..c14c61d2beaefd --- /dev/null +++ b/be/src/storage/index/snii/io/file_writer.h @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" + +namespace doris::snii::io { + +// Append-only writer (no seek-back), so the format can be produced in a single +// streaming pass compatible with S3FileWriter / StreamSinkFileWriter / packed +// writer. All container bytes are written front-to-back; back-references are +// resolved by writing metadata last. +class FileWriter { +public: + virtual ~FileWriter() = default; + + virtual Status append(Slice data) = 0; + virtual Status finalize() = 0; + virtual uint64_t bytes_written() const = 0; +}; + +} // namespace doris::snii::io diff --git a/be/src/storage/index/snii/io/io_metrics.h b/be/src/storage/index/snii/io/io_metrics.h new file mode 100644 index 00000000000000..0e1c628ede0137 --- /dev/null +++ b/be/src/storage/index/snii/io/io_metrics.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace doris::snii::io { + +// Object-storage access metrics collected at FileReader boundaries. +struct IoMetrics { + uint64_t read_at_calls = 0; // BE-internal logical read requests issued + uint64_t serial_rounds = 0; // dependent serial I/O rounds + uint64_t range_gets = 0; // remote range GETs after cache coalescing + uint64_t remote_bytes = 0; // bytes fetched from remote + uint64_t total_request_bytes = 0; // sum of requested lengths before cache +}; + +inline IoMetrics delta(const IoMetrics& after, const IoMetrics& before) { + IoMetrics out; + out.read_at_calls = after.read_at_calls - before.read_at_calls; + out.serial_rounds = after.serial_rounds - before.serial_rounds; + out.range_gets = after.range_gets - before.range_gets; + out.remote_bytes = after.remote_bytes - before.remote_bytes; + out.total_request_bytes = after.total_request_bytes - before.total_request_bytes; + return out; +} + +} // namespace doris::snii::io diff --git a/be/src/storage/index/snii/io/local_file.cpp b/be/src/storage/index/snii/io/local_file.cpp new file mode 100644 index 00000000000000..ff363f70bf6e12 --- /dev/null +++ b/be/src/storage/index/snii/io/local_file.cpp @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/io/local_file.h" + +#include +#include +#include + +#include +#include + +namespace doris::snii::io { +namespace { + +std::string errno_msg(const char* what) { + return std::string(what) + ": " + std::strerror(errno); +} + +} // namespace + +LocalFileReader::~LocalFileReader() { + if (fd_ >= 0) ::close(fd_); +} + +Status LocalFileReader::open(const std::string& path) { + fd_ = ::open(path.c_str(), O_RDONLY); + if (fd_ < 0) return Status::Error(errno_msg("open")); + struct stat st; + if (::fstat(fd_, &st) != 0) + return Status::Error(errno_msg("fstat")); + size_ = static_cast(st.st_size); + return Status::OK(); +} + +Status LocalFileReader::read_at(uint64_t offset, size_t len, std::vector* out) { + if (fd_ < 0) return Status::Error("read_at on unopened file"); + // Non-wrapping bounds check (offset+len could overflow uint64 on a corrupt arg). + if (offset > size_ || len > size_ - offset) { + return Status::Error( + "read_at past end of file"); + } + out->resize(len); + size_t done = 0; + while (done < len) { + ssize_t n = ::pread(fd_, out->data() + done, len - done, static_cast(offset + done)); + if (n < 0) { + if (errno == EINTR) continue; + return Status::Error(errno_msg("pread")); + } + if (n == 0) + return Status::Error( + "pread returned 0 before len"); + done += static_cast(n); + } + return Status::OK(); +} + +LocalFileWriter::~LocalFileWriter() { + if (fd_ >= 0) ::close(fd_); // best-effort: dtor cannot surface a flush error +} + +Status LocalFileWriter::open(const std::string& path) { + fd_ = ::open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd_ < 0) return Status::Error(errno_msg("open")); + buf_.reserve(kBufCapacity); + return Status::OK(); +} + +Status LocalFileWriter::write_all(const uint8_t* data, size_t len) { + size_t done = 0; + while (done < len) { + ssize_t n = ::write(fd_, data + done, len - done); + if (n < 0) { + if (errno == EINTR) continue; + return Status::Error(errno_msg("write")); + } + done += static_cast(n); + } + return Status::OK(); +} + +Status LocalFileWriter::flush_buffer() { + if (buf_.empty()) return Status::OK(); + RETURN_IF_ERROR(write_all(buf_.data(), buf_.size())); + buf_.clear(); + return Status::OK(); +} + +Status LocalFileWriter::append(Slice data) { + if (fd_ < 0) return Status::Error("append on unopened file"); + const size_t len = data.size(); + if (len == 0) return Status::OK(); + // Spans larger than the buffer go straight to the fd (after flushing pending + // bytes) to avoid a pointless copy and an oversized buffer. + if (len >= kBufCapacity) { + RETURN_IF_ERROR(flush_buffer()); + RETURN_IF_ERROR(write_all(data.data(), len)); + bytes_written_ += len; + return Status::OK(); + } + if (buf_.size() + len > kBufCapacity) RETURN_IF_ERROR(flush_buffer()); + buf_.insert(buf_.end(), data.data(), data.data() + len); + bytes_written_ += len; + return Status::OK(); +} + +Status LocalFileWriter::finalize() { + if (fd_ < 0) return Status::Error("finalize on unopened file"); + RETURN_IF_ERROR(flush_buffer()); + if (::fsync(fd_) != 0) return Status::Error(errno_msg("fsync")); + if (::close(fd_) != 0) { + fd_ = -1; + return Status::Error(errno_msg("close")); + } + fd_ = -1; + return Status::OK(); +} + +} // namespace doris::snii::io diff --git a/be/src/storage/index/snii/io/local_file.h b/be/src/storage/index/snii/io/local_file.h new file mode 100644 index 00000000000000..aaa26a8fdc681a --- /dev/null +++ b/be/src/storage/index/snii/io/local_file.h @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/io/file_reader.h" +#include "storage/index/snii/io/file_writer.h" + +namespace doris::snii::io { + +// Local-filesystem FileReader. Uses pread for positional, thread-safe reads +// (so concurrent batch fetches do not contend on a shared file offset). +class LocalFileReader : public FileReader { +public: + LocalFileReader() = default; + ~LocalFileReader() override; + + LocalFileReader(const LocalFileReader&) = delete; + LocalFileReader& operator=(const LocalFileReader&) = delete; + + Status open(const std::string& path); + Status read_at(uint64_t offset, size_t len, std::vector* out) override; + uint64_t size() const override { return size_; } + +private: + int fd_ = -1; + uint64_t size_ = 0; +}; + +// Local-filesystem append-only FileWriter. Appends accumulate in a fixed +// userspace buffer and are flushed to the fd in large chunks, collapsing the +// many tiny per-append ::write() syscalls of the build path (e.g. ~53k writes +// averaging ~683 B each) into a handful of big writes. The produced file is +// byte-identical to the unbuffered path; only the syscall count drops. +class LocalFileWriter : public FileWriter { +public: + LocalFileWriter() = default; + ~LocalFileWriter() override; + + LocalFileWriter(const LocalFileWriter&) = delete; + LocalFileWriter& operator=(const LocalFileWriter&) = delete; + + Status open(const std::string& path); + Status append(Slice data) override; + Status finalize() override; + uint64_t bytes_written() const override { return bytes_written_; } + +private: + // Userspace write buffer size. 256 KiB amortizes the write() syscall cost over + // many appends while keeping transient RAM negligible vs the index sections. + static constexpr size_t kBufCapacity = 256u * 1024; + + // Flushes the userspace buffer to the fd with a robust partial-write loop. + Status flush_buffer(); + // Writes a raw byte span straight to the fd (used for spans larger than the + // buffer, bypassing a needless copy). + Status write_all(const uint8_t* data, size_t len); + + int fd_ = -1; + uint64_t bytes_written_ = 0; + std::vector buf_; +}; + +} // namespace doris::snii::io diff --git a/be/src/storage/index/snii/io/metered_file_reader.cpp b/be/src/storage/index/snii/io/metered_file_reader.cpp new file mode 100644 index 00000000000000..3a0b07abe67d78 --- /dev/null +++ b/be/src/storage/index/snii/io/metered_file_reader.cpp @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/io/metered_file_reader.h" + +#include + +namespace doris::snii::io { +namespace { + +// Inclusive [first, last] block ids touched by a validated [offset, offset+len). +// Empty len touches no block (callers guard len==0 before calling this). +void block_range(uint64_t offset, size_t len, size_t block_size, uint64_t* first, uint64_t* last) { + *first = offset / block_size; + *last = (offset + len - 1) / block_size; +} + +} // namespace + +MeteredFileReader::MeteredFileReader(FileReader* inner, size_t block_size) + : inner_(inner), block_size_(block_size) {} + +void MeteredFileReader::reset_metrics() { + resident_.clear(); + metrics_ = IoMetrics {}; +} + +Status MeteredFileReader::validate_range(uint64_t offset, size_t len) const { + if (inner_ == nullptr) + return Status::Error("metered: null inner reader"); + if (block_size_ == 0) + return Status::Error("metered: zero block size"); + const uint64_t total = inner_->size(); + if (offset > total || len > total - offset) { + return Status::Error( + "metered: read range past end"); + } + return Status::OK(); +} + +// Accounts the FileCache effect of touching [offset, offset+len): newly missed +// blocks become coalesced remote GETs and remote bytes. Returns true iff any +// block missed. (Single contiguous span -> at most one coalesced run.) +bool MeteredFileReader::account_blocks(uint64_t offset, size_t len) { + if (len == 0) return false; + uint64_t first = 0, last = 0; + block_range(offset, len, block_size_, &first, &last); + + bool any_miss = false; + bool in_run = false; // currently inside a contiguous run of missing blocks + const uint64_t total = inner_->size(); + for (uint64_t b = first; b <= last; ++b) { + if (resident_.count(b)) { + in_run = false; + continue; + } + resident_.insert(b); + any_miss = true; + const uint64_t block_start = b * block_size_; + metrics_.remote_bytes += std::min(block_size_, total - block_start); + if (!in_run) { + ++metrics_.range_gets; // start of a new coalesced GET + in_run = true; + } + } + return any_miss; +} + +Status MeteredFileReader::read_at(uint64_t offset, size_t len, std::vector* out) { + if (out == nullptr) + return Status::Error("metered: null out"); + RETURN_IF_ERROR(validate_range(offset, len)); + ++metrics_.read_at_calls; + metrics_.total_request_bytes += len; + // A single blocking read: any miss forces one serial round (the next offset is + // not known until these bytes return). + if (account_blocks(offset, len)) ++metrics_.serial_rounds; + return inner_->read_at(offset, len, out); +} + +Status MeteredFileReader::read_batch(const std::vector& ranges, + std::vector>* outs) { + if (outs == nullptr) + return Status::Error("metered: null batch out"); + for (const Range& r : ranges) { + RETURN_IF_ERROR(validate_range(r.offset, r.len)); + } + + // Gather the union of touched blocks so coalescing spans the whole batch, and + // the entire batch counts as at most one serial round. + std::vector blocks; + for (const Range& r : ranges) { + metrics_.total_request_bytes += r.len; + if (r.len == 0) continue; + uint64_t first = 0, last = 0; + block_range(r.offset, r.len, block_size_, &first, &last); + for (uint64_t b = first; b <= last; ++b) blocks.push_back(b); + } + metrics_.read_at_calls += ranges.size(); + + std::sort(blocks.begin(), blocks.end()); + blocks.erase(std::unique(blocks.begin(), blocks.end()), blocks.end()); + + bool any_miss = false; + const uint64_t total = inner_->size(); + uint64_t prev_miss = 0; + bool have_prev = false; + for (uint64_t b : blocks) { + if (resident_.count(b)) continue; + resident_.insert(b); + any_miss = true; + metrics_.remote_bytes += std::min(block_size_, total - b * block_size_); + if (!have_prev || b != prev_miss + 1) ++metrics_.range_gets; // new run + prev_miss = b; + have_prev = true; + } + if (any_miss) ++metrics_.serial_rounds; + + // Delegate the actual byte fetch to the inner reader's batch path, so a backend + // that fetches a batch concurrently (e.g. S3FileReader) realizes the planned + // round as parallel GETs (matching the single serial round accounted above). + return inner_->read_batch(ranges, outs); +} + +} // namespace doris::snii::io diff --git a/be/src/storage/index/snii/io/metered_file_reader.h b/be/src/storage/index/snii/io/metered_file_reader.h new file mode 100644 index 00000000000000..c53453316b105b --- /dev/null +++ b/be/src/storage/index/snii/io/metered_file_reader.h @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "storage/index/snii/io/file_reader.h" +#include "storage/index/snii/io/io_metrics.h" + +namespace doris::snii::io { + +// A FileReader decorator that models an object-storage FileCache: reads are +// aligned to fixed (default 1MiB) blocks; only not-yet-resident blocks become +// remote range GETs (adjacent misses are coalesced). It is the single shared +// "yardstick" through which both single blocking reads and batched concurrent +// reads are measured. +// +// - read_at(): a single blocking read. Any cache miss => +1 serial round +// (the cursor must wait for bytes before the next offset is known). +// - read_batch(): all ranges submitted concurrently => the whole batch is at +// most one serial round (+1 iff any range misses). +class MeteredFileReader : public FileReader { +public: + explicit MeteredFileReader(FileReader* inner, size_t block_size = (1u << 20)); + + Status read_at(uint64_t offset, size_t len, std::vector* out) override; + Status read_batch(const std::vector& ranges, + std::vector>* outs) override; + uint64_t size() const override { return inner_->size(); } + + const IoMetrics& metrics() const { return metrics_; } + const IoMetrics* io_metrics() const override { return &metrics_; } + // Clears counters AND the resident block set, modelling a cold (cache-empty) query. + void reset_metrics(); + +private: + Status validate_range(uint64_t offset, size_t len) const; + + // Accounts the cache effect of touching [offset, offset+len): records misses, + // coalesced GETs, and remote bytes. Returns true iff at least one block missed. + bool account_blocks(uint64_t offset, size_t len); + + FileReader* inner_; + size_t block_size_; + std::unordered_set resident_; + IoMetrics metrics_; +}; + +} // namespace doris::snii::io diff --git a/be/src/storage/index/snii/query/bm25_scorer.cpp b/be/src/storage/index/snii/query/bm25_scorer.cpp new file mode 100644 index 00000000000000..587d564a09e328 --- /dev/null +++ b/be/src/storage/index/snii/query/bm25_scorer.cpp @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/bm25_scorer.h" + +#include +#include + +namespace doris::snii::query { + +double decode_norm(uint8_t encoded) { + return encoded == 0 ? 1.0 : static_cast(encoded); +} + +uint8_t encode_norm(uint64_t doc_length) { + const uint64_t clamped = std::clamp(doc_length, 1, 255); + return static_cast(clamped); +} + +ScorerContext ScorerContext::make(uint64_t n, uint64_t df) { + ScorerContext ctx; + ctx.df_ = df; + const double nn = static_cast(n); + const double dff = static_cast(df); + // idf = log(1 + (N - df + 0.5) / (df + 0.5)); always positive for df <= N. + ctx.idf_ = std::log(1.0 + (nn - dff + 0.5) / (dff + 0.5)); + return ctx; +} + +double ScorerContext::score(uint32_t tf, uint8_t encoded_norm, double avgdl, + const Bm25Params& params) const { + const double dl = decode_norm(encoded_norm); + const double tff = static_cast(tf); + const double denom = tff + params.k1 * (1.0 - params.b + params.b * dl / avgdl); + return idf_ * (tff * (params.k1 + 1.0)) / denom; +} + +double ScorerContext::max_score(uint32_t max_freq, uint8_t min_norm, double avgdl, + const Bm25Params& params) const { + // The score grows monotonically with tf and shrinks with dl, so the per-window + // upper bound uses the window's largest tf and smallest dl (min encoded norm). + return score(max_freq, min_norm, avgdl, params); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/bm25_scorer.h b/be/src/storage/index/snii/query/bm25_scorer.h new file mode 100644 index 00000000000000..71a15f0decc1b5 --- /dev/null +++ b/be/src/storage/index/snii/query/bm25_scorer.h @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +// Bm25Scorer -- classic Okapi BM25 relevance scoring over SNII native stats. +// +// Per query term, idf is precomputed once from the collection statistics: +// idf = log(1 + (N - df + 0.5) / (df + 0.5)) +// where N = indexed doc count and df = the term's document frequency. The +// per-document contribution of a term then is: +// score = idf * (tf * (k1 + 1)) / (tf + k1 * (1 - b + b * dl / avgdl)) +// where tf is the in-doc term frequency, dl the document length decoded from the +// 1-byte encoded norm, and avgdl the average document length. +// +// Norm encode/decode (DOCUMENTED CONTRACT): the writer stores doc length as a +// byte-quantized value floor-clamped to [1, 255]; decode is the identity map +// back to a double length. encode_norm(len) = clamp(len, 1, 255); +// decode_norm(b) = (b == 0 ? 1.0 : (double)b). This keeps short docs (len <= 255) +// exact and saturates longer docs at 255, matching the reference oracle. +namespace doris::snii::query { + +// BM25 free parameters. Defaults are the classic Lucene/Elasticsearch values. +struct Bm25Params { + double k1 = 1.2; + double b = 0.75; +}; + +// Decodes a 1-byte encoded norm into a document length. byte 0 maps to 1.0 to +// avoid a zero-length divisor; otherwise it is the byte value itself. +double decode_norm(uint8_t encoded); + +// Encodes a document length into a 1-byte norm (clamped to [1, 255]). Provided +// so writers and test oracles share one quantization. +uint8_t encode_norm(uint64_t doc_length); + +// Per-term scoring context: the precomputed idf and the term's df. Built once per +// query term, then reused for every candidate document of that term. +class ScorerContext { +public: + // Builds the context from collection size n (indexed doc count) and the term's + // document frequency df. avgdl and params are supplied per score call. + static ScorerContext make(uint64_t n, uint64_t df); + + double idf() const { return idf_; } + uint64_t df() const { return df_; } + + // Scores one document occurrence: tf is the in-doc term frequency, encoded_norm + // the doc's 1-byte length norm, avgdl the collection average length. + double score(uint32_t tf, uint8_t encoded_norm, double avgdl, const Bm25Params& params) const; + + // Upper bound on score() over any document, given a window's maximum tf and the + // shortest doc length in the window (smallest dl maximizes the score). Used by + // the WAND-style block-max pruner. max_freq is the window's max tf; min_norm is + // the smallest encoded norm (=> smallest dl => largest score). + double max_score(uint32_t max_freq, uint8_t min_norm, double avgdl, + const Bm25Params& params) const; + +private: + double idf_ = 0.0; + uint64_t df_ = 0; +}; + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/boolean_query.cpp b/be/src/storage/index/snii/query/boolean_query.cpp new file mode 100644 index 00000000000000..ab26dee1005efb --- /dev/null +++ b/be/src/storage/index/snii/query/boolean_query.cpp @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/boolean_query.h" + +#include +#include +#include +#include + +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/internal/docid_conjunction.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_union.h" +#include "storage/index/snii/reader/dict_block_cache.h" + +namespace doris::snii::query { + +namespace { + +std::vector unique_terms(const std::vector& terms) { + std::vector out; + out.reserve(terms.size()); + for (const std::string& term : terms) out.emplace_back(term); + std::sort(out.begin(), out.end()); + out.erase(std::unique(out.begin(), out.end()), out.end()); + return out; +} + +Status resolve_or_postings(const reader::LogicalIndexReader& idx, + const std::vector& terms, + std::vector* postings) { + postings->clear(); + // Request-scoped (stack-local, single-threaded) cache: OR terms that fall in the + // same on-demand DICT block read + zstd-decode + CRC-verify that block once + // instead of once per term. The resolved DictEntry is copied out, so the cache + // (and any pin it holds) can die when this returns. The shared reader stays const + // and lock-free -- no lock is ever held across the decode/IO. + reader::DictBlockCache dict_cache; + for (std::string_view term : unique_terms(terms)) { + bool found = false; + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + RETURN_IF_ERROR(idx.lookup(term, &found, &entry, &frq_base, &prx_base, &dict_cache)); + if (!found) continue; + + postings->push_back({std::move(entry), frq_base, prx_base}); + } + return Status::OK(); +} + +} // namespace + +Status boolean_or(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids) { + if (docids == nullptr) + return Status::Error("boolean_or: null out"); + docids->clear(); + if (terms.empty()) return Status::OK(); + + std::vector postings; + RETURN_IF_ERROR(resolve_or_postings(idx, terms, &postings)); + return internal::build_docid_union(idx, postings, docids); +} + +Status boolean_or(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids, QueryProfile* profile) { + QueryProfileScope profile_scope(idx.reader(), profile); + return boolean_or(idx, terms, docids); +} + +Status boolean_or(const reader::LogicalIndexReader& idx, const std::vector& terms, + DocIdSink* sink) { + if (sink == nullptr) + return Status::Error("boolean_or: null sink"); + if (terms.empty()) return Status::OK(); + + std::vector postings; + RETURN_IF_ERROR(resolve_or_postings(idx, terms, &postings)); + return internal::emit_docid_union(idx, postings, sink); +} + +Status boolean_and(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids) { + if (docids == nullptr) + return Status::Error("boolean_and: null out"); + docids->clear(); + if (terms.empty()) return Status::OK(); + + io::BatchRangeFetcher round1(idx.reader()); + std::vector plans; + bool all_present = false; + RETURN_IF_ERROR(internal::plan_terms(idx, terms, &round1, &plans, &all_present, + /*need_positions=*/false)); + if (!all_present) return Status::OK(); + if (round1.pending() > 0) RETURN_IF_ERROR(round1.fetch()); + RETURN_IF_ERROR(internal::open_preludes(round1, &plans, + /*need_positions=*/false)); + return internal::build_docid_only_conjunction(idx, round1, plans, docids); +} + +Status boolean_and(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids, QueryProfile* profile) { + QueryProfileScope profile_scope(idx.reader(), profile); + return boolean_and(idx, terms, docids); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/boolean_query.h b/be/src/storage/index/snii/query/boolean_query.h new file mode 100644 index 00000000000000..40cbb25644fdc1 --- /dev/null +++ b/be/src/storage/index/snii/query/boolean_query.h @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/query_profile.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// boolean_or -- MATCH_ANY semantics: return the sorted docid set containing at +// least one query term. Empty terms or all-absent terms produce an empty +// result. Duplicate input terms are ignored semantically and do not duplicate +// output docids. +namespace doris::snii::query { + +Status boolean_or(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids); +Status boolean_or(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids, QueryProfile* profile); +Status boolean_or(const reader::LogicalIndexReader& idx, const std::vector& terms, + DocIdSink* sink); + +// boolean_and (MATCH all-terms): sorted docid set of docs containing EVERY +// term, no positional constraint. Valid on docs-only indexes. Empty terms or +// any absent term -> empty result. +Status boolean_and(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids); +Status boolean_and(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids, QueryProfile* profile); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/count_query.cpp b/be/src/storage/index/snii/query/count_query.cpp new file mode 100644 index 00000000000000..b6361ffe9f7ea3 --- /dev/null +++ b/be/src/storage/index/snii/query/count_query.cpp @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/count_query.h" + +#include +#include + +#include "roaring/roaring.hh" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/query/internal/query_test_counters.h" + +namespace doris::snii::query { + +using format::DictEntry; +using reader::LogicalIndexReader; + +Status count_only_term_df(const LogicalIndexReader& idx, std::string_view term, uint64_t* count) { + if (count == nullptr) { + return Status::Error("count_only_term_df: null out"); + } + bool found = false; + DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + RETURN_IF_ERROR(idx.lookup(term, &found, &entry, &frq_base, &prx_base)); + *count = found ? entry.df : 0; + SNII_QUERY_COUNT(count_fastpath_hits); + return Status::OK(); +} + +Status count_only_two_term_phrase_bigram_df(const LogicalIndexReader& idx, const std::string& left, + const std::string& right, bool* handled, + uint64_t* count) { + if (handled == nullptr || count == nullptr) { + return Status::Error( + "count_only_two_term_phrase_bigram_df: null out"); + } + *handled = false; + *count = 0; + // The normal 2-term phrase path rejects positionless indexes with an error + // BEFORE consulting bigrams; the count path must fall through so the same + // error surfaces instead of a silently fabricated count. + if (!idx.has_positions()) { + return Status::OK(); + } + if (!format::is_phrase_bigram_indexable_term(left) || + !format::is_phrase_bigram_indexable_term(right)) { + return Status::OK(); + } + // Fresh deferred segments have no hidden pair postings. Let the normal + // phrase execution own the positions verification without spending a probe + // here and another one when it falls through from the count fast path. + if (idx.phrase_bigrams_deferred()) { + return Status::OK(); + } + + bool found = false; + DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + SNII_QUERY_COUNT(bigram_probe_attempts); + RETURN_IF_ERROR(idx.lookup(format::make_phrase_bigram_term(left, right), &found, &entry, + &frq_base, &prx_base)); + if (!found) { + // Miss is ambiguous on df-pruned segments (G01 min / G15 max) and + // merely "empty" on legacy ones; both fall through so + // TryTwoTermPhraseBigram stays the single owner of the miss semantics. + return Status::OK(); + } + // Docid membership IS the phrase answer for a materialized bigram (G01 + // part B stores them docs-only), so its df is the exact phrase doc count. + *handled = true; + *count = entry.df; + SNII_QUERY_COUNT(count_fastpath_hits); + return Status::OK(); +} + +Status fabricate_null_disjoint_count_bitmap(uint64_t count, const roaring::Roaring& nulls, + roaring::Roaring* out) { + if (out == nullptr) { + return Status::Error( + "fabricate_null_disjoint_count_bitmap: null out"); + } + roaring::Roaring result; + if (count > 0) { + // [0, count + |nulls|) holds at least `count` non-null ids: at most + // |nulls| of its members are null. count counts only non-null docs, so + // the window end never exceeds the segment doc count (row space). + const uint64_t window_end = count + nulls.cardinality(); + if (window_end > uint64_t(std::numeric_limits::max()) + 1) { + return Status::Error( + "fabricate_null_disjoint_count_bitmap: count {} + null count {} exceeds the " + "uint32 docid domain (corrupt df or null bitmap)", + count, nulls.cardinality()); + } + result.addRange(0, window_end); + result -= nulls; + uint32_t last_kept = 0; + // Keep exactly the first `count` survivors (select ranks are 0-based). + if (!result.select(static_cast(count - 1), &last_kept)) { + return Status::Error( + "fabricate_null_disjoint_count_bitmap: window [0, {}) holds fewer than {} " + "non-null ids (corrupt df or null bitmap)", + window_end, count); + } + result.removeRange(uint64_t(last_kept) + 1, window_end); + } + *out = std::move(result); + return Status::OK(); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/count_query.h b/be/src/storage/index/snii/query/count_query.h new file mode 100644 index 00000000000000..3f17138973b93e --- /dev/null +++ b/be/src/storage/index/snii/query/count_query.h @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// Forward-declare the CRoaring C++ bitmap so this header stays free of the +// (large) roaring include; the concrete type is only needed in the .cpp. +namespace roaring { +class Roaring; +} // namespace roaring + +// count_query -- G02 count-only fast path primitives. They answer "how many +// docs match" from DICT ENTRIES ALONE (per-term df), never touching the .frq +// posting bytes, for the two shapes whose df IS the exact per-segment match +// count: +// +// - single exact term: DictEntry.df is by definition the number of docs +// containing the term in this segment (nulls are never in postings, so df +// already matches MATCH's "null never matches" semantics); +// - 2-term consecutive phrase answered by a hidden phrase-bigram dict HIT: +// per the G01 contract, bigram docid membership IS the phrase answer +// (TryTwoTermPhraseBigram in phrase_query.cpp), so the bigram entry's df +// is the exact phrase doc count. +// +// Everything else -- multi-term OR/AND, prefix/regexp/wildcard expansion, +// pruned or absent bigrams, positionless indexes -- must fall through to the +// normal decode path (*handled stays false). Deletes/extra predicates are a +// CALLER responsibility: these primitives count docs in the raw segment and +// are only safe when the caller proved nothing else filters the row space +// (see SniiIndexReader::_try_count_only_fastpath and +// SegmentIterator guards in count_on_index_fastpath.h). +namespace doris::snii::query { + +// df of `term` in this segment without decoding postings. An absent term is a +// deterministic answer too: *count = 0 (mirrors term_query's empty result). +// Increments the count_fastpath_hits test seam. +Status count_only_term_df(const reader::LogicalIndexReader& idx, std::string_view term, + uint64_t* count); + +// Count-only answer for the 2-term consecutive phrase [left, right] via the +// hidden phrase-bigram term. *handled = true ONLY on a bigram dict HIT (a G01 +// survivor on pruned segments, or any materialized pair on legacy segments); +// then *count = bigram df. Every miss falls through (*handled = false) -- +// including the legacy "miss == no adjacency" case -- so the generic phrase +// path stays the single owner of miss semantics. Also falls through when the +// index has no positions (the normal phrase path errors there and the fast +// path must not mask it) or when either term is not bigram-indexable. +// Increments the count_fastpath_hits test seam only when handled. +Status count_only_two_term_phrase_bigram_df(const reader::LogicalIndexReader& idx, + const std::string& left, const std::string& right, + bool* handled, uint64_t* count); + +// Builds the fabricated count bitmap for a segment WITH a null bitmap: exactly +// `count` row ids DISJOINT from `nulls` (the first `count` non-null row ids, +// all < count + |nulls|). Why disjoint: the MATCH machinery unconditionally +// subtracts the segment null bitmap from every index result +// (FunctionMatchBase -> InvertedIndexResultBitmap::mask_out_null). Real +// postings never contain null docs -- the writer adds NO tokens for a null doc +// (scalar add_nulls) and a NULL array row is stored as an empty range (zero +// tokens) -- so that subtraction is a no-op on true results and df already IS +// the exact match count regardless of nulls. A naive [0, df) range however MAY +// collide with null row ids and be shrunk by mask_out_null; picking the ids +// from the non-null space makes the subtraction provably a no-op, preserving +// cardinality == df end to end. +// +// The window bound is doc-count-free: count counts only non-null docs, so +// count + |nulls| <= segment doc count and [0, count + |nulls|) always holds +// >= count non-null ids; every fabricated id therefore stays inside the +// segment's [0, num_rows) row space. Errors (id space would exceed the uint32 +// docid domain, or the window unexpectedly holds fewer than `count` survivors) +// only occur on a corrupt index; callers treat them as "fall through to the +// decode path", never as a fabricated answer. +Status fabricate_null_disjoint_count_bitmap(uint64_t count, const roaring::Roaring& nulls, + roaring::Roaring* out); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/docid_conjunction.cpp b/be/src/storage/index/snii/query/docid_conjunction.cpp new file mode 100644 index 00000000000000..7cc212b28b0b5b --- /dev/null +++ b/be/src/storage/index/snii/query/docid_conjunction.cpp @@ -0,0 +1,875 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/internal/docid_conjunction.h" + +#include +#include +#include +#include + +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/query/internal/docid_set_ops.h" +#include "storage/index/snii/reader/windowed_posting.h" + +namespace doris::snii::query::internal { + +using format::DictEntry; +using format::DictEntryEnc; +using format::DictEntryKind; +using format::FrqPreludeReader; +using format::WindowMeta; +using reader::LogicalIndexReader; + +namespace { + +using CandidateIt = std::vector::const_iterator; + +constexpr uint32_t kBoundedSpanBitsetDocs = 16 * 1024; +constexpr size_t kBoundedSpanBitsetWords = kBoundedSpanBitsetDocs / 64; +constexpr size_t kBoundedSpanBitsetMinInput = 32; + +struct CandidateRange { + size_t begin = 0; + size_t end = 0; +}; + +Status slim_frq_docs_len(const DictEntry& entry, uint64_t win_len, uint64_t* out) { + if (entry.frq_docs_len > win_len) { + return Status::Error( + "docid_conjunction: slim frq_docs_len exceeds frq window"); + } + *out = entry.frq_docs_len > 0 ? entry.frq_docs_len : win_len; + return Status::OK(); +} + +Status add_u64(uint64_t lhs, uint64_t rhs, const char* message, uint64_t* out) { + if (rhs > std::numeric_limits::max() - lhs) { + return Status::Error(message); + } + *out = lhs + rhs; + return Status::OK(); +} + +Status posting_abs_offset(const LogicalIndexReader& idx, uint64_t base, uint64_t delta, + const char* message, uint64_t* out) { + uint64_t with_base = 0; + RETURN_IF_ERROR(add_u64(idx.section_refs().posting_region.offset, base, message, &with_base)); + return add_u64(with_base, delta, message, out); +} + +Status configure_term_plan(const LogicalIndexReader& idx, bool need_positions, + io::BatchRangeFetcher* fetcher, TermPlan* p) { + p->df = p->entry.df; + p->pod_ref = (p->entry.kind == DictEntryKind::kPodRef); + p->windowed = p->pod_ref && p->entry.enc == DictEntryEnc::kWindowed; + if (p->windowed) { + uint64_t prelude_abs = 0; + RETURN_IF_ERROR(posting_abs_offset(idx, p->frq_base, p->entry.frq_off_delta, + "docid_conjunction: prelude offset overflow", + &prelude_abs)); + p->prelude_handle = fetcher->add(prelude_abs, p->entry.prelude_len); + } else if (p->pod_ref) { + uint64_t foff = 0; + uint64_t flen = 0; + uint64_t poff = 0; + uint64_t plen = 0; + RETURN_IF_ERROR(idx.resolve_frq_window(p->entry, p->frq_base, &foff, &flen)); + uint64_t frq_fetch = flen; + RETURN_IF_ERROR(slim_frq_docs_len(p->entry, flen, &frq_fetch)); + p->frq_handle = fetcher->add(foff, frq_fetch); + if (need_positions) { + RETURN_IF_ERROR(idx.resolve_prx_window(p->entry, p->prx_base, &poff, &plen)); + p->prx_handle = fetcher->add(poff, plen); + } + } + return Status::OK(); +} + +std::vector all_windows(const FrqPreludeReader& prelude) { + std::vector ws(prelude.n_windows()); + for (uint32_t i = 0; i < prelude.n_windows(); ++i) ws[i] = i; + return ws; +} + +std::vector ascending_df_order(const std::vector& plans) { + std::vector order(plans.size()); + for (size_t i = 0; i < plans.size(); ++i) order[i] = i; + std::sort(order.begin(), order.end(), + [&](size_t a, size_t b) { return plans[a].df < plans[b].df; }); + return order; +} + +Status first_docid_in_window(const WindowMeta& meta, uint32_t window_ordinal, uint32_t* first) { + if (window_ordinal == 0) { + *first = 0; + return Status::OK(); + } + if (meta.win_base >= std::numeric_limits::max()) { + return Status::Error( + "docid_conjunction: window base exceeds docid range"); + } + *first = static_cast(meta.win_base + 1); + if (*first > meta.last_docid) { + return Status::Error( + "docid_conjunction: invalid window docid range"); + } + return Status::OK(); +} + +Status is_dense_full_window(const WindowMeta& meta, uint32_t window_ordinal, bool* full) { + uint32_t first = 0; + RETURN_IF_ERROR(first_docid_in_window(meta, window_ordinal, &first)); + const uint64_t width = static_cast(meta.last_docid) - first + 1; + *full = meta.doc_count == width; + return Status::OK(); +} + +Status append_docid_range(uint32_t first, uint32_t last, std::vector* out) { + if (last < first) { + return Status::Error( + "docid_conjunction: invalid dense docid range"); + } + const uint64_t count64 = static_cast(last) - first + 1; + if (count64 > static_cast(std::numeric_limits::max() - out->size())) { + return Status::Error( + "docid_conjunction: dense docid range too large"); + } + out->reserve(out->size() + static_cast(count64)); + uint32_t docid = first; + while (true) { + out->push_back(docid); + if (docid == last) break; + ++docid; + } + return Status::OK(); +} + +CandidateRange find_candidate_range(const std::vector& candidates, size_t* search_begin, + uint32_t first, uint32_t last) { + const auto from = candidates.begin() + *search_begin; + const auto begin = std::lower_bound(from, candidates.end(), first); + const auto end = std::upper_bound(begin, candidates.end(), last); + *search_begin = static_cast(end - candidates.begin()); + return {.begin = static_cast(begin - candidates.begin()), + .end = static_cast(end - candidates.begin())}; +} + +void append_candidate_range(CandidateIt begin, CandidateIt end, std::vector* out) { + out->insert(out->end(), begin, end); +} + +void append_new_chunk_docids_to_out(const DocidChunk& chunk, size_t chunk_docids_begin, + std::vector* out) { + out->insert(out->end(), chunk.docids.begin() + chunk_docids_begin, chunk.docids.end()); +} + +void clear_ordinals_if_all_term_docs_selected(const std::vector& term_docids, + DocidChunk* chunk) { + if (chunk->docids.size() == term_docids.size() && !chunk->docids.empty() && + chunk->docids.front() == term_docids.front() && + chunk->docids.back() == term_docids.back()) { + chunk->prx_doc_ordinals.clear(); + } +} + +bool append_term_docs_if_candidates_cover_span(CandidateIt begin, CandidateIt end, + const std::vector& term_docids, + std::vector* out, DocidChunk* chunk) { + const uint32_t first = term_docids.front(); + const uint32_t last = term_docids.back(); + const uint64_t width = static_cast(last) - first + 1; + const size_t candidate_count = static_cast(end - begin); + if (width > candidate_count) { + return false; + } + + const auto span_begin = *begin == first ? begin : std::lower_bound(begin, end, first); + if (span_begin == end || *span_begin != first) { + return false; + } + if (static_cast(end - span_begin) < width) { + return false; + } + + const auto span_last = span_begin + static_cast(width) - 1; + if (*span_last != last) { + return false; + } + + const size_t chunk_docids_begin = chunk->docids.size(); + chunk->docids.insert(chunk->docids.end(), term_docids.begin(), term_docids.end()); + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + return true; +} + +Status append_candidate_range_with_ordinals(CandidateIt begin, CandidateIt end, uint32_t first, + uint32_t last, std::vector* out, + DocidChunk* chunk) { + const size_t candidate_count = static_cast(end - begin); + chunk->docids.reserve(candidate_count); + const uint64_t width = static_cast(last) - first + 1; + if (width > std::numeric_limits::max()) { + return Status::Error( + "docid_conjunction: dense window exceeds doc count range"); + } + chunk->prx_doc_count = static_cast(width); + const bool full_dense_range = + candidate_count == width && begin != end && *begin == first && *(end - 1) == last; + const size_t chunk_docids_begin = chunk->docids.size(); + if (full_dense_range) { + chunk->docids.insert(chunk->docids.end(), begin, end); + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + return Status::OK(); + } + chunk->prx_doc_ordinals.reserve(chunk->prx_doc_ordinals.size() + candidate_count); + for (auto it = begin; it != end; ++it) { + chunk->docids.push_back(*it); + chunk->prx_doc_ordinals.push_back(*it - first); + } + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + return Status::OK(); +} + +bool intersect_dense_term_span_with_ordinals(CandidateIt begin, CandidateIt end, + const std::vector& term_docids, + size_t candidate_count, std::vector* out, + DocidChunk* chunk) { + const uint32_t first = term_docids.front(); + const uint32_t last = term_docids.back(); + const uint64_t width = static_cast(last) - first + 1; + if (term_docids.size() > width) { + return false; + } + const uint64_t missing_count = width - term_docids.size(); + if (missing_count != 0 && + (missing_count * 8 > width || missing_count >= candidate_count || + missing_count > static_cast(std::numeric_limits::max()))) { + return false; + } + + const size_t chunk_docids_begin = chunk->docids.size(); + if (missing_count == 0) { + for (auto it = begin; it != end; ++it) { + if (*it < first) { + continue; + } + if (*it > last) { + break; + } + chunk->docids.push_back(*it); + chunk->prx_doc_ordinals.push_back(*it - first); + } + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + clear_ordinals_if_all_term_docs_selected(term_docids, chunk); + return true; + } + + std::vector missing; + missing.reserve(static_cast(missing_count)); + uint32_t expect = first; + for (uint32_t docid : term_docids) { + while (expect < docid) { + missing.push_back(expect); + ++expect; + } + if (docid < std::numeric_limits::max()) { + expect = docid + 1; + } + } + while (expect <= last) { + missing.push_back(expect); + if (expect == std::numeric_limits::max()) { + break; + } + ++expect; + } + + size_t miss = 0; + for (auto it = begin; it != end; ++it) { + if (*it < first) { + continue; + } + if (*it > last) { + break; + } + while (miss < missing.size() && missing[miss] < *it) { + ++miss; + } + if (miss < missing.size() && missing[miss] == *it) { + continue; + } + chunk->docids.push_back(*it); + chunk->prx_doc_ordinals.push_back(static_cast(*it - first - miss)); + } + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + clear_ordinals_if_all_term_docs_selected(term_docids, chunk); + return true; +} + +bool intersect_bounded_span_with_ordinals(CandidateIt begin, CandidateIt end, + const std::vector& term_docids, + size_t candidate_count, std::vector* out, + DocidChunk* chunk) { + if (candidate_count < kBoundedSpanBitsetMinInput || + term_docids.size() < kBoundedSpanBitsetMinInput) { + return false; + } + + const uint32_t first = std::min(*begin, term_docids.front()); + const uint32_t last = std::max(*(end - 1), term_docids.back()); + const uint64_t width = static_cast(last) - first + 1; + if (width > kBoundedSpanBitsetDocs || term_docids.size() > width) { + return false; + } + + const auto word_count = static_cast((width + 63) >> 6); + std::array bits; + std::fill_n(bits.begin(), word_count, 0); + for (uint32_t docid : term_docids) { + const uint32_t off = docid - first; + bits[off >> 6] |= 1ULL << (off & 63); + } + + std::array ordinal_base; + uint32_t ordinal = 0; + for (size_t word = 0; word < word_count; ++word) { + ordinal_base[word] = ordinal; + ordinal += static_cast(__builtin_popcountll(bits[word])); + } + + const size_t chunk_docids_begin = chunk->docids.size(); + for (auto it = begin; it != end; ++it) { + const uint32_t off = *it - first; + const size_t word = off >> 6; + const uint64_t mask = 1ULL << (off & 63); + if ((bits[word] & mask) == 0) { + continue; + } + chunk->docids.push_back(*it); + chunk->prx_doc_ordinals.push_back( + ordinal_base[word] + + static_cast(__builtin_popcountll(bits[word] & (mask - 1)))); + } + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + clear_ordinals_if_all_term_docs_selected(term_docids, chunk); + return true; +} + +size_t log2_ceil(size_t n) { + if (n <= 1) return 1; + --n; + size_t bits = 0; + while (n != 0) { + ++bits; + n >>= 1; + } + return bits; +} + +void intersect_window_candidate_range(CandidateIt begin, CandidateIt end, + const std::vector& term_docids, uint32_t first, + uint32_t last, std::vector* out) { + const size_t candidate_count = static_cast(end - begin); + if (candidate_count == 0 || term_docids.empty()) return; + + const uint64_t width = static_cast(last) - first + 1; + const uint64_t missing_count = term_docids.size() <= width ? width - term_docids.size() : width; + if (term_docids.size() <= width && missing_count != 0 && missing_count * 8 <= width && + missing_count < candidate_count) { + std::vector missing; + missing.reserve(static_cast(missing_count)); + uint32_t expect = first; + for (uint32_t docid : term_docids) { + while (expect < docid) { + missing.push_back(expect); + ++expect; + } + if (docid < std::numeric_limits::max()) expect = docid + 1; + } + while (expect <= last) { + missing.push_back(expect); + if (expect == std::numeric_limits::max()) break; + ++expect; + } + size_t miss = 0; + for (auto it = begin; it != end; ++it) { + while (miss < missing.size() && missing[miss] < *it) ++miss; + if (miss == missing.size() || missing[miss] != *it) out->push_back(*it); + } + return; + } + + const size_t probes_per_candidate = log2_ceil(term_docids.size()) + 1; + if (candidate_count < term_docids.size() / probes_per_candidate) { + for (auto it = begin; it != end; ++it) { + if (std::binary_search(term_docids.begin(), term_docids.end(), *it)) { + out->push_back(*it); + } + } + return; + } + std::set_intersection(begin, end, term_docids.begin(), term_docids.end(), + std::back_inserter(*out)); +} + +Status intersect_window_candidate_range_with_ordinals(CandidateIt begin, CandidateIt end, + const std::vector& term_docids, + std::vector* out, + DocidChunk* chunk) { + if (term_docids.size() > std::numeric_limits::max()) { + return Status::Error( + "docid_conjunction: prx doc count exceeds u32"); + } + chunk->prx_doc_count = static_cast(term_docids.size()); + if (begin == end || term_docids.empty()) return Status::OK(); + + const size_t candidate_count = static_cast(end - begin); + const size_t max_matches = std::min(candidate_count, term_docids.size()); + out->reserve(out->size() + max_matches); + chunk->docids.reserve(chunk->docids.size() + max_matches); + if (candidate_count == term_docids.size() && *begin == term_docids.front() && + *(end - 1) == term_docids.back() && std::equal(begin, end, term_docids.begin())) { + const size_t chunk_docids_begin = chunk->docids.size(); + chunk->docids.insert(chunk->docids.end(), begin, end); + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + return Status::OK(); + } + if (append_term_docs_if_candidates_cover_span(begin, end, term_docids, out, chunk)) { + return Status::OK(); + } + + chunk->prx_doc_ordinals.reserve(chunk->prx_doc_ordinals.size() + max_matches); + if (intersect_dense_term_span_with_ordinals(begin, end, term_docids, candidate_count, out, + chunk)) { + return Status::OK(); + } + if (intersect_bounded_span_with_ordinals(begin, end, term_docids, candidate_count, out, + chunk)) { + return Status::OK(); + } + + const size_t probes_per_candidate = log2_ceil(term_docids.size()) + 1; + if (candidate_count < term_docids.size() / probes_per_candidate) { + const size_t chunk_docids_begin = chunk->docids.size(); + size_t doc_index = 0; + for (auto it = begin; it != end; ++it) { + const auto found = + std::lower_bound(term_docids.begin() + doc_index, term_docids.end(), *it); + if (found == term_docids.end()) break; + doc_index = static_cast(found - term_docids.begin()); + if (*found != *it) continue; + chunk->docids.push_back(*it); + chunk->prx_doc_ordinals.push_back(static_cast(doc_index)); + ++doc_index; + } + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + clear_ordinals_if_all_term_docs_selected(term_docids, chunk); + return Status::OK(); + } + + const size_t probes_per_term_doc = log2_ceil(candidate_count) + 1; + if (term_docids.size() < candidate_count / probes_per_term_doc) { + const size_t chunk_docids_begin = chunk->docids.size(); + auto candidate_it = begin; + for (size_t doc_index = 0; doc_index < term_docids.size(); ++doc_index) { + const uint32_t docid = term_docids[doc_index]; + candidate_it = std::lower_bound(candidate_it, end, docid); + if (candidate_it == end) break; + if (*candidate_it != docid) continue; + chunk->docids.push_back(docid); + chunk->prx_doc_ordinals.push_back(static_cast(doc_index)); + ++candidate_it; + } + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + clear_ordinals_if_all_term_docs_selected(term_docids, chunk); + return Status::OK(); + } + + const size_t chunk_docids_begin = chunk->docids.size(); + size_t doc_index = 0; + for (auto it = begin; it != end; ++it) { + while (doc_index < term_docids.size() && term_docids[doc_index] < *it) { + ++doc_index; + } + if (doc_index == term_docids.size()) break; + if (term_docids[doc_index] != *it) continue; + chunk->docids.push_back(*it); + chunk->prx_doc_ordinals.push_back(static_cast(doc_index)); + ++doc_index; + } + append_new_chunk_docids_to_out(*chunk, chunk_docids_begin, out); + clear_ordinals_if_all_term_docs_selected(term_docids, chunk); + return Status::OK(); +} + +bool should_scan_all_windows(const LogicalIndexReader& idx, const TermPlan& p, + size_t candidate_count) { + const size_t window_count = p.prelude.n_windows(); + if (candidate_count > window_count * 64) return true; + + const uint64_t doc_count = idx.stats().doc_count; + const bool near_full = doc_count != 0 && static_cast(p.df) * 10 >= doc_count * 9; + return near_full && candidate_count > window_count * 4; +} + +Status decode_flat_docids_only(const io::BatchRangeFetcher& round1, const TermPlan& p, + std::vector* docids) { + Slice dd; + if (p.pod_ref) { + dd = round1.get(p.frq_handle); + } else { + RETURN_IF_ERROR(inline_dd_region(p.entry, &dd)); + } + return format::decode_dd_region(dd, p.entry.dd_meta, /*win_base=*/0, docids); +} + +struct WindowWork { + uint32_t ordinal = 0; + WindowMeta meta; + CandidateRange candidates; + size_t handle = 0; + bool dense_full = false; +}; + +Status emit_dense_full_window_docids(const WindowWork& f, const std::vector* candidates, + std::vector& out, DocidSource* source) { + uint32_t first = 0; + RETURN_IF_ERROR(first_docid_in_window(f.meta, f.ordinal, &first)); + if (source != nullptr) { + DocidChunk chunk; + chunk.windowed = true; + chunk.window = f.ordinal; + chunk.prx_doc_count = f.meta.doc_count; + if (candidates == nullptr) { + RETURN_IF_ERROR(append_docid_range(first, f.meta.last_docid, &chunk.docids)); + } else { + const auto begin = candidates->begin() + f.candidates.begin; + const auto end = candidates->begin() + f.candidates.end; + RETURN_IF_ERROR(append_candidate_range_with_ordinals(begin, end, first, + f.meta.last_docid, &out, &chunk)); + } + source->chunks.push_back(std::move(chunk)); + } + if (candidates == nullptr) { + RETURN_IF_ERROR(append_docid_range(first, f.meta.last_docid, &out)); + } else if (source == nullptr) { + append_candidate_range(candidates->begin() + f.candidates.begin, + candidates->begin() + f.candidates.end, &out); + } + return Status::OK(); +} + +Status emit_decoded_window_docids(const WindowWork& f, const io::BatchRangeFetcher& fetcher, + const std::vector* candidates, + std::vector& out, DocidSource* source, + std::vector& docs, std::vector& freqs, + std::vector>& positions) { + docs.clear(); + freqs.clear(); + positions.clear(); + RETURN_IF_ERROR(reader::decode_window_slices(f.meta, fetcher.get(f.handle), Slice(), Slice(), + /*want_positions=*/false, /*want_freq=*/false, + &docs, &freqs, &positions)); + if (source != nullptr) { + DocidChunk chunk; + chunk.windowed = true; + chunk.window = f.ordinal; + if (candidates == nullptr) { + chunk.docids = docs; + if (docs.size() > std::numeric_limits::max()) { + return Status::Error( + "docid_conjunction: prx doc count exceeds u32"); + } + chunk.prx_doc_count = static_cast(docs.size()); + source->chunks.push_back(std::move(chunk)); + } else { + const auto begin = candidates->begin() + f.candidates.begin; + const auto end = candidates->begin() + f.candidates.end; + RETURN_IF_ERROR( + intersect_window_candidate_range_with_ordinals(begin, end, docs, &out, &chunk)); + if (!chunk.docids.empty()) { + source->chunks.push_back(std::move(chunk)); + } + } + } + if (candidates == nullptr) { + out.insert(out.end(), docs.begin(), docs.end()); + return Status::OK(); + } + if (source != nullptr) { + return Status::OK(); + } + uint32_t first = 0; + RETURN_IF_ERROR(first_docid_in_window(f.meta, f.ordinal, &first)); + intersect_window_candidate_range(candidates->begin() + f.candidates.begin, + candidates->begin() + f.candidates.end, docs, first, + f.meta.last_docid, &out); + return Status::OK(); +} + +Status collect_windowed_docids_only(const LogicalIndexReader& idx, const TermPlan& p, + const std::vector& windows, + const std::vector* candidates, + std::vector* out, DocidSource* source) { + io::BatchRangeFetcher fetcher(idx.reader(), reader::kSameTermCoalesceGap); + std::vector work; + work.reserve(windows.size()); + out->reserve(candidates == nullptr ? p.entry.df : candidates->size()); + size_t candidate_search_begin = 0; + for (uint32_t w : windows) { + WindowMeta meta; + RETURN_IF_ERROR(p.prelude.window(w, &meta)); + uint32_t first = 0; + RETURN_IF_ERROR(first_docid_in_window(meta, w, &first)); + CandidateRange candidate_range; + if (candidates != nullptr) { + candidate_range = find_candidate_range(*candidates, &candidate_search_begin, first, + meta.last_docid); + if (candidate_range.begin == candidate_range.end) { + continue; + } + } + bool dense_full = false; + RETURN_IF_ERROR(is_dense_full_window(meta, w, &dense_full)); + if (dense_full) { + work.push_back(WindowWork { + .ordinal = w, .meta = meta, .candidates = candidate_range, .dense_full = true}); + continue; + } + + reader::WindowAbsRange range; + RETURN_IF_ERROR(reader::windowed_window_range( + idx, p.entry, p.frq_base, p.prx_base, p.prelude, w, + /*want_positions=*/false, /*want_freq=*/false, &range)); + WindowWork f; + f.ordinal = w; + f.meta = meta; + f.candidates = candidate_range; + f.handle = fetcher.add(range.dd_off, range.dd_len); + work.push_back(f); + } + if (fetcher.pending() > 0) { + RETURN_IF_ERROR(fetcher.fetch()); + } + + std::vector docs; + std::vector freqs; + std::vector> positions; + for (const WindowWork& f : work) { + if (f.dense_full) { + RETURN_IF_ERROR(emit_dense_full_window_docids(f, candidates, *out, source)); + continue; + } + RETURN_IF_ERROR(emit_decoded_window_docids(f, fetcher, candidates, *out, source, docs, + freqs, positions)); + } + return Status::OK(); +} + +Status collect_docids_only(const LogicalIndexReader& idx, const io::BatchRangeFetcher& round1, + const TermPlan& p, const std::vector* candidates, + std::vector* out, DocidSource* source) { + if (p.windowed) { + std::vector windows; + if (candidates == nullptr) { + windows = all_windows(p.prelude); + } else if (should_scan_all_windows(idx, p, candidates->size())) { + // Dense candidate sets cover most windows; for near-full terms this also + // avoids a thousands-to-millions probe covering-window cursor pass with no + // byte win. + windows = all_windows(p.prelude); + } else { + p.prelude.select_covering_windows(*candidates, &windows); + } + return collect_windowed_docids_only(idx, p, windows, candidates, out, source); + } + + std::vector term_docids; + RETURN_IF_ERROR(decode_flat_docids_only(round1, p, &term_docids)); + if (source != nullptr) { + DocidChunk chunk; + if (term_docids.size() > std::numeric_limits::max()) { + return Status::Error( + "docid_conjunction: prx doc count exceeds u32"); + } + chunk.prx_doc_count = static_cast(term_docids.size()); + if (candidates == nullptr) { + chunk.docids = term_docids; + } else if (!term_docids.empty()) { + const auto begin = std::ranges::lower_bound(*candidates, term_docids.front()); + const auto end = std::upper_bound(begin, candidates->end(), term_docids.back()); + RETURN_IF_ERROR(intersect_window_candidate_range_with_ordinals(begin, end, term_docids, + out, &chunk)); + } + if (candidates == nullptr || !chunk.docids.empty()) { + source->chunks.push_back(std::move(chunk)); + } + } + if (candidates == nullptr) { + *out = std::move(term_docids); + return Status::OK(); + } + if (source != nullptr) { + return Status::OK(); + } + *out = intersect_sorted(*candidates, term_docids); + return Status::OK(); +} + +Status run_docid_only_conjunction_impl(const LogicalIndexReader& idx, + const io::BatchRangeFetcher& round1, + const std::vector& plans, + const std::vector* initial_candidates, + std::vector* candidates, + std::vector* sources) { + if (sources != nullptr) { + sources->assign(plans.size(), DocidSource {}); + } + candidates->clear(); + if (plans.empty()) { + // No terms: the result is the initial candidate set verbatim (or empty). + if (initial_candidates != nullptr) { + *candidates = *initial_candidates; + } + return Status::OK(); + } + if (initial_candidates != nullptr && initial_candidates->empty()) { + return Status::OK(); + } + const std::vector order = ascending_df_order(plans); + for (size_t k = 0; k < order.size(); ++k) { + const size_t ti = order[k]; + std::vector next; + DocidSource* source = sources == nullptr ? nullptr : &(*sources)[ti]; + // k == 0 intersects against the (const) initial_candidates DIRECTLY, so + // the whole set is never copied once per plan -- the previous code seeded + // *candidates = *initial_candidates before the loop, which for a single + // plan (e.g. one phrase-prefix tail verified against the leading-term + // expected docids) was an O(|initial|) copy per call with no benefit. + // k > 0 chains on the previous term's already-whittled result. + const std::vector* input_candidates = k == 0 ? initial_candidates : candidates; + RETURN_IF_ERROR( + collect_docids_only(idx, round1, plans[ti], input_candidates, &next, source)); + if (source != nullptr && k + 1 == order.size()) { + source->docids_are_final_candidates = true; + } + *candidates = std::move(next); + if (candidates->empty()) { + return Status::OK(); + } + } + return Status::OK(); +} + +} // namespace + +Status resolve_query_term(const LogicalIndexReader& idx, const std::string& term, + ResolvedQueryTerm* resolved, bool* found) { + *found = false; + RETURN_IF_ERROR( + idx.lookup(term, found, &resolved->entry, &resolved->frq_base, &resolved->prx_base)); + return Status::OK(); +} + +Status plan_terms(const LogicalIndexReader& idx, const std::vector& terms, + io::BatchRangeFetcher* fetcher, std::vector* plans, bool* all_present, + bool need_positions) { + *all_present = true; + plans->resize(terms.size()); + for (size_t i = 0; i < terms.size(); ++i) { + ResolvedQueryTerm resolved; + bool found = false; + RETURN_IF_ERROR(resolve_query_term(idx, terms[i], &resolved, &found)); + if (!found) { + *all_present = false; + return Status::OK(); + } + TermPlan& p = (*plans)[i]; + p.order = i; + p.entry = std::move(resolved.entry); + p.frq_base = resolved.frq_base; + p.prx_base = resolved.prx_base; + RETURN_IF_ERROR(configure_term_plan(idx, need_positions, fetcher, &p)); + } + return Status::OK(); +} + +Status plan_resolved_terms(const LogicalIndexReader& idx, + const std::vector& terms, + io::BatchRangeFetcher* fetcher, std::vector* plans, + bool need_positions) { + plans->resize(terms.size()); + for (size_t i = 0; i < terms.size(); ++i) { + TermPlan& p = (*plans)[i]; + p.order = i; + p.entry = terms[i].entry; + p.frq_base = terms[i].frq_base; + p.prx_base = terms[i].prx_base; + RETURN_IF_ERROR(configure_term_plan(idx, need_positions, fetcher, &p)); + } + return Status::OK(); +} + +Status open_preludes(const io::BatchRangeFetcher& fetcher, std::vector* plans, + bool need_positions) { + for (TermPlan& p : *plans) { + if (!p.windowed) continue; + RETURN_IF_ERROR(FrqPreludeReader::open(fetcher.get(p.prelude_handle), &p.prelude)); + if (need_positions && !p.prelude.has_prx()) { + return Status::Error( + "docid_conjunction: windowed prelude has no positions"); + } + } + return Status::OK(); +} + +Status inline_dd_region(const DictEntry& entry, Slice* out) { + if (entry.dd_meta.disk_len > entry.frq_bytes.size()) { + return Status::Error( + "docid_conjunction: inline dd region exceeds frq bytes"); + } + *out = Slice(entry.frq_bytes.data(), static_cast(entry.dd_meta.disk_len)); + return Status::OK(); +} + +Status build_docid_only_conjunction(const LogicalIndexReader& idx, + const io::BatchRangeFetcher& round1, + const std::vector& plans, + std::vector* candidates) { + return run_docid_only_conjunction_impl(idx, round1, plans, nullptr, candidates, nullptr); +} + +Status build_docid_only_conjunction(const LogicalIndexReader& idx, + const io::BatchRangeFetcher& round1, + const std::vector& plans, + std::vector* candidates, + std::vector* sources) { + return run_docid_only_conjunction_impl(idx, round1, plans, nullptr, candidates, sources); +} + +Status filter_docids_by_conjunction(const LogicalIndexReader& idx, + const io::BatchRangeFetcher& round1, + const std::vector& plans, + const std::vector& initial_candidates, + std::vector* candidates, + std::vector* sources) { + return run_docid_only_conjunction_impl(idx, round1, plans, &initial_candidates, candidates, + sources); +} + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/docid_posting_reader.cpp b/be/src/storage/index/snii/query/docid_posting_reader.cpp new file mode 100644 index 00000000000000..be58d24a88c869 --- /dev/null +++ b/be/src/storage/index/snii/query/docid_posting_reader.cpp @@ -0,0 +1,380 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/internal/docid_posting_reader.h" + +#include +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/reader/windowed_posting.h" + +namespace doris::snii::query::internal { + +using format::DictEntry; +using format::DictEntryEnc; +using format::DictEntryKind; +using format::FrqPreludeReader; +using format::WindowMeta; +using reader::LogicalIndexReader; + +namespace { + +Status decode_flat_docs(const DictEntry& entry, Slice dd_region, std::vector* docids) { + return format::decode_dd_region(dd_region, entry.dd_meta, + /*win_base=*/0, docids); +} + +Status decode_inline_docs(const DictEntry& entry, std::vector* docids) { + if (entry.dd_meta.disk_len > entry.frq_bytes.size()) { + return Status::Error( + "docid_posting_reader: inline dd region exceeds frq bytes"); + } + return decode_flat_docs( + entry, Slice(entry.frq_bytes.data(), static_cast(entry.dd_meta.disk_len)), + docids); +} + +Status slim_docs_fetch_len(const DictEntry& entry, uint64_t win_len, uint64_t* out) { + if (entry.frq_docs_len > win_len) { + return Status::Error( + "docid_posting_reader: slim frq_docs_len exceeds frq window"); + } + *out = entry.frq_docs_len > 0 ? entry.frq_docs_len : win_len; + return Status::OK(); +} + +Status add_u64(uint64_t lhs, uint64_t rhs, const char* message, uint64_t* out) { + if (rhs > std::numeric_limits::max() - lhs) { + return Status::Error(message); + } + *out = lhs + rhs; + return Status::OK(); +} + +Status prelude_abs(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t frq_base, + uint64_t* out) { + uint64_t with_base = 0; + RETURN_IF_ERROR(add_u64(idx.section_refs().posting_region.offset, frq_base, + "docid_posting_reader: prelude offset overflow", &with_base)); + return add_u64(with_base, entry.frq_off_delta, "docid_posting_reader: prelude offset overflow", + out); +} + +Status validate_windowed_docs_prefix(const DictEntry& entry) { + if (entry.prelude_len == 0) { + return Status::Error( + "docid_posting_reader: windowed entry has no prelude"); + } + if (entry.prelude_len > entry.frq_docs_len) { + return Status::Error( + "docid_posting_reader: prelude_len exceeds docs prefix"); + } + if (entry.frq_docs_len > entry.frq_len) { + return Status::Error( + "docid_posting_reader: docs prefix exceeds frq_len"); + } + return Status::OK(); +} + +struct FlatPlan { + size_t out_index = 0; + const DictEntry* entry = nullptr; + size_t handle = 0; +}; + +struct WindowPlan { + size_t out_index = 0; + const ResolvedDocidPosting* posting = nullptr; + size_t prefix_handle = 0; +}; + +Status plan_flat_docs(const LogicalIndexReader& idx, const ResolvedDocidPosting& posting, + io::BatchRangeFetcher* fetcher, FlatPlan* plan) { + uint64_t win_abs = 0; + uint64_t win_len = 0; + RETURN_IF_ERROR(idx.resolve_frq_window(posting.entry, posting.frq_base, &win_abs, &win_len)); + uint64_t docs_len = 0; + RETURN_IF_ERROR(slim_docs_fetch_len(posting.entry, win_len, &docs_len)); + plan->handle = fetcher->add(win_abs, docs_len); + return Status::OK(); +} + +Status plan_window_prefix(const LogicalIndexReader& idx, WindowPlan* plan, + io::BatchRangeFetcher* fetcher) { + const ResolvedDocidPosting& posting = *plan->posting; + RETURN_IF_ERROR(validate_windowed_docs_prefix(posting.entry)); + uint64_t abs = 0; + RETURN_IF_ERROR(prelude_abs(idx, posting.entry, posting.frq_base, &abs)); + plan->prefix_handle = fetcher->add(abs, posting.entry.frq_docs_len); + return Status::OK(); +} + +// Records a non-inline (flat or windowed) posting into the per-encoding plan lists, +// adding the windowed prefix range to the shared fetcher. Flat ranges are added in a +// later pass (after all preludes are registered). Shared by the batched and streamed +// docid readers so both fetch the whole OR in one round. `posting` must outlive the +// plans (its address is captured); callers pass an element of a stable vector. +Status plan_noninline_posting(const LogicalIndexReader& idx, const ResolvedDocidPosting& posting, + size_t out_index, io::BatchRangeFetcher* fetcher, + std::vector* flat_plans, + std::vector* window_plans) { + if (posting.entry.enc == DictEntryEnc::kWindowed) { + WindowPlan plan; + plan.out_index = out_index; + plan.posting = &posting; + RETURN_IF_ERROR(plan_window_prefix(idx, &plan, fetcher)); + window_plans->push_back(std::move(plan)); + return Status::OK(); + } + FlatPlan plan; + plan.out_index = out_index; + plan.entry = &posting.entry; + flat_plans->push_back(plan); + return Status::OK(); +} + +Status window_dd_slice(Slice dd_block, const WindowMeta& meta, Slice* out) { + if (meta.dd_off > dd_block.size() || meta.dd_disk_len > dd_block.size() - meta.dd_off) { + return Status::Error( + "docid_posting_reader: window dd range out of prefix"); + } + *out = dd_block.subslice(static_cast(meta.dd_off), + static_cast(meta.dd_disk_len)); + return Status::OK(); +} + +Status first_docid_in_window(const WindowMeta& meta, uint32_t window_ordinal, uint32_t* first) { + if (window_ordinal == 0) { + *first = 0; + return Status::OK(); + } + if (meta.win_base >= std::numeric_limits::max()) { + return Status::Error( + "docid_posting_reader: window base exceeds docid range"); + } + *first = static_cast(meta.win_base + 1); + if (*first > meta.last_docid) { + return Status::Error( + "docid_posting_reader: invalid window docid range"); + } + return Status::OK(); +} + +Status is_dense_full_window(const WindowMeta& meta, uint32_t window_ordinal, bool* full) { + uint32_t first = 0; + RETURN_IF_ERROR(first_docid_in_window(meta, window_ordinal, &first)); + const uint64_t width = static_cast(meta.last_docid) - first + 1; + *full = meta.doc_count == width; + return Status::OK(); +} + +Status decode_flat_plan(const io::BatchRangeFetcher& fetcher, const FlatPlan& plan, + std::vector* out) { + return decode_flat_docs(*plan.entry, fetcher.get(plan.handle), out); +} + +Status decode_window_prefix_plan(const io::BatchRangeFetcher& fetcher, const WindowPlan& plan, + DocIdSink* sink); + +Status decode_window_prefix_plan(const io::BatchRangeFetcher& fetcher, const WindowPlan& plan, + std::vector* out) { + VectorDocIdSink sink(*out); + return decode_window_prefix_plan(fetcher, plan, &sink); +} + +Status decode_window_prefix_plan(const io::BatchRangeFetcher& fetcher, const WindowPlan& plan, + DocIdSink* sink) { + const DictEntry& entry = plan.posting->entry; + const Slice prefix = fetcher.get(plan.prefix_handle); + if (entry.prelude_len > prefix.size()) { + return Status::Error( + "docid_posting_reader: short docs prefix"); + } + const size_t prelude_len = static_cast(entry.prelude_len); + FrqPreludeReader prelude; + RETURN_IF_ERROR(FrqPreludeReader::open(prefix.subslice(0, prelude_len), &prelude)); + const uint64_t dd_block_len = prelude.dd_block_len(); + if (dd_block_len > static_cast(std::numeric_limits::max()) - prelude_len) { + return Status::Error( + "docid_posting_reader: docs prefix length overflow"); + } + const size_t expected_prefix_len = prelude_len + static_cast(dd_block_len); + if (prefix.size() != expected_prefix_len) { + return Status::Error( + "docid_posting_reader: docs prefix length mismatch"); + } + const Slice dd_block = prefix.subslice(prelude_len, prefix.size() - prelude_len); + std::vector docs; + std::vector freqs; + std::vector> positions; + for (uint32_t w = 0; w < prelude.n_windows(); ++w) { + WindowMeta meta; + Slice dd_region; + RETURN_IF_ERROR(prelude.window(w, &meta)); + RETURN_IF_ERROR(window_dd_slice(dd_block, meta, &dd_region)); + bool dense_full = false; + RETURN_IF_ERROR(is_dense_full_window(meta, w, &dense_full)); + if (dense_full) { + uint32_t first = 0; + RETURN_IF_ERROR(first_docid_in_window(meta, w, &first)); + RETURN_IF_ERROR(sink->append_range(first, static_cast(meta.last_docid) + 1)); + continue; + } + docs.clear(); + freqs.clear(); + positions.clear(); + RETURN_IF_ERROR(reader::decode_window_slices( + meta, dd_region, Slice(), Slice(), /*want_positions=*/false, + /*want_freq=*/false, &docs, &freqs, &positions)); + RETURN_IF_ERROR(sink->append_sorted(docs)); + } + return Status::OK(); +} + +} // namespace + +Status read_docid_posting(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t frq_base, + uint64_t prx_base, std::vector* docids) { + if (docids == nullptr) { + return Status::Error("docid_posting_reader: null out"); + } + docids->clear(); + VectorDocIdSink sink(*docids); + return read_docid_posting(idx, entry, frq_base, prx_base, &sink); +} + +Status read_docid_posting(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t frq_base, + uint64_t prx_base, DocIdSink* sink) { + if (sink == nullptr) { + return Status::Error("docid_posting_reader: null sink"); + } + ResolvedDocidPosting posting {entry, frq_base, prx_base}; + if (posting.entry.kind == DictEntryKind::kInline) { + std::vector docs; + RETURN_IF_ERROR(decode_inline_docs(posting.entry, &docs)); + return sink->append_sorted(docs); + } + + io::BatchRangeFetcher docs_fetcher(idx.reader()); + if (posting.entry.enc == DictEntryEnc::kWindowed) { + WindowPlan plan; + plan.out_index = 0; + plan.posting = &posting; + RETURN_IF_ERROR(plan_window_prefix(idx, &plan, &docs_fetcher)); + if (docs_fetcher.pending() > 0) RETURN_IF_ERROR(docs_fetcher.fetch()); + return decode_window_prefix_plan(docs_fetcher, plan, sink); + } + + FlatPlan plan; + plan.out_index = 0; + plan.entry = &posting.entry; + RETURN_IF_ERROR(plan_flat_docs(idx, posting, &docs_fetcher, &plan)); + if (docs_fetcher.pending() > 0) RETURN_IF_ERROR(docs_fetcher.fetch()); + std::vector docs; + RETURN_IF_ERROR(decode_flat_plan(docs_fetcher, plan, &docs)); + return sink->append_sorted(docs); +} + +Status read_docid_postings_batched(const LogicalIndexReader& idx, + const std::vector& postings, + std::vector>* docids) { + if (docids == nullptr) { + return Status::Error( + "docid_posting_reader: null batched out"); + } + docids->clear(); + docids->resize(postings.size()); + + std::vector flat_plans; + std::vector window_plans; + io::BatchRangeFetcher docs_fetcher(idx.reader()); + + for (size_t i = 0; i < postings.size(); ++i) { + const ResolvedDocidPosting& posting = postings[i]; + if (posting.entry.kind == DictEntryKind::kInline) { + RETURN_IF_ERROR(decode_inline_docs(posting.entry, &(*docids)[i])); + continue; + } + RETURN_IF_ERROR( + plan_noninline_posting(idx, posting, i, &docs_fetcher, &flat_plans, &window_plans)); + } + + for (FlatPlan& plan : flat_plans) { + const ResolvedDocidPosting& posting = postings[plan.out_index]; + RETURN_IF_ERROR(plan_flat_docs(idx, posting, &docs_fetcher, &plan)); + } + if (docs_fetcher.pending() > 0) RETURN_IF_ERROR(docs_fetcher.fetch()); + + for (const FlatPlan& plan : flat_plans) { + RETURN_IF_ERROR(decode_flat_plan(docs_fetcher, plan, &(*docids)[plan.out_index])); + } + for (const WindowPlan& plan : window_plans) { + RETURN_IF_ERROR(decode_window_prefix_plan(docs_fetcher, plan, &(*docids)[plan.out_index])); + } + return Status::OK(); +} + +Status emit_docid_postings_streamed(const LogicalIndexReader& idx, + const std::vector& postings, + DocIdSink* sink) { + if (sink == nullptr) { + return Status::Error( + "docid_posting_reader: null streamed sink"); + } + + std::vector flat_plans; + std::vector window_plans; + io::BatchRangeFetcher docs_fetcher(idx.reader()); + // One scratch buffer reused across flat/inline postings: clear() keeps capacity, + // so at most one growth total instead of a fresh vector per posting. + std::vector scratch; + + for (size_t i = 0; i < postings.size(); ++i) { + const ResolvedDocidPosting& posting = postings[i]; + if (posting.entry.kind == DictEntryKind::kInline) { + scratch.clear(); + RETURN_IF_ERROR(decode_inline_docs(posting.entry, &scratch)); + RETURN_IF_ERROR(sink->append_sorted(scratch)); + continue; + } + RETURN_IF_ERROR( + plan_noninline_posting(idx, posting, i, &docs_fetcher, &flat_plans, &window_plans)); + } + + for (FlatPlan& plan : flat_plans) { + const ResolvedDocidPosting& posting = postings[plan.out_index]; + RETURN_IF_ERROR(plan_flat_docs(idx, posting, &docs_fetcher, &plan)); + } + if (docs_fetcher.pending() > 0) RETURN_IF_ERROR(docs_fetcher.fetch()); + + for (const FlatPlan& plan : flat_plans) { + scratch.clear(); + RETURN_IF_ERROR(decode_flat_plan(docs_fetcher, plan, &scratch)); + RETURN_IF_ERROR(sink->append_sorted(scratch)); + } + for (const WindowPlan& plan : window_plans) { + RETURN_IF_ERROR(decode_window_prefix_plan(docs_fetcher, plan, sink)); + } + return Status::OK(); +} + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/docid_set_ops.cpp b/be/src/storage/index/snii/query/docid_set_ops.cpp new file mode 100644 index 00000000000000..5f8931f4146e16 --- /dev/null +++ b/be/src/storage/index/snii/query/docid_set_ops.cpp @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/internal/docid_set_ops.h" + +#include +#include +#include +#include + +namespace doris::snii::query::internal { + +std::vector intersect_sorted(const std::vector& a, + const std::vector& b) { + std::vector out; + out.reserve(std::min(a.size(), b.size())); + std::set_intersection(a.begin(), a.end(), b.begin(), b.end(), std::back_inserter(out)); + return out; +} + +void union_sorted_into(std::vector* acc, const std::vector& next) { + std::vector merged; + merged.reserve(acc->size() + next.size()); + std::set_union(acc->begin(), acc->end(), next.begin(), next.end(), std::back_inserter(merged)); + *acc = std::move(merged); +} + +std::vector union_sorted_many(const std::vector>& lists, + size_t reserve_cap) { + constexpr size_t kLinearFanInMax = 8; + struct Cursor { + uint32_t docid = 0; + size_t list = 0; + size_t offset = 0; + }; + struct GreaterDocId { + bool operator()(const Cursor& a, const Cursor& b) const { return a.docid > b.docid; } + }; + + size_t non_empty = 0; + size_t total = 0; + std::priority_queue, GreaterDocId> heap; + for (size_t i = 0; i < lists.size(); ++i) { + if (lists[i].empty()) continue; + ++non_empty; + total += lists[i].size(); + heap.push(Cursor {lists[i][0], i, 0}); + } + // The union is at most `total` (exactly that for disjoint inputs); reserve to it + // so the output grows in a single allocation rather than O(log) geometric + // reallocations. Cap guards against over-reserving for heavily-overlapping inputs. + const size_t reserve_hint = std::min(total, reserve_cap); + if (non_empty == 0) return {}; + if (non_empty == 1) { + for (const std::vector& docs : lists) { + if (!docs.empty()) return docs; + } + } + + if (non_empty <= kLinearFanInMax) { + std::vector offsets(lists.size(), 0); + std::vector out; + out.reserve(reserve_hint); + bool has_last = false; + uint32_t last = 0; + for (;;) { + bool found = false; + uint32_t next = 0; + for (size_t i = 0; i < lists.size(); ++i) { + if (offsets[i] >= lists[i].size()) continue; + const uint32_t docid = lists[i][offsets[i]]; + if (!found || docid < next) { + found = true; + next = docid; + } + } + if (!found) break; + if (!has_last || next != last) { + out.push_back(next); + last = next; + has_last = true; + } + for (size_t i = 0; i < lists.size(); ++i) { + while (offsets[i] < lists[i].size() && lists[i][offsets[i]] == next) { + ++offsets[i]; + } + } + } + return out; + } + + std::vector out; + out.reserve(reserve_hint); + bool has_last = false; + uint32_t last = 0; + while (!heap.empty()) { + const Cursor cur = heap.top(); + heap.pop(); + if (!has_last || cur.docid != last) { + out.push_back(cur.docid); + last = cur.docid; + has_last = true; + } + const size_t next_offset = cur.offset + 1; + const std::vector& docs = lists[cur.list]; + if (next_offset < docs.size()) { + heap.push(Cursor {docs[next_offset], cur.list, next_offset}); + } + } + return out; +} + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/docid_sink.h b/be/src/storage/index/snii/query/docid_sink.h new file mode 100644 index 00000000000000..604d3dde8bdbb8 --- /dev/null +++ b/be/src/storage/index/snii/query/docid_sink.h @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" + +namespace doris::snii::query { + +// Bulk docid handoff for query operators. Each span is sorted ascending; callers +// that need a single vector can use VectorDocIdSink. +class DocIdSink { +public: + virtual ~DocIdSink() = default; + virtual Status append_sorted(std::span docids) = 0; + virtual Status append_range(uint32_t first, uint64_t last_exclusive) = 0; + + // True iff the sink deduplicates and globally orders on its own (e.g. a Roaring + // bitmap via addMany/addRange). For such sinks a multi-term OR can stream each + // posting straight in -- skipping the per-term vector materialization plus the + // K-way merge accumulator. Sinks that hand back a single globally-sorted, + // deduplicated vector (VectorDocIdSink) keep the default false, so callers + // materialize + merge before appending. The gate must stay conservative: + // streaming several postings into a non-dedup sink would break that contract. + virtual bool dedups() const { return false; } +}; + +class VectorDocIdSink final : public DocIdSink { +public: + explicit VectorDocIdSink(std::vector& docids) : docids_(docids) {} + + Status append_sorted(std::span docids) override { + docids_.insert(docids_.end(), docids.begin(), docids.end()); + return Status::OK(); + } + + Status append_range(uint32_t first, uint64_t last_exclusive) override { + if (last_exclusive <= first) { + return Status::OK(); + } + if (last_exclusive > static_cast(std::numeric_limits::max()) + 1) { + return Status::Error( + "docid_sink: range exceeds uint32 docid space"); + } + const uint64_t count = last_exclusive - first; + if (count > static_cast(docids_.max_size() - docids_.size())) { + return Status::Error("docid_sink: range too large"); + } + // GEOMETRIC BULK reserve -- never an exact one: append_range can be + // called once per docid run for a query, and an exact + // reserve(size()+count) caps capacity at "just enough" so the next + // append reallocates + memcpys the whole accumulated vector -- + // quadratic total memcpy across runs (same anti-pattern as the writer's + // add_nulls). Doubling on overflow keeps the O(count) amortization AND + // makes one large range pay at most one reallocation. + const size_t need = docids_.size() + static_cast(count); + if (need > docids_.capacity()) { + docids_.reserve(std::max(need, docids_.capacity() * 2)); + } + for (uint64_t docid = first; docid < last_exclusive; ++docid) { + docids_.push_back(static_cast(docid)); + } + return Status::OK(); + } + +private: + std::vector& docids_; +}; + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/docid_union.cpp b/be/src/storage/index/snii/query/docid_union.cpp new file mode 100644 index 00000000000000..f1296b18cd685e --- /dev/null +++ b/be/src/storage/index/snii/query/docid_union.cpp @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/internal/docid_union.h" + +#include + +#include "storage/index/snii/query/internal/docid_set_ops.h" + +namespace doris::snii::query::internal { + +Status build_docid_union(const reader::LogicalIndexReader& idx, + const std::vector& postings, + std::vector* out) { + if (out == nullptr) + return Status::Error("docid_union: null out"); + out->clear(); + if (postings.empty()) return Status::OK(); + + std::vector> docs_by_posting; + RETURN_IF_ERROR(read_docid_postings_batched(idx, postings, &docs_by_posting)); + *out = union_sorted_many(docs_by_posting); + return Status::OK(); +} + +Status emit_docid_union(const reader::LogicalIndexReader& idx, + const std::vector& postings, DocIdSink* sink) { + if (sink == nullptr) + return Status::Error("docid_union: null sink"); + if (postings.empty()) return Status::OK(); + // A dedup-capable sink (Roaring) orders + dedups across postings itself, so stream + // each posting straight in over a single shared fetch round -- no per-term vector + // or K-way merge accumulator. A plain (non-dedup) sink keeps the materialize+merge + // path so its single-span contract (globally sorted, deduplicated) holds. + if (sink->dedups()) { + return emit_docid_postings_streamed(idx, postings, sink); + } + std::vector acc; + RETURN_IF_ERROR(build_docid_union(idx, postings, &acc)); + if (acc.empty()) return Status::OK(); + return sink->append_sorted(acc); +} + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/docid_conjunction.h b/be/src/storage/index/snii/query/internal/docid_conjunction.h new file mode 100644 index 00000000000000..229f76da6bbf9c --- /dev/null +++ b/be/src/storage/index/snii/query/internal/docid_conjunction.h @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +namespace doris::snii::query::internal { + +struct ResolvedQueryTerm { + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; +}; + +struct TermPlan { + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + uint32_t df = 0; + size_t order = 0; + size_t frq_handle = 0; + size_t prx_handle = 0; + size_t prelude_handle = 0; + bool pod_ref = false; + bool windowed = false; + format::FrqPreludeReader prelude; +}; + +struct DocidChunk { + std::vector docids; + std::vector prx_doc_ordinals; + uint32_t prx_doc_count = 0; + bool windowed = false; + uint32_t window = 0; +}; + +struct DocidSource { + std::vector chunks; + bool docids_are_final_candidates = false; +}; + +Status resolve_query_term(const reader::LogicalIndexReader& idx, const std::string& term, + ResolvedQueryTerm* resolved, bool* found); + +Status plan_terms(const reader::LogicalIndexReader& idx, const std::vector& terms, + io::BatchRangeFetcher* fetcher, std::vector* plans, bool* all_present, + bool need_positions); + +Status plan_resolved_terms(const reader::LogicalIndexReader& idx, + const std::vector& terms, + io::BatchRangeFetcher* fetcher, std::vector* plans, + bool need_positions); + +Status open_preludes(const io::BatchRangeFetcher& fetcher, std::vector* plans, + bool need_positions); + +Status inline_dd_region(const format::DictEntry& entry, Slice* out); + +Status build_docid_only_conjunction(const reader::LogicalIndexReader& idx, + const io::BatchRangeFetcher& round1, + const std::vector& plans, + std::vector* candidates); + +Status build_docid_only_conjunction(const reader::LogicalIndexReader& idx, + const io::BatchRangeFetcher& round1, + const std::vector& plans, + std::vector* candidates, + std::vector* sources); + +Status filter_docids_by_conjunction(const reader::LogicalIndexReader& idx, + const io::BatchRangeFetcher& round1, + const std::vector& plans, + const std::vector& initial_candidates, + std::vector* candidates, + std::vector* sources); + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/docid_posting_reader.h b/be/src/storage/index/snii/query/internal/docid_posting_reader.h new file mode 100644 index 00000000000000..0230a1f526039a --- /dev/null +++ b/be/src/storage/index/snii/query/internal/docid_posting_reader.h @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +namespace doris::snii::query::internal { + +struct ResolvedDocidPosting { + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; +}; + +// Decodes the docid-only posting for a resolved term. The caller owns term +// lookup and can batch/plan lookups independently; this module owns only the +// three posting encodings (inline, slim pod_ref, windowed pod_ref). +Status read_docid_posting(const reader::LogicalIndexReader& idx, const format::DictEntry& entry, + uint64_t frq_base, uint64_t prx_base, std::vector* docids); + +Status read_docid_posting(const reader::LogicalIndexReader& idx, const format::DictEntry& entry, + uint64_t frq_base, uint64_t prx_base, query::DocIdSink* sink); + +// Batch counterpart for multi-term docid-only operators. Windowed terms share one +// prelude fetch round and one docid fetch round, so OR-style operators pay by +// stage rather than by term. +Status read_docid_postings_batched(const reader::LogicalIndexReader& idx, + const std::vector& postings, + std::vector>* docids); + +// Streaming counterpart of read_docid_postings_batched for a dedup-capable sink +// (DocIdSink::dedups()==true, e.g. a Roaring bitmap). Shares the exact same single +// docid fetch round, but decodes each posting straight into the sink -- dense-full +// windows via append_range (run-preserving), the rest via append_sorted from one +// reused scratch buffer -- so no per-term vector or K-way merge accumulator is +// materialized. The sink dedups/orders across postings. One I/O round is preserved. +Status emit_docid_postings_streamed(const reader::LogicalIndexReader& idx, + const std::vector& postings, + query::DocIdSink* sink); + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/docid_set_ops.h b/be/src/storage/index/snii/query/internal/docid_set_ops.h new file mode 100644 index 00000000000000..f651e930a1cdb3 --- /dev/null +++ b/be/src/storage/index/snii/query/internal/docid_set_ops.h @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +namespace doris::snii::query::internal { + +std::vector intersect_sorted(const std::vector& a, + const std::vector& b); + +void union_sorted_into(std::vector* acc, const std::vector& next); + +// Sorted-deduplicated union of many sorted lists. The output is reserved by the +// summed input size (the union is at most the total of all inputs; for disjoint +// inputs it is exactly that), capped by `reserve_cap` so heavily-overlapping +// inputs (union << total) do not over-reserve. Default cap = no cap. +std::vector union_sorted_many(const std::vector>& lists, + size_t reserve_cap = std::numeric_limits::max()); + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/docid_union.h b/be/src/storage/index/snii/query/internal/docid_union.h new file mode 100644 index 00000000000000..3243c082bbeec2 --- /dev/null +++ b/be/src/storage/index/snii/query/internal/docid_union.h @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +namespace doris::snii::query::internal { + +// Reads already-resolved docid postings in planned batches, merges them as a +// sorted deduplicated union, then emits one bulk span to the sink. +Status build_docid_union(const reader::LogicalIndexReader& idx, + const std::vector& postings, + std::vector* out); + +Status emit_docid_union(const reader::LogicalIndexReader& idx, + const std::vector& postings, DocIdSink* sink); + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/position_math.h b/be/src/storage/index/snii/query/internal/position_math.h new file mode 100644 index 00000000000000..6db2a0ee4b599b --- /dev/null +++ b/be/src/storage/index/snii/query/internal/position_math.h @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +namespace doris::snii::query::internal { + +inline bool build_position_offsets(size_t count, std::vector* out) { + if (count >= std::numeric_limits::max()) { + return false; + } + out->clear(); + out->reserve(count); + uint32_t offset = 0; + while (out->size() < count) { + out->push_back(offset); + ++offset; + } + return true; +} + +inline bool add_position_offset(uint32_t start, uint32_t offset, uint32_t* out) { + if (start > std::numeric_limits::max() - offset) return false; + *out = start + offset; + return true; +} + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/query_test_counters.h b/be/src/storage/index/snii/query/internal/query_test_counters.h new file mode 100644 index 00000000000000..24c6bbf243e390 --- /dev/null +++ b/be/src/storage/index/snii/query/internal/query_test_counters.h @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +// Deterministic op-count seam for the phrase-query hot path (T24/G01). The +// counters let the phrase UTs assert routing/complexity directly: +// - expected_docids_build : how many times the multi-tail phrase-prefix branch +// materializes the expected-docid vector for a query. +// Hoisted out of the per-tail loop, so == 1 per query +// (was == tail_hits when rebuilt inside the loop). +// - anchor_iterations : total sparsest-anchor outer-enumeration size summed +// over candidate docs (== docs x min_span). Anchoring +// on the shortest per-doc span instead of the hardcoded +// phrase-position-0 span shrinks this when the leading +// exact term is high-frequency. +// - bigram_hits : 2-term phrases answered DIRECTLY by a hidden +// phrase-bigram posting (dict hit). +// - bigram_fallbacks : 2-term phrases whose bigram term MISSED the dict on +// a bigram-df-PRUNED segment (meta declares a non-zero +// threshold), rerouted to the generic positions- +// verification path, plus fresh deferred segments +// that enter the same path from resident metadata. +// Stays 0 on legacy (unpruned) segments, where a +// miss keeps meaning "empty result". +// - bigram_probe_attempts : hidden pair dictionary probes attempted by the +// 2-term fast path. Fresh deferred segments must +// keep this at 0: their resident capability flag +// makes every pair probe provably unnecessary. +// - count_fastpath_hits : count-only (G02) answers produced from dict-entry +// df alone -- single-term df or a 2-term phrase +// bigram-dict-HIT df -- with NO posting decode +// (count_query.cpp). Guard fall-throughs (pruned or +// absent bigram, non-count context upstream) leave +// it unchanged. +// +// The seam is active only under SNII_QUERY_TEST_COUNTERS, which is auto-enabled by +// the library-wide BE_TEST define (be/CMakeLists.txt `if (MAKE_TEST)`) used to +// build doris_be_test. Because BE_TEST is applied to the whole BE tree, both the +// phrase_query.cpp increments AND the test translation unit that reads them observe +// the SAME process-wide singleton (the inline function below has one instance +// across every including TU). In a release build BE_TEST is undefined, the struct +// and singleton do not exist, and SNII_QUERY_COUNT/SNII_QUERY_ADD expand to +// ((void)0): zero overhead and NO global mutable state on the production query +// path. +// +// CONCURRENCY: the singleton is intentionally unsynchronized. It is a +// single-threaded, test-only seam -- one phrase query at a time -- and is never +// touched on the production path. Do NOT read or write it from concurrent tests. +// Reset it between test cases with `query_test_counters() = {}`. +#if defined(BE_TEST) && !defined(SNII_QUERY_TEST_COUNTERS) +#define SNII_QUERY_TEST_COUNTERS +#endif + +#ifdef SNII_QUERY_TEST_COUNTERS + +namespace doris::snii::query::internal { + +struct QueryTestCounters { + uint64_t expected_docids_build = 0; + uint64_t anchor_iterations = 0; + uint64_t bigram_hits = 0; + uint64_t bigram_fallbacks = 0; + uint64_t bigram_probe_attempts = 0; + uint64_t count_fastpath_hits = 0; +}; + +// `inline` gives a single shared instance across all TUs that include this header +// (phrase_query.cpp and the test), so counter increments made in the library are +// visible to the test that reads them. +inline QueryTestCounters& query_test_counters() { + static QueryTestCounters counters; + return counters; +} + +} // namespace doris::snii::query::internal + +// NOLINTBEGIN(clang-diagnostic-unused-macros): expanded by phrase_query.cpp, not by this header's TU +#define SNII_QUERY_COUNT(field) (++::doris::snii::query::internal::query_test_counters().field) +#define SNII_QUERY_ADD(field, n) \ + (::doris::snii::query::internal::query_test_counters().field += (n)) +// NOLINTEND(clang-diagnostic-unused-macros) + +#else + +#define SNII_QUERY_COUNT(field) ((void)0) +#define SNII_QUERY_ADD(field, n) ((void)0) + +#endif // SNII_QUERY_TEST_COUNTERS diff --git a/be/src/storage/index/snii/query/internal/regex_prefix.h b/be/src/storage/index/snii/query/internal/regex_prefix.h new file mode 100644 index 00000000000000..8fa872766b1e1d --- /dev/null +++ b/be/src/storage/index/snii/query/internal/regex_prefix.h @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +namespace doris::snii::query::internal { + +// Computes the dictionary-enumeration prefix used to narrow regexp term +// expansion. For left-anchored ("^") patterns it derives a tight common prefix +// from RE2::PossibleMatchRange (e.g. "^(order)" -> "order", where a naive literal +// scan stops at '(' and yields ""); otherwise it falls back to a conservative +// literal-prefix scan. The returned prefix only bounds how many dictionary terms +// visit_prefix_terms enumerates -- final term acceptance is always decided by +// RE2::FullMatch -- so an over-wide prefix can only enumerate extra terms and +// never drops a real match. `re` must be the compiled pattern (re.ok()). +std::string regex_enum_prefix(std::string_view pattern, const re2::RE2& re); + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/term_expansion.h b/be/src/storage/index/snii/query/internal/term_expansion.h new file mode 100644 index 00000000000000..7db5474d50bfff --- /dev/null +++ b/be/src/storage/index/snii/query/internal/term_expansion.h @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +namespace doris::snii::query::internal { + +using TermMatcher = std::function; + +// Enumerates dictionary terms from `enum_prefix`, filters them with `matches`, +// and emits the sorted docid union for matching entries. PrefixHit carries the +// DictEntry and block bases, so callers avoid a second lookup per expanded term. +Status emit_expanded_docid_union(const reader::LogicalIndexReader& idx, + std::string_view enum_prefix, const TermMatcher& matches, + DocIdSink* const sink, int32_t max_expansions = 0); + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/internal/wildcard_matcher.h b/be/src/storage/index/snii/query/internal/wildcard_matcher.h new file mode 100644 index 00000000000000..78eec25097a997 --- /dev/null +++ b/be/src/storage/index/snii/query/internal/wildcard_matcher.h @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace doris::snii::query::internal { + +// Glob matcher with reusable scratch. '*' matches >=0 bytes, '?' matches exactly +// one byte, every other byte is literal; matching is anchored at both ends (full +// match). The matching result is bit-for-bit identical to the former per-call DP +// in wildcard_query.cpp: the only change is that the two DP rows are constructed +// once and reused (assign(), never reallocated once capacity is large enough) +// across every term in a single expansion. A whole-dictionary scan therefore +// performs O(1) heap allocations for scratch instead of O(2N) -- two small +// std::vector constructions per visited term. +// +// The allocator is templated only so deterministic allocation-counting tests can +// inject a CountingAllocator; production constructs WildcardMatcher<> (default +// std::allocator). The matcher is request-scoped (a stack local of the calling +// wildcard_query frame), holds no shared mutable state, and is not thread-safe by +// design: each query owns its own instance. +template > +class WildcardMatcher { +public: + explicit WildcardMatcher(std::string_view pattern) : pattern_(pattern) {} + + bool operator()(std::string_view text) { + const size_t n = text.size() + 1; + prev_.assign(n, 0); // reuses the buffer; no realloc once capacity >= n + curr_.assign(n, 0); + prev_[0] = 1; + for (char p : pattern_) { + std::fill(curr_.begin(), curr_.end(), 0); + if (p == '*') { + curr_[0] = prev_[0]; + for (size_t i = 1; i < n; ++i) { + curr_[i] = prev_[i] || curr_[i - 1]; + } + } else { + for (size_t i = 1; i < n; ++i) { + curr_[i] = prev_[i - 1] && (p == '?' || p == text[i - 1]); + } + } + prev_.swap(curr_); + } + return prev_[text.size()] != 0; + } + + // Test-only debug accessor: the production path never depends on it. Reports + // the larger of the two scratch-row capacities so perf tests can assert the + // buffer stops reallocating after warmup. + size_t scratch_capacity() const { return std::max(prev_.capacity(), curr_.capacity()); } + +private: + std::string_view pattern_; + std::vector prev_; + std::vector curr_; +}; + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/phrase_query.cpp b/be/src/storage/index/snii/query/phrase_query.cpp new file mode 100644 index 00000000000000..a83cbfeb60618f --- /dev/null +++ b/be/src/storage/index/snii/query/phrase_query.cpp @@ -0,0 +1,1633 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/phrase_query.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/query/internal/docid_conjunction.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_set_ops.h" +#include "storage/index/snii/query/internal/docid_union.h" +#include "storage/index/snii/query/internal/position_math.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/windowed_posting.h" + +// phrase_query implements MATCH_PHRASE with WINDOW (sub-block) SKIPPING for +// high-df windowed terms (design spec section 6.2): +// 1. Resolve every term; reject if any is absent. +// 2. Batch-read each windowed term's prelude + each slim/inline term's full +// docid posting in one round; open the two-level prelude readers. +// 3. Pick the DRIVER = smallest-df term; materialize it fully -> the initial +// candidate docid set. +// 4. For every other term in ascending-df order, narrow the candidate set: +// - slim/inline: intersect with its (already decoded) full posting. +// - windowed: locate_window() the CURRENT candidates -> the SET of +// windows covering them; batch-fetch ONLY those windows' +// .frq docid regions; keep candidates present in some +// covering window. A high-df term thus reads +// O(candidates) windows instead of its whole O(df) +// posting. +// 5. Fetch PRX only for retained chunks and run the positional phrase check +// (term[0]@p, term[1]@p+1, ...) on the survivors. +// The result is identical to a full-read intersection; only the bytes read for +// high-df windowed terms shrink. +namespace doris::snii::query { + +using query::internal::DocidChunk; +using query::internal::DocidSource; +using query::internal::ResolvedQueryTerm; +using query::internal::TermPlan; +using reader::LogicalIndexReader; + +namespace { + +struct ExpectedTailPositions { + uint32_t docid = 0; + size_t positions_begin = 0; + size_t positions_end = 0; +}; + +struct ExpectedTailPositionSet { + std::vector docs; + std::vector positions; + + void clear() { + docs.clear(); + positions.clear(); + } + + void reserve_docs(size_t count) { + docs.reserve(count); + positions.reserve(count); + } +}; + +// One decoded chunk of a term's posting: a windowed term's covering window, or +// a slim/inline term's single posting. `docids` is decoded in the conjunction +// phase (and reused by the streaming cursor -- the dd region is decoded exactly +// once); `prx` is the on-disk positions bytes, decoded lazily by the cursor +// (once per chunk) during phrase verification. +struct PosChunk { + std::vector docids; // ascending, absolute + // Empty means the chunk keeps every PRX doc in on-disk order. Non-empty means + // `docids[i]` corresponds to on-disk local document ordinal + // `prx_doc_ordinals[i]`, allowing PRX decode to skip positions for docs that + // were removed by the docid-only conjunction. + std::vector prx_doc_ordinals; + uint32_t prx_doc_count = 0; + Slice prx; // .prx window bytes (reference fetcher/round1/entry) + bool windowed = false; + uint32_t window = 0; +}; + +// A term's retained posting as an ordered list of chunks (windowed: covering +// windows in docid order; slim/inline: one). The referenced prx bytes live in +// `round1` / the per-term fetchers kept alive in phrase_query::owners for the +// whole query, so the cursor can decode positions during verification. +struct PosSource { + std::vector chunks; +}; + +struct PhraseExecutionState { + std::vector srcs; + std::vector> owners; + std::vector candidates; +}; + +struct PhraseTermMapping { + std::vector unique_terms; + std::vector phrase_plan_index; +}; + +PhraseTermMapping BuildPhraseTermMapping(const std::vector& terms) { + PhraseTermMapping mapping; + mapping.phrase_plan_index.reserve(terms.size()); + for (const std::string& term : terms) { + auto it = std::ranges::find(mapping.unique_terms, term); + if (it == mapping.unique_terms.end()) { + mapping.phrase_plan_index.push_back(mapping.unique_terms.size()); + mapping.unique_terms.push_back(term); + continue; + } + mapping.phrase_plan_index.push_back(static_cast(it - mapping.unique_terms.begin())); + } + return mapping; +} + +Status phrase_bigram_enabled(const LogicalIndexReader& idx, bool* enabled) { + ResolvedQueryTerm sentinel; + return internal::resolve_query_term(idx, format::make_phrase_bigram_sentinel_term(), &sentinel, + enabled); +} + +Status TryTwoTermPhraseBigram(const LogicalIndexReader& idx, const std::vector& terms, + std::vector* const docids, bool* handled) { + *handled = false; + if (terms.size() != 2) { + return Status::OK(); + } + if (!format::is_phrase_bigram_indexable_term(terms[0]) || + !format::is_phrase_bigram_indexable_term(terms[1])) { + return Status::OK(); + } + // A fresh direct-load segment persisted this capability in its resident + // per-index meta. It has no pair postings or sentinel by construction, so a + // synthetic pair lookup cannot help and can become one lookup per tail for + // MATCH_PHRASE_PREFIX. Go straight to the existing positions fallback. + if (idx.phrase_bigrams_deferred()) { + SNII_QUERY_COUNT(bigram_fallbacks); + return Status::OK(); + } + + ResolvedQueryTerm resolved; + bool found = false; + SNII_QUERY_COUNT(bigram_probe_attempts); + RETURN_IF_ERROR(internal::resolve_query_term( + idx, format::make_phrase_bigram_term(terms[0], terms[1]), &resolved, &found)); + if (found) { + // Docid membership IS the phrase answer (the pair was adjacent when the + // bigram token was emitted); positions are never read here, which is what + // lets the writer store bigram postings docs-only (G01 part B). + SNII_QUERY_COUNT(bigram_hits); + *handled = true; + return internal::read_docid_posting(idx, resolved.entry, resolved.frq_base, + resolved.prx_base, docids); + } + + // Bigram dict MISS. When THIS segment's meta declares bigram df-pruning + // (G01 min and/or G15 max), the miss is ambiguous -- the pair may have been + // pruned (df below the min or above the max threshold) rather than absent + // -- so fall back to the generic positions-verification phrase path + // (*handled stays false). The fallback is the full-fidelity implementation. + // A min-pruned pair is low-df by definition, so its candidate set is small; + // a max-pruned (stopword-like) pair instead verifies positions over a + // LARGE unigram candidate set -- the deliberate G15 trade: those pairs' + // dict + posting bytes buy almost no selectivity, and the ratio config + // keeps the gate to near-ubiquitous pairs. Segments WITHOUT the meta field + // keep the legacy contract: the writer materialized EVERY adjacent pair, so + // miss == no adjacency == empty result. + if (idx.bigram_prune_min_df() > 0 || idx.bigram_prune_max_df() > 0) { + SNII_QUERY_COUNT(bigram_fallbacks); + return Status::OK(); + } + + bool enabled = false; + RETURN_IF_ERROR(phrase_bigram_enabled(idx, &enabled)); + if (!enabled) { + return Status::OK(); + } + docids->clear(); + *handled = true; + return Status::OK(); +} + +Status append_prx_doc_ordinal(size_t ordinal, std::vector* out) { + if (ordinal > std::numeric_limits::max()) { + return Status::Error( + "phrase_query: prx doc ordinal exceeds u32"); + } + out->push_back(static_cast(ordinal)); + return Status::OK(); +} + +Status append_selected_ordinal(size_t doc_index, const std::vector& prx_doc_ordinals, + std::vector* selected_ordinals) { + if (!prx_doc_ordinals.empty()) { + selected_ordinals->push_back(prx_doc_ordinals[doc_index]); + return Status::OK(); + } + return append_prx_doc_ordinal(doc_index, selected_ordinals); +} + +Status append_selected_doc(size_t doc_index, uint32_t docid, + const std::vector& prx_doc_ordinals, + std::vector* selected_docids, + std::vector* selected_ordinals) { + selected_docids->push_back(docid); + return append_selected_ordinal(doc_index, prx_doc_ordinals, selected_ordinals); +} + +Status materialize_selected_prefix(size_t count, size_t capacity, + const std::vector& docids, + const std::vector& prx_doc_ordinals, + std::vector* selected_docids, + std::vector* selected_ordinals) { + selected_docids->reserve(capacity); + selected_ordinals->reserve(capacity); + selected_docids->insert(selected_docids->end(), docids.begin(), docids.begin() + count); + for (size_t i = 0; i < count; ++i) { + RETURN_IF_ERROR(append_selected_ordinal(i, prx_doc_ordinals, selected_ordinals)); + } + return Status::OK(); +} + +Status materialize_selected_prefix_if_needed(bool* selected_all, size_t count, size_t capacity, + const std::vector& docids, + const std::vector& prx_doc_ordinals, + std::vector* selected_docids, + std::vector* selected_ordinals) { + if (!*selected_all) { + return Status::OK(); + } + *selected_all = false; + return materialize_selected_prefix(count, capacity, docids, prx_doc_ordinals, selected_docids, + selected_ordinals); +} + +Status SelectCandidateDocsForPrx(std::vector* docids, + std::vector* prx_doc_ordinals, uint32_t prx_doc_count, + const std::vector& candidates, PosChunk* chunk) { + chunk->docids.clear(); + chunk->prx_doc_ordinals.clear(); + if (prx_doc_count == 0 && docids->size() > std::numeric_limits::max()) { + return Status::Error( + "phrase_query: prx doc count exceeds u32"); + } + chunk->prx_doc_count = + prx_doc_count == 0 ? static_cast(docids->size()) : prx_doc_count; + if (docids->empty() || candidates.empty()) { + return Status::OK(); + } + if (!prx_doc_ordinals->empty() && prx_doc_ordinals->size() != docids->size()) { + return Status::Error( + "phrase_query: prx ordinal/docid count mismatch"); + } + + std::vector selected_docids; + std::vector selected_ordinals; + bool selected_all = true; + const size_t selected_capacity = std::min(docids->size(), candidates.size()); + + auto candidate_it = std::ranges::lower_bound(candidates, docids->front()); + size_t candidate_index = static_cast(candidate_it - candidates.begin()); + for (size_t doc_index = 0; doc_index < docids->size(); ++doc_index) { + const uint32_t docid = (*docids)[doc_index]; + while (candidate_index < candidates.size() && candidates[candidate_index] < docid) { + ++candidate_index; + } + if (candidate_index == candidates.size()) { + RETURN_IF_ERROR(materialize_selected_prefix_if_needed( + &selected_all, doc_index, selected_capacity, *docids, *prx_doc_ordinals, + &selected_docids, &selected_ordinals)); + break; + } + if (candidates[candidate_index] != docid) { + RETURN_IF_ERROR(materialize_selected_prefix_if_needed( + &selected_all, doc_index, selected_capacity, *docids, *prx_doc_ordinals, + &selected_docids, &selected_ordinals)); + continue; + } + + if (!selected_all) { + RETURN_IF_ERROR(append_selected_doc(doc_index, docid, *prx_doc_ordinals, + &selected_docids, &selected_ordinals)); + } + ++candidate_index; + } + + if (selected_all) { + chunk->docids = std::move(*docids); + chunk->prx_doc_ordinals = std::move(*prx_doc_ordinals); + docids->clear(); + prx_doc_ordinals->clear(); + return Status::OK(); + } + if (selected_docids.empty()) { + return Status::OK(); + } + chunk->docids = std::move(selected_docids); + chunk->prx_doc_ordinals = std::move(selected_ordinals); + return Status::OK(); +} + +// PRX byte ranges for every candidate-bearing chunk across all phrase terms are +// added to one shared BatchRangeFetcher and fetched in a single batched round +// (T02). Pass 1 records, for each chunk that needs on-disk PRX bytes, where to +// write the fetched slice back: which plan's PosSource, which chunk within it, +// and the fetcher handle. +struct PrxRangeAssignment { + size_t plan_index; + size_t chunk_index; + size_t handle; +}; + +void record_prx_assignment(std::vector* assignments, size_t plan_index, + size_t chunk_index, size_t handle) { + assignments->push_back(PrxRangeAssignment { + .plan_index = plan_index, .chunk_index = chunk_index, .handle = handle}); +} + +Status BuildFlatPositionSource(const LogicalIndexReader& idx, const io::BatchRangeFetcher& round1, + DocidSource* doc_source, const TermPlan& p, + const std::vector& candidates, size_t plan_index, + io::BatchRangeFetcher* prx_fetcher, + std::vector* assignments, PosSource* src) { + PosChunk chunk; + std::vector docids; + std::vector prx_doc_ordinals; + const bool docids_are_final_candidates = + doc_source->docids_are_final_candidates && !doc_source->chunks.empty(); + if (!doc_source->chunks.empty()) { + DocidChunk& doc_chunk = doc_source->chunks.front(); + docids = std::move(doc_chunk.docids); + prx_doc_ordinals = std::move(doc_chunk.prx_doc_ordinals); + chunk.prx_doc_count = doc_chunk.prx_doc_count; + } + // pod_ref PRX bytes are read from the shared fetcher (one batched round for the + // whole phrase); inline PRX bytes already live in the dict entry. The pod_ref + // range is added unconditionally to keep the bytes read identical to the prior + // per-term fetch(); the handle is only recorded as an assignment when the chunk + // is kept (an empty chunk reads the same bytes but needs no backfill). + bool has_prx_handle = false; + size_t prx_handle = 0; + if (p.pod_ref) { + uint64_t poff = 0; + uint64_t plen = 0; + RETURN_IF_ERROR(idx.resolve_prx_window(p.entry, p.prx_base, &poff, &plen)); + prx_handle = prx_fetcher->add(poff, plen); + has_prx_handle = true; + } else { + chunk.prx = Slice(p.entry.prx_bytes); + } + if (docids.empty()) { + Slice dd; + if (p.pod_ref) { + dd = round1.get(p.frq_handle); + } else { + RETURN_IF_ERROR(internal::inline_dd_region(p.entry, &dd)); + } + RETURN_IF_ERROR(format::decode_dd_region(dd, p.entry.dd_meta, + /*win_base=*/0, &docids)); + if (docids.size() > std::numeric_limits::max()) { + return Status::Error( + "phrase_query: prx doc count exceeds u32"); + } + chunk.prx_doc_count = static_cast(docids.size()); + } + if (docids_are_final_candidates) { + chunk.docids = std::move(docids); + chunk.prx_doc_ordinals = std::move(prx_doc_ordinals); + if (!chunk.docids.empty()) { + if (has_prx_handle) { + record_prx_assignment(assignments, plan_index, src->chunks.size(), prx_handle); + } + src->chunks.push_back(std::move(chunk)); + } + return Status::OK(); + } + RETURN_IF_ERROR(SelectCandidateDocsForPrx(&docids, &prx_doc_ordinals, chunk.prx_doc_count, + candidates, &chunk)); + if (!chunk.docids.empty()) { + if (has_prx_handle) { + record_prx_assignment(assignments, plan_index, src->chunks.size(), prx_handle); + } + src->chunks.push_back(std::move(chunk)); + } + return Status::OK(); +} + +bool ChunkMayContainCandidate(const DocidChunk& chunk, const std::vector& candidates) { + if (chunk.docids.empty() || candidates.empty()) { + return false; + } + const auto it = std::ranges::lower_bound(candidates, chunk.docids.front()); + return it != candidates.end() && *it <= chunk.docids.back(); +} + +Status DecodeWindowedPositionSource(const LogicalIndexReader& idx, const TermPlan& p, + DocidSource* doc_source, + const std::vector& candidates, size_t plan_index, + io::BatchRangeFetcher* prx_fetcher, + std::vector* assignments, PosSource* src) { + for (size_t i = 0; i < doc_source->chunks.size(); ++i) { + DocidChunk& doc_chunk = doc_source->chunks[i]; + if (!doc_source->docids_are_final_candidates && + !ChunkMayContainCandidate(doc_chunk, candidates)) { + continue; + } + if (!doc_chunk.windowed) { + return Status::Error( + "phrase_query: expected windowed doc chunk"); + } + PosChunk chunk; + if (doc_source->docids_are_final_candidates) { + chunk.docids = std::move(doc_chunk.docids); + chunk.prx_doc_ordinals = std::move(doc_chunk.prx_doc_ordinals); + chunk.prx_doc_count = doc_chunk.prx_doc_count; + } else { + RETURN_IF_ERROR(SelectCandidateDocsForPrx(&doc_chunk.docids, + &doc_chunk.prx_doc_ordinals, + doc_chunk.prx_doc_count, candidates, &chunk)); + } + if (chunk.docids.empty()) { + continue; + } + + reader::WindowAbsRange range; + RETURN_IF_ERROR(reader::windowed_window_range( + idx, p.entry, p.frq_base, p.prx_base, p.prelude, doc_chunk.window, + /*want_positions=*/true, /*want_freq=*/false, &range)); + chunk.windowed = true; + chunk.window = doc_chunk.window; + const size_t prx_handle = prx_fetcher->add(range.prx_off, range.prx_len); + record_prx_assignment(assignments, plan_index, src->chunks.size(), prx_handle); + src->chunks.push_back(std::move(chunk)); + } + return Status::OK(); +} + +Status BuildPositionSourcesForCandidates( + const LogicalIndexReader& idx, const io::BatchRangeFetcher& round1, + const std::vector& plans, std::vector* doc_sources, + const std::vector& candidates, + std::vector>* owners, std::vector* srcs) { + srcs->assign(plans.size(), PosSource {}); + // All phrase terms share one PRX fetcher: pass 1 adds every candidate-bearing + // chunk's PRX range and records a backfill assignment; a single fetch() then + // issues one batched read (one serial round on a remote reader); pass 2 fills + // in each chunk's PRX slice. This collapses the prior per-term fetch() -- O(n) + // serial remote rounds for an n-term phrase -- into one. + auto prx_fetcher = + std::make_unique(idx.reader(), reader::kSameTermCoalesceGap); + std::vector assignments; + for (size_t i = 0; i < plans.size(); ++i) { + const TermPlan& p = plans[i]; + if (p.windowed) { + RETURN_IF_ERROR(DecodeWindowedPositionSource(idx, p, &(*doc_sources)[i], candidates, i, + prx_fetcher.get(), &assignments, + &(*srcs)[i])); + continue; + } + RETURN_IF_ERROR(BuildFlatPositionSource(idx, round1, &(*doc_sources)[i], p, candidates, i, + prx_fetcher.get(), &assignments, &(*srcs)[i])); + } + if (prx_fetcher->pending() > 0) { + RETURN_IF_ERROR(prx_fetcher->fetch()); + } + for (const PrxRangeAssignment& a : assignments) { + (*srcs)[a.plan_index].chunks[a.chunk_index].prx = prx_fetcher->get(a.handle); + } + // Keep the fetcher alive only when some chunk slice references its buffers. + if (!assignments.empty()) { + owners->push_back(std::move(prx_fetcher)); + } + return Status::OK(); +} + +class PosChunkDecoder { +public: + void reset() { + chunk_ = nullptr; + offsets_by_prx_ordinal_ = false; + } + + Status decode(const PosChunk& chunk) { + chunk_ = &chunk; + ByteSource ps(chunk.prx); + offsets_by_prx_ordinal_ = false; + if (chunk.prx_doc_ordinals.empty()) { + RETURN_IF_ERROR(format::read_prx_window_csr(&ps, &pflat_, &poff_)); + } else if (should_decode_full_prx_window(chunk)) { + RETURN_IF_ERROR(format::read_prx_window_csr(&ps, &pflat_, &poff_)); + offsets_by_prx_ordinal_ = true; + } else { + RETURN_IF_ERROR(format::read_prx_window_csr_selective(&ps, chunk.prx_doc_ordinals, + &pflat_, &poff_)); + } + if (offsets_by_prx_ordinal_) { + if (poff_.size() != static_cast(chunk.prx_doc_count) + 1) { + return Status::Error( + "phrase_query: full prx doc-count mismatch"); + } + } else if (poff_.size() != chunk.docids.size() + 1) { + return Status::Error( + "phrase_query: selected prx/doc-count mismatch"); + } + if (poff_.back() > pflat_.size()) { + return Status::Error( + "phrase_query: prx final offset out of range"); + } + return Status::OK(); + } + + Status positions(size_t doc_index, std::pair* out) const { + if (chunk_ == nullptr || doc_index >= chunk_->docids.size()) { + return Status::Error( + "phrase_query: decoded chunk doc index out of range"); + } + const size_t pos_index = + offsets_by_prx_ordinal_ ? chunk_->prx_doc_ordinals[doc_index] : doc_index; + if (pos_index + 1 >= poff_.size()) { + return Status::Error( + "phrase_query: prx ordinal offset out of range"); + } + const uint32_t begin = poff_[pos_index]; + const uint32_t end = poff_[pos_index + 1]; + if (begin == end) { + *out = {nullptr, nullptr}; + return Status::OK(); + } + if (end > pflat_.size()) { + return Status::Error( + "phrase_query: prx offset out of range"); + } + *out = {pflat_.data() + begin, pflat_.data() + end}; + return Status::OK(); + } + + inline __attribute__((always_inline)) std::pair + positions_unchecked(size_t doc_index) const { + const size_t pos_index = + offsets_by_prx_ordinal_ ? chunk_->prx_doc_ordinals[doc_index] : doc_index; + const uint32_t begin = poff_[pos_index]; + const uint32_t end = poff_[pos_index + 1]; + if (begin == end) { + return {nullptr, nullptr}; + } + return {pflat_.data() + begin, pflat_.data() + end}; + } + +private: + static bool should_decode_full_prx_window(const PosChunk& chunk) { + return chunk.prx_doc_count != 0 && + static_cast(chunk.prx_doc_ordinals.size()) * 2 >= chunk.prx_doc_count; + } + + const PosChunk* chunk_ = nullptr; + bool offsets_by_prx_ordinal_ = false; + std::vector pflat_; + std::vector poff_; +}; + +// Streaming position cursor over one term's retained chunks. It advances ONLY +// forward (callers seek ascending candidate docids), decodes each chunk's +// docids once (reused from the conjunction phase) and each chunk's positions at +// most once (lazily, into a flat CSR whose capacity is retained across chunks). +// No per-doc allocation, no per-candidate docid binary search: positions are +// addressed by the doc's local index within its chunk. This is the read-side +// dual of the windowed posting layout -- the S3-native batch fetch already +// pulled every needed chunk into memory; the cursor is pure in-memory column +// iteration. +class PostingCursor { +public: + void init(const PosSource* src) { + src_ = src; + ci_ = 0; + li_ = 0; + decoded_pos_chunk_ = kNoChunk; + decoder_.reset(); + } + + // Positions the cursor at `target` (guaranteed present: candidates are the + // intersection of exactly these chunks' docids). Monotonic forward advance. + Status seek(uint32_t target) { + while (ci_ < src_->chunks.size() && + (src_->chunks[ci_].docids.empty() || src_->chunks[ci_].docids.back() < target)) { + ++ci_; + li_ = 0; + } + if (ci_ >= src_->chunks.size()) { + return Status::Error( + "phrase_query: cursor exhausted before target docid"); + } + const std::vector& d = src_->chunks[ci_].docids; + while (li_ < d.size() && d[li_] < target) { + ++li_; + } + if (li_ >= d.size() || d[li_] != target) { + return Status::Error( + "phrase_query: candidate missing from posting chunk"); + } + return Status::OK(); + } + + // [begin,end) of the current doc's positions, decoding the current chunk's + // .prx exactly once (cached). Must follow a seek that landed on a real doc. + Status positions(std::pair* out) { + if (ci_ >= src_->chunks.size() || li_ >= src_->chunks[ci_].docids.size()) { + return Status::Error( + "phrase_query: cursor positions out of range"); + } + if (decoded_pos_chunk_ != ci_) { + RETURN_IF_ERROR(decoder_.decode(src_->chunks[ci_])); + decoded_pos_chunk_ = ci_; + } + return decoder_.positions(li_, out); + } + + Status next(uint32_t* docid, std::pair* out) { + while (ci_ < src_->chunks.size() && + (src_->chunks[ci_].docids.empty() || li_ >= src_->chunks[ci_].docids.size())) { + ++ci_; + li_ = 0; + } + if (ci_ >= src_->chunks.size()) { + return Status::Error( + "phrase_query: cursor exhausted before next docid"); + } + *docid = src_->chunks[ci_].docids[li_]; + RETURN_IF_ERROR(positions(out)); + ++li_; + return Status::OK(); + } + +private: + static constexpr size_t kNoChunk = static_cast(-1); + + const PosSource* src_ = nullptr; + size_t ci_ = 0; // current chunk + size_t li_ = 0; // current local doc index within the chunk + size_t decoded_pos_chunk_ = kNoChunk; // which chunk decoder_ currently holds + PosChunkDecoder decoder_; +}; + +class PhrasePositionLoader { +public: + PhrasePositionLoader(size_t plan_count, std::vector& srcs) + : cursors_(plan_count), plan_spans_(plan_count), loaded_epoch_(plan_count, 0) { + for (size_t i = 0; i < plan_count; ++i) { + cursors_[i].init(&srcs[i]); + } + } + + void begin_doc(uint32_t docid) { + docid_ = docid; + ++epoch_; + if (epoch_ == 0) { + std::ranges::fill(loaded_epoch_, 0); + epoch_ = 1; + } + } + + Status positions_for_phrase_pos(const std::vector& phrase_plan_index, size_t phrase_pos, + std::pair* out) { + const size_t plan_index = phrase_plan_index[phrase_pos]; + if (loaded_epoch_[plan_index] != epoch_) { + RETURN_IF_ERROR(cursors_[plan_index].seek(docid_)); + RETURN_IF_ERROR(cursors_[plan_index].positions(&plan_spans_[plan_index])); + loaded_epoch_[plan_index] = epoch_; + } + *out = plan_spans_[plan_index]; + return Status::OK(); + } + +private: + std::vector cursors_; + std::vector> plan_spans_; + std::vector loaded_epoch_; + uint32_t docid_ = 0; + uint32_t epoch_ = 0; +}; + +bool ContainsTwoTermPhrase(std::pair left_span, + std::pair right_span, + uint32_t right_delta) { + const uint32_t* left = left_span.first; + const uint32_t* right = right_span.first; + if (left == left_span.second || right == right_span.second) { + return false; + } + const uint32_t max_start = std::numeric_limits::max() - right_delta; + if (left + 1 == left_span.second && right + 1 == right_span.second) { + return *left <= max_start && *right == *left + right_delta; + } + while (left != left_span.second && right != right_span.second) { + if (*left > max_start) { + return false; + } + const uint32_t want = *left + right_delta; + while (right != right_span.second && *right < want) { + ++right; + } + if (right == right_span.second) { + return false; + } + if (*right == want) { + return true; + } + ++left; + } + return false; +} + +size_t SelectPhraseVerificationPair(const std::vector& plans, + const std::vector& phrase_plan_index) { + size_t best_left = 0; + uint64_t best_score = std::numeric_limits::max(); + for (size_t left = 0; left + 1 < phrase_plan_index.size(); ++left) { + const uint64_t score = static_cast(plans[phrase_plan_index[left]].df) + + plans[phrase_plan_index[left + 1]].df; + if (score < best_score) { + best_score = score; + best_left = left; + } + } + return best_left; +} + +void CollectTwoTermPhraseStarts(std::pair left_span, + std::pair right_span, + uint32_t right_delta, uint32_t left_offset, + std::vector& starts) { + starts.clear(); + const uint32_t* left = left_span.first; + const uint32_t* right = right_span.first; + const uint32_t max_left = std::numeric_limits::max() - right_delta; + while (left != left_span.second && right != right_span.second) { + if (*left > max_left) { + return; + } + const uint32_t want = *left + right_delta; + while (right != right_span.second && *right < want) { + ++right; + } + if (right == right_span.second) { + return; + } + if (*right == want && *left >= left_offset) { + starts.push_back(*left - left_offset); + } + ++left; + } +} + +Status EmitTwoTermPhraseStreaming(const std::vector& phrase_plan_index, + const std::vector& position_offsets, + std::vector& srcs, + const std::vector& candidates, + std::vector* docids) { + const size_t left_plan = phrase_plan_index[0]; + const size_t right_plan = phrase_plan_index[1]; + const uint32_t right_delta = position_offsets[1] - position_offsets[0]; + + if (left_plan == right_plan) { + PostingCursor cursor; + cursor.init(&srcs[left_plan]); + for (uint32_t expected_docid : candidates) { + uint32_t docid = 0; + std::pair span; + RETURN_IF_ERROR(cursor.next(&docid, &span)); + if (docid != expected_docid) { + return Status::Error( + "phrase_query: repeated-term cursor/docid mismatch"); + } + if (ContainsTwoTermPhrase(span, span, right_delta)) { + docids->push_back(docid); + } + } + return Status::OK(); + } + + PostingCursor left_cursor; + PostingCursor right_cursor; + left_cursor.init(&srcs[left_plan]); + right_cursor.init(&srcs[right_plan]); + for (uint32_t expected_docid : candidates) { + uint32_t left_docid = 0; + uint32_t right_docid = 0; + std::pair left_span; + std::pair right_span; + RETURN_IF_ERROR(left_cursor.next(&left_docid, &left_span)); + RETURN_IF_ERROR(right_cursor.next(&right_docid, &right_span)); + if (left_docid != expected_docid || right_docid != expected_docid) { + return Status::Error( + "phrase_query: two-term cursor/docid mismatch"); + } + if (ContainsTwoTermPhrase(left_span, right_span, right_delta)) { + docids->push_back(expected_docid); + } + } + return Status::OK(); +} + +void EmitTwoTermPhraseChunkPair(const PosChunk& left, const PosChunk& right, + const PosChunkDecoder& left_decoder, + const PosChunkDecoder& right_decoder, uint32_t right_delta, + std::vector& docids) { + size_t li = static_cast(std::ranges::lower_bound(left.docids, right.docids.front()) - + left.docids.begin()); + size_t ri = static_cast(std::ranges::lower_bound(right.docids, left.docids.front()) - + right.docids.begin()); + while (li < left.docids.size() && ri < right.docids.size()) { + const uint32_t left_docid = left.docids[li]; + const uint32_t right_docid = right.docids[ri]; + if (left_docid < right_docid) { + ++li; + continue; + } + if (right_docid < left_docid) { + ++ri; + continue; + } + + const std::pair left_span = + left_decoder.positions_unchecked(li); + const std::pair right_span = + right_decoder.positions_unchecked(ri); + if (ContainsTwoTermPhrase(left_span, right_span, right_delta)) { + docids.push_back(left_docid); + } + ++li; + ++ri; + } +} + +Status EmitTwoTermPhraseChunkMerge(const std::vector& phrase_plan_index, + const std::vector& position_offsets, + std::vector& srcs, + std::vector* const docids) { + const size_t left_plan = phrase_plan_index[0]; + const size_t right_plan = phrase_plan_index[1]; + const uint32_t right_delta = position_offsets[1] - position_offsets[0]; + const PosSource& left_src = srcs[left_plan]; + const PosSource& right_src = srcs[right_plan]; + + PosChunkDecoder left_decoder; + PosChunkDecoder right_decoder; + auto decoded_left_chunk = static_cast(-1); + auto decoded_right_chunk = static_cast(-1); + size_t left_chunk = 0; + size_t right_chunk = 0; + while (left_chunk < left_src.chunks.size() && right_chunk < right_src.chunks.size()) { + const PosChunk& left = left_src.chunks[left_chunk]; + const PosChunk& right = right_src.chunks[right_chunk]; + if (left.docids.empty()) { + ++left_chunk; + continue; + } + if (right.docids.empty()) { + ++right_chunk; + continue; + } + if (left.docids.back() < right.docids.front()) { + ++left_chunk; + continue; + } + if (right.docids.back() < left.docids.front()) { + ++right_chunk; + continue; + } + + if (decoded_left_chunk != left_chunk) { + RETURN_IF_ERROR(left_decoder.decode(left)); + decoded_left_chunk = left_chunk; + } + if (decoded_right_chunk != right_chunk) { + RETURN_IF_ERROR(right_decoder.decode(right)); + decoded_right_chunk = right_chunk; + } + + EmitTwoTermPhraseChunkPair(left, right, left_decoder, right_decoder, right_delta, *docids); + + const uint32_t left_last = left.docids.back(); + const uint32_t right_last = right.docids.back(); + if (left_last <= right_last) { + ++left_chunk; + } + if (right_last <= left_last) { + ++right_chunk; + } + } + return Status::OK(); +} + +bool PhraseStartMatchesAllTerms( + uint32_t start, size_t phrase_len, size_t pair_left, size_t pair_right, + const std::vector& position_offsets, + const std::vector>& span) { + for (size_t t = 0; t < phrase_len; ++t) { + if (t == pair_left || t == pair_right) { + continue; + } + uint32_t want = 0; + if (!internal::add_position_offset(start, position_offsets[t], &want)) { + return false; + } + if (!std::binary_search(span[t].first, span[t].second, want)) { + return false; + } + } + return true; +} + +Status EmitSingleTermPhraseStreaming(const std::vector& phrase_plan_index, + std::vector& srcs, + const std::vector& candidates, + std::vector* docids) { + PhrasePositionLoader loader(srcs.size(), srcs); + for (uint32_t d : candidates) { + loader.begin_doc(d); + std::pair single_span; + RETURN_IF_ERROR(loader.positions_for_phrase_pos(phrase_plan_index, 0, &single_span)); + if (single_span.first != single_span.second) { + docids->push_back(d); + } + } + return Status::OK(); +} + +Status EmitMultiTermPhraseStreaming(const std::vector& plans, + const std::vector& phrase_plan_index, + const std::vector& position_offsets, + std::vector& srcs, + const std::vector& candidates, + std::vector* docids) { + const size_t phrase_len = phrase_plan_index.size(); + PhrasePositionLoader loader(plans.size(), srcs); + std::vector> span(phrase_len); + std::vector starts; + const size_t pair_left = SelectPhraseVerificationPair(plans, phrase_plan_index); + const size_t pair_right = pair_left + 1; + for (uint32_t d : candidates) { + loader.begin_doc(d); + std::pair left_span; + std::pair right_span; + RETURN_IF_ERROR(loader.positions_for_phrase_pos(phrase_plan_index, pair_left, &left_span)); + RETURN_IF_ERROR( + loader.positions_for_phrase_pos(phrase_plan_index, pair_right, &right_span)); + + CollectTwoTermPhraseStarts(left_span, right_span, + position_offsets[pair_right] - position_offsets[pair_left], + position_offsets[pair_left], starts); + if (starts.empty()) { + continue; + } + + span[pair_left] = left_span; + span[pair_right] = right_span; + for (size_t pp = 0; pp < phrase_len; ++pp) { + if (pp == pair_left || pp == pair_right) { + continue; + } + RETURN_IF_ERROR(loader.positions_for_phrase_pos(phrase_plan_index, pp, &span[pp])); + } + + for (uint32_t start : starts) { + if (PhraseStartMatchesAllTerms(start, phrase_len, pair_left, pair_right, + position_offsets, span)) { + docids->push_back(d); + break; + } + } + } + return Status::OK(); +} + +// Single streaming pass over the candidates: for each (ascending) candidate, +// gather positions lazily, and test the consecutive-phrase predicate +// (term[0]@p, term[1]@p+1, ...). Multi-term phrases first test the cheapest +// adjacent pair by df before decoding the remaining terms for that document. +// Cursors decode each retained chunk at most once and address positions by +// local index -- no per-candidate docid binary search, no full-candidate +// position materialization. Candidates are ascending so the emitted docids are +// already sorted. +Status EmitPhraseStreaming(const std::vector& plans, + const std::vector& phrase_plan_index, + const std::vector& position_offsets, + std::vector& srcs, const std::vector& candidates, + std::vector* docids) { + const size_t phrase_len = phrase_plan_index.size(); + if (phrase_len == 1) { + return EmitSingleTermPhraseStreaming(phrase_plan_index, srcs, candidates, docids); + } + if (phrase_len == 2) { + if (phrase_plan_index[0] != phrase_plan_index[1]) { + return EmitTwoTermPhraseChunkMerge(phrase_plan_index, position_offsets, srcs, docids); + } + return EmitTwoTermPhraseStreaming(phrase_plan_index, position_offsets, srcs, candidates, + docids); + } + return EmitMultiTermPhraseStreaming(plans, phrase_plan_index, position_offsets, srcs, + candidates, docids); +} + +// candidate_prefilter (optional): an ascending docid set the phrase must ALSO +// lie in. When provided, the leading-term conjunction is intersected with it so +// only docs in the prefilter get their positions read. Docs outside the +// prefilter cannot contribute (the caller guarantees the final answer is a +// subset), so this is result-preserving while cutting the position decode -- +// used by phrase-prefix to restrict the huge leading-phrase candidate set to +// the docs that also carry some tail expansion. +Status BuildPhraseExecutionState(const LogicalIndexReader& idx, io::BatchRangeFetcher* round1, + std::vector* plans, PhraseExecutionState* state, + const std::vector* candidate_prefilter = nullptr) { + if (round1->pending() > 0) { + RETURN_IF_ERROR(round1->fetch()); + } + RETURN_IF_ERROR(internal::open_preludes(*round1, plans, + /*need_positions=*/true)); + + state->owners.clear(); + state->candidates.clear(); + std::vector doc_sources; + if (candidate_prefilter != nullptr) { + if (candidate_prefilter->empty()) { + return Status::OK(); + } + RETURN_IF_ERROR(internal::filter_docids_by_conjunction( + idx, *round1, *plans, *candidate_prefilter, &state->candidates, &doc_sources)); + } else { + RETURN_IF_ERROR(internal::build_docid_only_conjunction(idx, *round1, *plans, + &state->candidates, &doc_sources)); + } + if (state->candidates.empty()) { + return Status::OK(); + } + RETURN_IF_ERROR(BuildPositionSourcesForCandidates( + idx, *round1, *plans, &doc_sources, state->candidates, &state->owners, &state->srcs)); + return Status::OK(); +} + +Status ExecutePhrasePlans(const LogicalIndexReader& idx, io::BatchRangeFetcher* round1, + std::vector* plans, + const std::vector& phrase_plan_index, + std::vector* docids) { + PhraseExecutionState state; + RETURN_IF_ERROR(BuildPhraseExecutionState(idx, round1, plans, &state)); + if (state.candidates.empty()) { + return Status::OK(); + } + + std::vector position_offsets; + if (!internal::build_position_offsets(phrase_plan_index.size(), &position_offsets)) { + return Status::Error( + "phrase_query: phrase length exceeds doc position range"); + } + return EmitPhraseStreaming(*plans, phrase_plan_index, position_offsets, state.srcs, + state.candidates, docids); +} + +Status ExecuteResolvedPhraseTerms(const LogicalIndexReader& idx, + const std::vector& terms, + std::vector* docids) { + io::BatchRangeFetcher round1(idx.reader()); + std::vector plans; + RETURN_IF_ERROR(internal::plan_resolved_terms(idx, terms, &round1, &plans, + /*need_positions=*/false)); + std::vector phrase_plan_index(terms.size()); + std::iota(phrase_plan_index.begin(), phrase_plan_index.end(), 0); + return ExecutePhrasePlans(idx, &round1, &plans, phrase_plan_index, docids); +} + +Status CollectExpectedTailPositions(const std::vector& plans, + const std::vector& position_offsets, + std::vector& srcs, + const std::vector& candidates, + ExpectedTailPositionSet* out) { + const size_t n = plans.size(); + std::vector cur(n); + for (size_t i = 0; i < n; ++i) { + cur[i].init(&srcs[i]); + } + + std::vector ordered(n); + for (size_t i = 0; i < n; ++i) { + ordered[plans[i].order] = &cur[i]; + } + + std::vector> span(n); + for (uint32_t d : candidates) { + for (size_t i = 0; i < n; ++i) { + RETURN_IF_ERROR(cur[i].seek(d)); + } + for (size_t pp = 0; pp < n; ++pp) { + RETURN_IF_ERROR(ordered[pp]->positions(&span[pp])); + } + + // Anchor the outer enumeration on the SPARSEST exact term (smallest + // per-doc position span), not the hardcoded phrase-position-0 term. The + // set of valid phrase starts is anchor-independent -- each valid start + // maps 1:1 to exactly one anchor position (anchor_pos = start + + // offset[anchor]) -- so enumerating the shortest span and binary-searching + // the others yields the identical result set with the fewest outer + // iterations. A leading high-frequency exact term no longer forces + // O(|span[0]|) work per candidate doc. + size_t anchor = 0; + auto best = static_cast(span[0].second - span[0].first); + for (size_t t = 1; t < n; ++t) { + const auto sz = static_cast(span[t].second - span[t].first); + if (sz < best) { + best = sz; + anchor = t; + } + } + const uint32_t anchor_off = position_offsets[anchor]; + SNII_QUERY_ADD(anchor_iterations, best); + + const size_t expected_begin = out->positions.size(); + for (const uint32_t* p = span[anchor].first; p != span[anchor].second; ++p) { + const uint32_t anchor_pos = *p; + // Underflow guard: a general anchor (offset > 0) can sit at a position + // smaller than its offset, which would wrap `start`. Such a position + // admits no valid phrase start and is skipped. (The old span[0] anchor + // had offset 0 and could never underflow.) + if (anchor_pos < anchor_off) { + continue; + } + const uint32_t start = anchor_pos - anchor_off; + bool ok = true; + for (size_t t = 0; t < n; ++t) { + if (t == anchor) { + continue; // the anchor term's position is satisfied by construction + } + uint32_t want = 0; + if (!internal::add_position_offset(start, position_offsets[t], &want)) { + ok = false; + break; + } + if (!std::binary_search(span[t].first, span[t].second, want)) { + ok = false; + break; + } + } + uint32_t tail_pos = 0; + if (ok && internal::add_position_offset(start, position_offsets[n], &tail_pos)) { + out->positions.push_back(tail_pos); + } + } + const size_t expected_end = out->positions.size(); + if (expected_end != expected_begin) { + out->docs.push_back({d, expected_begin, expected_end}); + } + } + return Status::OK(); +} + +Status CollectSingleTermExpectedTailPositions(std::vector& srcs, + const std::vector& candidates, + uint32_t tail_offset, ExpectedTailPositionSet* out) { + PostingCursor cursor; + cursor.init(srcs.data()); + out->reserve_docs(out->docs.size() + candidates.size()); + + for (uint32_t d : candidates) { + RETURN_IF_ERROR(cursor.seek(d)); + std::pair span; + RETURN_IF_ERROR(cursor.positions(&span)); + + const size_t expected_begin = out->positions.size(); + for (const uint32_t* p = span.first; p != span.second; ++p) { + uint32_t tail_pos = 0; + if (internal::add_position_offset(*p, tail_offset, &tail_pos)) { + out->positions.push_back(tail_pos); + } + } + const size_t expected_end = out->positions.size(); + if (expected_end != expected_begin) { + out->docs.push_back({d, expected_begin, expected_end}); + } + } + return Status::OK(); +} + +Status CollectExpectedTailPositions(const LogicalIndexReader& idx, + const std::vector& exact_terms, + ExpectedTailPositionSet* out, + const std::vector* candidate_prefilter = nullptr) { + out->clear(); + io::BatchRangeFetcher round1(idx.reader()); + std::vector plans; + RETURN_IF_ERROR(internal::plan_resolved_terms(idx, exact_terms, &round1, &plans, + /*need_positions=*/false)); + + PhraseExecutionState state; + RETURN_IF_ERROR(BuildPhraseExecutionState(idx, &round1, &plans, &state, candidate_prefilter)); + if (state.candidates.empty()) { + return Status::OK(); + } + out->reserve_docs(state.candidates.size()); + std::vector position_offsets; + if (!internal::build_position_offsets(plans.size() + 1, &position_offsets)) { + return Status::Error( + "phrase_prefix_query: phrase length exceeds doc position range"); + } + if (plans.size() == 1) { + return CollectSingleTermExpectedTailPositions(state.srcs, state.candidates, + position_offsets[1], out); + } + return CollectExpectedTailPositions(plans, position_offsets, state.srcs, state.candidates, out); +} + +bool contains_any_position(const ExpectedTailPositionSet& expected, + const ExpectedTailPositions& wanted, + std::pair actual) { + for (size_t i = wanted.positions_begin; i < wanted.positions_end; ++i) { + if (std::binary_search(actual.first, actual.second, expected.positions[i])) { + return true; + } + } + return false; +} + +// Upper bound on prefix expansions whose position cursors are held resident at +// once. The old per-tail loop verified a single expansion at a time (one +// PosSource + one PRX buffer live); the merged sweep below holds up to this many +// tail PosSources + cursors + PRX buffers simultaneously so it can read every +// tail's docid/prx bytes in ONE batched round and verify them in a single +// forward pass. `max_expansions` may be unbounded (<= 0), so this hard cap keeps +// resident memory bounded independent of the query: expansions beyond the cap +// are processed as additional capped groups (each a fresh single fetch) whose +// emitted docids are unioned. 32 mirrors the small-fan-in style of +// kLinearFanInMax (docid_set_ops), tightened because each cursor here also holds +// decoded PRX rather than plain docids. +constexpr size_t kMaxTailMergeBatch = 32; + +// Phrase-prefix only reads the residual tails' docid union to prefilter the +// leading-phrase candidate set when the smallest leading term's df reaches this +// -- i.e. when the leading candidate set is large enough that decoding all its +// positions dwarfs an extra docid-only union read. Below it the direct path +// (decode positions for the whole, small candidate set) is cheaper. 1<<16 is a +// "large term" scale (roughly a segment's worth); tuned to skip small/selective +// leading phrases while catching common ones like "GET images" (100M+ df). +constexpr uint32_t kPrefixLeadingPrefilterMinDf = 1u << 16; + +// Merged multi-tail verification for ONE resident-capped group of prefix +// expansions (`tails`, already truncated by max_expansions upstream). This +// replaces the per-tail verify-then-union loop: instead of re-planning + TWO +// remote rounds (docid, then prx) + a separate doc-walk PER tail and unioning N +// result lists, it plans every tail into ONE shared round1 fetch, intersects +// each tail with `expected_docids` in memory (no I/O), builds every surviving +// tail's position source in ONE batched PRX round, then sweeps the group's tail +// cursors over the ascending `expected` docs a SINGLE time -- emitting each doc +// at most once as soon as ANY tail has a position adjacent to a leading match. +// +// The emitted set is byte-identical to the per-tail path's +// UNION_{tail in group} { d : d in tail INTERSECT expected AND +// contains_any_position(expected, doc_d, pos_tail(d)) } +// because each tail's PosSource is still built from its OWN final-candidate +// docids (the shared-candidate argument is ignored for final-candidate sources), +// so pos_tail(d) and the per-doc position test are unchanged; only the I/O rounds +// (2N -> 2) and the N separate unions (-> one forward sweep) collapse. Emitted +// docids are ascending and unique because `expected.docs` is ascending/unique and +// each doc is pushed at most once. Bigram postings are NEVER consulted: every +// tail is verified against its unigram positions here. +Status CollectMergedTailMatches(const LogicalIndexReader& idx, std::vector tails, + const ExpectedTailPositionSet& expected, + const std::vector& expected_docids, + std::vector* out) { + const size_t n = tails.size(); + if (n == 0 || expected.docs.empty()) { + return Status::OK(); + } + + // Plan every tail into ONE fetcher so their docid postings + windowed + // preludes are read in a single batched round (the per-tail path issued one + // round per tail). Each tail keeps its own single-term plan vector so the + // conjunction filter below consumes it directly, without slicing a shared + // plan vector (whose prelude readers own decoded directory buffers). + io::BatchRangeFetcher round1(idx.reader()); + std::vector> tail_plans(n); + for (size_t i = 0; i < n; ++i) { + std::vector one; + one.push_back(std::move(tails[i])); + RETURN_IF_ERROR(internal::plan_resolved_terms(idx, one, &round1, &tail_plans[i], + /*need_positions=*/false)); + } + if (round1.pending() > 0) { + RETURN_IF_ERROR(round1.fetch()); + } + for (size_t i = 0; i < n; ++i) { + RETURN_IF_ERROR(internal::open_preludes(round1, &tail_plans[i], + /*need_positions=*/true)); + } + + // Per-tail candidate docids (tail posting INTERSECT expected) and the aligned + // final-candidate doc sources feeding the batched position builder. The + // conjunction reads only already-fetched round1 bytes; a single-plan filter + // marks its one source docids_are_final_candidates, so the position builder + // materializes each tail's PosSource directly over its own candidate docs. + // Tails whose intersection is empty are dropped here (exactly as the old + // per-tail early return did), so no dead tail decodes its full posting. + std::vector> tail_candidates(n); + std::vector active_plans; + std::vector active_sources; + std::vector active_index; // active slot -> tail index (into tail_candidates) + for (size_t i = 0; i < n; ++i) { + std::vector tail_source; + RETURN_IF_ERROR(internal::filter_docids_by_conjunction( + idx, round1, tail_plans[i], expected_docids, &tail_candidates[i], &tail_source)); + if (tail_candidates[i].empty()) { + continue; // this expansion has no doc in the expected set: nothing to verify + } + active_plans.push_back(std::move(tail_plans[i].front())); + active_sources.push_back(tail_source.empty() ? DocidSource {} + : std::move(tail_source.front())); + active_index.push_back(i); + } + if (active_plans.empty()) { + return Status::OK(); + } + + // ONE batched PRX round for every retained chunk across all surviving tails + // (vs one round per tail before). `candidates` is intentionally empty: every + // source is a final-candidate source, so the builder addresses positions by + // the source's own docids and never consults the shared candidate list. + std::vector> owners; + std::vector srcs; + const std::vector no_shared_candidates; + RETURN_IF_ERROR(BuildPositionSourcesForCandidates(idx, round1, active_plans, &active_sources, + no_shared_candidates, &owners, &srcs)); + + // Single forward sweep over the ascending expected docs. For each doc probe + // only the tails that actually posted it (per-tail ascending cursor over + // tail_candidates), decode positions once, and emit the doc the instant one + // tail's positions land adjacent to a leading match. Cursors advance strictly + // forward because expected.docs is strictly ascending and each cursor is + // sought at most once per doc. + std::vector cursors(active_plans.size()); + for (size_t a = 0; a < active_plans.size(); ++a) { + cursors[a].init(&srcs[a]); + } + std::vector tail_pos(active_plans.size(), 0); + for (const ExpectedTailPositions& doc : expected.docs) { + const uint32_t d = doc.docid; + bool matched = false; + for (size_t a = 0; a < active_plans.size() && !matched; ++a) { + std::vector& cand = tail_candidates[active_index[a]]; + size_t& ti = tail_pos[a]; + while (ti < cand.size() && cand[ti] < d) { + ++ti; + } + if (ti >= cand.size() || cand[ti] != d) { + continue; // this expansion has no posting at d + } + RETURN_IF_ERROR(cursors[a].seek(d)); + std::pair actual; + RETURN_IF_ERROR(cursors[a].positions(&actual)); + if (contains_any_position(expected, doc, actual)) { + matched = true; + } + } + if (matched) { + out->push_back(d); + } + } + return Status::OK(); +} + +} // namespace + +Status phrase_query(const LogicalIndexReader& idx, const std::vector& terms, + std::vector* const docids) { + if (docids == nullptr) { + return Status::Error("phrase_query: null out"); + } + docids->clear(); + if (terms.empty()) { + return Status::OK(); + } + if (terms.size() == 1) { + return term_query(idx, terms.front(), docids); + } + if (!idx.has_positions()) { + return Status::Error( + "phrase_query: index has no positions"); + } + bool handled_by_bigram = false; + RETURN_IF_ERROR(TryTwoTermPhraseBigram(idx, terms, docids, &handled_by_bigram)); + if (handled_by_bigram) { + return Status::OK(); + } + + // Round 1: preludes (windowed) + docid postings (slim/inline) batched + // together. Positions are fetched after the docid-only conjunction has + // produced final candidates, so phrase verification does not read PRX for + // windows later removed by the docid intersection. + io::BatchRangeFetcher round1(idx.reader()); + const PhraseTermMapping mapping = BuildPhraseTermMapping(terms); + std::vector plans; + bool all_present = false; + RETURN_IF_ERROR(internal::plan_terms(idx, mapping.unique_terms, &round1, &plans, &all_present, + /*need_positions=*/false)); + if (!all_present) { + return Status::OK(); + } + return ExecutePhrasePlans(idx, &round1, &plans, mapping.phrase_plan_index, docids); +} + +Status phrase_query(const LogicalIndexReader& idx, const std::vector& terms, + std::vector* const docids, QueryProfile* profile) { + QueryProfileScope profile_scope(idx.reader(), profile); + return phrase_query(idx, terms, docids); +} + +Status phrase_prefix_query(const LogicalIndexReader& idx, const std::vector& terms, + std::vector* const docids, int32_t max_expansions) { + if (docids == nullptr) { + return Status::Error("phrase_prefix_query: null out"); + } + docids->clear(); + if (terms.empty()) { + return Status::OK(); + } + if (terms.size() == 1) { + return prefix_query(idx, terms.front(), docids, max_expansions); + } + if (!idx.has_positions()) { + return Status::Error( + "phrase_prefix_query: index has no positions"); + } + std::vector exact_terms; + exact_terms.reserve(terms.size() - 1); + for (size_t i = 0; i + 1 < terms.size(); ++i) { + ResolvedQueryTerm resolved; + bool found = false; + RETURN_IF_ERROR(internal::resolve_query_term(idx, terms[i], &resolved, &found)); + if (!found) { + return Status::OK(); + } + exact_terms.push_back(std::move(resolved)); + } + + // Expand the tail prefix over REAL terms only: hidden phrase-bigram dict + // terms (and the sentinel) are rejected DURING enumeration so they never + // consume a max_expansions slot. The previous prefix_terms(max_expansions) + + // erase_if ordering counted hidden terms against the expansion budget FIRST + // and dropped them AFTER, so every hidden term inside the prefix range + // silently displaced one real tail past the truncated window -- and because + // G01 df-pruning changes how many hidden bigram terms a segment + // materializes, the surviving tail set (and thus the result) depended on + // the segment's bigram layout instead of the query. Bigram postings are + // NEVER consulted as tail evidence here, on pruned or legacy segments + // alike: every expanded tail below is verified against UNIGRAM postings and + // positions (single-tail via the generic streaming phrase path, multi-tail + // via CollectMergedTailMatches), which G01's diet leaves untouched. + std::vector tail_hits; + RETURN_IF_ERROR(idx.visit_prefix_terms(terms.back(), [&](LogicalIndexReader::PrefixHit&& hit, + bool* stop) { + if (format::is_phrase_bigram_term(hit.term)) { + return Status::OK(); // hidden term: never a tail, never a slot + } + tail_hits.push_back(std::move(hit)); + *stop = max_expansions > 0 && tail_hits.size() >= static_cast(max_expansions); + return Status::OK(); + })); + if (tail_hits.empty()) { + return Status::OK(); + } + if (tail_hits.size() == 1) { + std::vector resolved_terms = exact_terms; + resolved_terms.push_back(ResolvedQueryTerm {.entry = std::move(tail_hits.front().entry), + .frq_base = tail_hits.front().frq_base, + .prx_base = tail_hits.front().prx_base}); + return ExecuteResolvedPhraseTerms(idx, resolved_terms, docids); + } + + std::vector acc; + + // Bigram fast path for the SINGLE-leading-term case (the common phrase-prefix + // shape, e.g. "connection res*"): each expansion (lead, tail) is exactly a + // 2-term phrase, so a SURVIVING hidden bigram pair posting IS the adjacency + // answer with NO position read -- the very fast path the plain 2-term phrase + // uses. Partition the tails: expansions whose (lead, tail) pair the 2-term + // path can answer (present pair, or a legitimately-empty miss) contribute + // their docids directly; the rest (a pruning-ambiguous miss) fall through to + // the position-verification path below. This never sacrifices fidelity -- + // TryTwoTermPhraseBigram returns handled=false for exactly the misses that + // require positions -- and reads no bigram positions (docs-only postings). + std::vector verify_hits; + if (exact_terms.size() == 1 && !idx.phrase_bigrams_deferred()) { + const std::string& lead = terms[terms.size() - 2]; + verify_hits.reserve(tail_hits.size()); + for (LogicalIndexReader::PrefixHit& hit : tail_hits) { + std::vector pair_docs; + bool handled = false; + const std::vector pair_terms = {lead, hit.term}; + RETURN_IF_ERROR(TryTwoTermPhraseBigram(idx, pair_terms, &pair_docs, &handled)); + if (handled) { + internal::union_sorted_into(&acc, pair_docs); + } else { + verify_hits.push_back(std::move(hit)); + } + } + } else { + verify_hits = std::move(tail_hits); + } + + // Position-verify the residual expansions (multi-leading-term phrases, and + // single-leading tails whose bigram pair was pruned/ambiguous). + if (!verify_hits.empty()) { + // Narrow the leading-phrase candidate set to docs that ALSO carry some + // residual tail before decoding positions. The leading phrase can be + // ultra-common (e.g. "GET images" = 127M docs) while the answer is + // selective (its tails match 2.8M) -- decoding positions for the full + // 127M candidate set is the dominant cost. Reading the tails' docid + // union first (docid-only: few range reads, the SNII I/O strength) and + // passing it as a prefilter is result-preserving -- the answer is a + // subset of (leading candidates INTERSECT tail-union) -- and collapses + // the position decode to the intersection. Gated on a LARGE leading + // phrase: the tail-union read only pays off when the leading candidate + // set (bounded by the smallest leading term df) dwarfs the tail union, + // so a small/selective leading phrase keeps the direct path (no extra + // read). kPrefixLeadingPrefilterMinDf is that threshold. + uint32_t min_lead_df = std::numeric_limits::max(); + for (const ResolvedQueryTerm& t : exact_terms) { + min_lead_df = std::min(min_lead_df, t.entry.df); + } + // Sum of the residual tail dfs is an upper bound on the tail union size. + // The prefilter only narrows (and thus pays for its docid read) when the + // tail union is SMALLER than the leading candidate set -- i.e. the tails + // are selective. A broad tail prefix (e.g. "co*" in a log corpus where + // most docs carry some co-word) unions to ~everything: intersecting it + // saves no position decode while costing a full docid read. Gate on + // sum(tail_df) < min_lead_df so only a common-leading + selective-tail + // shape (e.g. "GET images sp*") takes the prefilter. + uint64_t tail_df_sum = 0; + for (const LogicalIndexReader::PrefixHit& hit : verify_hits) { + tail_df_sum += hit.entry.df; + } + const std::vector* prefilter = nullptr; + std::vector tail_union; + if (min_lead_df >= kPrefixLeadingPrefilterMinDf && tail_df_sum < min_lead_df) { + std::vector tail_postings; + tail_postings.reserve(verify_hits.size()); + for (const LogicalIndexReader::PrefixHit& hit : verify_hits) { + tail_postings.push_back({hit.entry, hit.frq_base, hit.prx_base}); + } + RETURN_IF_ERROR(internal::build_docid_union(idx, tail_postings, &tail_union)); + if (tail_union.empty()) { + *docids = std::move(acc); + return Status::OK(); + } + prefilter = &tail_union; + } + + ExpectedTailPositionSet expected; + RETURN_IF_ERROR(CollectExpectedTailPositions(idx, exact_terms, &expected, prefilter)); + if (!expected.docs.empty()) { + // Hoist the expected-docid projection out of the per-tail loop: it + // depends only on the (const) expected set and is ascending because + // expected.docs derives from ascending candidates -- satisfying + // filter_docids_by_conjunction's sorted-input contract. + std::vector expected_docids; + expected_docids.reserve(expected.docs.size()); + for (const ExpectedTailPositions& doc : expected.docs) { + expected_docids.push_back(doc.docid); + } + SNII_QUERY_COUNT(expected_docids_build); + + // Verify the residual expansions in resident-capped GROUPS: within a + // group every tail shares one docid fetch + one PRX fetch and is + // merged in a single forward sweep; across groups the emitted docids + // are unioned (associative/commutative + dedup, so grouping is + // result-invariant). + for (size_t start = 0; start < verify_hits.size(); start += kMaxTailMergeBatch) { + const size_t end = std::min(start + kMaxTailMergeBatch, verify_hits.size()); + std::vector group; + group.reserve(end - start); + for (size_t i = start; i < end; ++i) { + group.push_back(ResolvedQueryTerm {.entry = std::move(verify_hits[i].entry), + .frq_base = verify_hits[i].frq_base, + .prx_base = verify_hits[i].prx_base}); + } + std::vector group_docs; + RETURN_IF_ERROR(CollectMergedTailMatches(idx, std::move(group), expected, + expected_docids, &group_docs)); + internal::union_sorted_into(&acc, group_docs); + } + } + } + *docids = std::move(acc); + return Status::OK(); +} + +Status phrase_prefix_query(const LogicalIndexReader& idx, const std::vector& terms, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions) { + QueryProfileScope profile_scope(idx.reader(), profile); + return phrase_prefix_query(idx, terms, docids, max_expansions); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/phrase_query.h b/be/src/storage/index/snii/query/phrase_query.h new file mode 100644 index 00000000000000..59ef13469ad07f --- /dev/null +++ b/be/src/storage/index/snii/query/phrase_query.h @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/query_profile.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// phrase_query -- MATCH_PHRASE: return the sorted docid set in which the terms +// occur consecutively (for some i, every term k appears at position pos+k in +// the same doc). It first builds the docid conjunction with docs-only posting +// reads, then fetches PRX only for chunks that can contain final candidates: +// 1. read preludes / docs-only posting ranges and intersect per-term docids; +// 2. fetch retained PRX chunks and stream positions for survivors; +// 3. for each surviving doc, check that some position p exists with +// term[0]@p, term[1]@p+1, ... term[n-1]@p+(n-1). +// An empty term list -> empty result. Any term absent -> empty result. +namespace doris::snii::query { + +Status phrase_query(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids); +Status phrase_query(const reader::LogicalIndexReader& idx, const std::vector& terms, + std::vector* docids, QueryProfile* profile); + +// phrase_prefix_query -- MATCH_PHRASE_PREFIX: the last item in `terms` is a +// term prefix and preceding items are exact terms. For example {"quick", "bro"} +// matches "quick brown" and "quick bronze". Empty terms -> empty result. +Status phrase_prefix_query(const reader::LogicalIndexReader& idx, + const std::vector& terms, + std::vector* const docids, int32_t max_expansions = 0); +Status phrase_prefix_query(const reader::LogicalIndexReader& idx, + const std::vector& terms, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions = 0); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/prefix_query.cpp b/be/src/storage/index/snii/query/prefix_query.cpp new file mode 100644 index 00000000000000..d750868e6c2a04 --- /dev/null +++ b/be/src/storage/index/snii/query/prefix_query.cpp @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/prefix_query.h" + +#include +#include + +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/query/internal/term_expansion.h" + +namespace doris::snii::query { + +using reader::LogicalIndexReader; + +Status prefix_query(const LogicalIndexReader& idx, std::string_view prefix, + std::vector* const docids, int32_t max_expansions) { + if (docids == nullptr) { + return Status::Error("prefix_query: null out"); + } + docids->clear(); + VectorDocIdSink sink(*docids); + return prefix_query(idx, prefix, &sink, max_expansions); +} + +Status prefix_query(const LogicalIndexReader& idx, std::string_view prefix, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions) { + QueryProfileScope profile_scope(idx.reader(), profile); + return prefix_query(idx, prefix, docids, max_expansions); +} + +Status prefix_query(const LogicalIndexReader& idx, std::string_view prefix, DocIdSink* const sink, + int32_t max_expansions) { + if (sink == nullptr) { + return Status::Error("prefix_query: null sink"); + } + + return internal::emit_expanded_docid_union( + idx, prefix, [](std::string_view term) { return !format::is_phrase_bigram_term(term); }, + sink, max_expansions); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/prefix_query.h b/be/src/storage/index/snii/query/prefix_query.h new file mode 100644 index 00000000000000..6d17aa7b48b1c6 --- /dev/null +++ b/be/src/storage/index/snii/query/prefix_query.h @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/query_profile.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// prefix_query -- MATCH_PREFIX semantics: enumerate dictionary terms with the +// requested prefix, then return the sorted docid set containing any enumerated +// term. Empty prefix enumerates all terms. No matching terms -> empty result. +namespace doris::snii::query { + +Status prefix_query(const reader::LogicalIndexReader& idx, std::string_view prefix, + std::vector* const docids, int32_t max_expansions = 0); +Status prefix_query(const reader::LogicalIndexReader& idx, std::string_view prefix, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions = 0); +Status prefix_query(const reader::LogicalIndexReader& idx, std::string_view prefix, + DocIdSink* const sink, int32_t max_expansions = 0); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/query_profile.cpp b/be/src/storage/index/snii/query/query_profile.cpp new file mode 100644 index 00000000000000..b8155111a6dc34 --- /dev/null +++ b/be/src/storage/index/snii/query/query_profile.cpp @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/query_profile.h" + +#include +#include + +#include "storage/index/snii/io/file_reader.h" + +namespace doris::snii::query { + +QueryProfileScope::QueryProfileScope(io::FileReader* reader, QueryProfile* profile) + : reader_(reader), profile_(profile), start_(std::chrono::steady_clock::now()) { + if (profile_ == nullptr) return; + + *profile_ = QueryProfile {}; + if (reader_ == nullptr) return; + + const io::IoMetrics* metrics = reader_->io_metrics(); + if (metrics == nullptr) return; + + profile_->has_io_metrics = true; + profile_->io_before = *metrics; +} + +QueryProfileScope::~QueryProfileScope() { + finish(); +} + +void QueryProfileScope::finish() { + if (profile_ == nullptr || finished_) return; + finished_ = true; + + const auto end = std::chrono::steady_clock::now(); + const auto elapsed = std::chrono::duration_cast(end - start_).count(); + profile_->elapsed_ns = std::max(1, static_cast(elapsed)); + + if (!profile_->has_io_metrics || reader_ == nullptr) return; + const io::IoMetrics* metrics = reader_->io_metrics(); + if (metrics == nullptr) { + profile_->has_io_metrics = false; + return; + } + profile_->io_after = *metrics; + profile_->io_delta = io::delta(profile_->io_after, profile_->io_before); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/query_profile.h b/be/src/storage/index/snii/query/query_profile.h new file mode 100644 index 00000000000000..937046c67d3de6 --- /dev/null +++ b/be/src/storage/index/snii/query/query_profile.h @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "storage/index/snii/io/io_metrics.h" + +namespace doris::snii::io { +class FileReader; +} + +namespace doris::snii::query { + +struct QueryProfile { + uint64_t elapsed_ns = 0; + bool has_io_metrics = false; + io::IoMetrics io_before; + io::IoMetrics io_after; + io::IoMetrics io_delta; +}; + +class QueryProfileScope { +public: + QueryProfileScope(io::FileReader* reader, QueryProfile* profile); + ~QueryProfileScope(); + QueryProfileScope(const QueryProfileScope&) = delete; + QueryProfileScope& operator=(const QueryProfileScope&) = delete; + + void finish(); + +private: + io::FileReader* reader_ = nullptr; + QueryProfile* profile_ = nullptr; + std::chrono::steady_clock::time_point start_; + bool finished_ = false; +}; + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/regexp_query.cpp b/be/src/storage/index/snii/query/regexp_query.cpp new file mode 100644 index 00000000000000..d0115e1bc6907f --- /dev/null +++ b/be/src/storage/index/snii/query/regexp_query.cpp @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/regexp_query.h" + +#include + +#include +#include +#include +#include +#include + +#include "storage/index/snii/query/internal/regex_prefix.h" +#include "storage/index/snii/query/internal/term_expansion.h" + +namespace doris::snii::query { + +namespace { + +bool is_regex_metachar(char c) { + switch (c) { + case '.': + case '^': + case '$': + case '|': + case '(': + case ')': + case '[': + case ']': + case '*': + case '+': + case '?': + case '{': + case '}': + case '\\': + return true; + default: + return false; + } +} + +std::string literal_prefix_for_regex(std::string_view pattern) { + std::string out; + size_t i = 0; + if (!pattern.empty() && pattern.front() == '^') { + i = 1; + } + for (; i < pattern.size(); ++i) { + const char c = pattern[i]; + if (is_regex_metachar(c)) { + break; + } + out.push_back(c); + } + return out; +} + +} // namespace + +namespace internal { + +std::string regex_enum_prefix(std::string_view pattern, const re2::RE2& re) { + // Left-anchored patterns can yield a tighter enumeration prefix via RE2's + // PossibleMatchRange than the conservative literal scan (which stops at the + // first metacharacter). The prefix only bounds how many dictionary terms are + // enumerated; final acceptance is still decided by RE2::FullMatch, so an + // over-wide prefix is always safe. Fall back to the literal scan otherwise. + if (!pattern.empty() && pattern.front() == '^' && re.ok()) { + std::string min_prefix; + std::string max_prefix; + if (re.PossibleMatchRange(&min_prefix, &max_prefix, 256) && !min_prefix.empty() && + !max_prefix.empty() && min_prefix.front() == max_prefix.front()) { + const auto mismatch_pair = std::ranges::mismatch(min_prefix, max_prefix); + const auto common_len = + static_cast(std::distance(min_prefix.begin(), mismatch_pair.in1)); + if (common_len > 0) { + return min_prefix.substr(0, common_len); + } + } + } + return literal_prefix_for_regex(pattern); +} + +} // namespace internal + +Status regexp_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, int32_t max_expansions) { + if (docids == nullptr) { + return Status::Error("regexp_query: null out"); + } + docids->clear(); + VectorDocIdSink sink(*docids); + return regexp_query(idx, pattern, &sink, max_expansions); +} + +Status regexp_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions) { + QueryProfileScope profile_scope(idx.reader(), profile); + return regexp_query(idx, pattern, docids, max_expansions); +} + +Status regexp_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + DocIdSink* const sink, int32_t max_expansions) { + if (sink == nullptr) { + return Status::Error("regexp_query: null sink"); + } + + re2::RE2::Options options; + options.set_log_errors(false); // Do not spam the BE log on user-supplied bad patterns. + const re2::RE2 re(re2::StringPiece(pattern.data(), pattern.size()), options); + if (!re.ok()) { + return Status::Error( + std::string("regexp_query: invalid regex: ") + re.error()); + } + + // RE2::FullMatch is anchored at both ends, matching std::regex_match whole-term + // semantics. Unsupported constructs (backreferences, lookaround) fail re.ok() + // above and return InvalidArgument rather than throwing. + const std::string enum_prefix = internal::regex_enum_prefix(pattern, re); + return internal::emit_expanded_docid_union( + idx, enum_prefix, + [&re](std::string_view term) { + return re2::RE2::FullMatch(re2::StringPiece(term.data(), term.size()), re); + }, + sink, max_expansions); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/regexp_query.h b/be/src/storage/index/snii/query/regexp_query.h new file mode 100644 index 00000000000000..b6b48ed57cfb6f --- /dev/null +++ b/be/src/storage/index/snii/query/regexp_query.h @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/query_profile.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// regexp_query -- MATCH_REGEXP semantics over dictionary terms. The pattern is +// evaluated with RE2::FullMatch (anchored at both ends), so it must match the +// whole term. Invalid/unsupported patterns (e.g. backreferences, lookaround) +// return Status::InvalidArgument instead of throwing. Matching terms are executed +// as a sorted deduplicated docid union. +namespace doris::snii::query { + +Status regexp_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, int32_t max_expansions = 0); +Status regexp_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions = 0); +Status regexp_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + DocIdSink* const sink, int32_t max_expansions = 0); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/scoring_query.cpp b/be/src/storage/index/snii/query/scoring_query.cpp new file mode 100644 index 00000000000000..0d994bbfc16be7 --- /dev/null +++ b/be/src/storage/index/snii/query/scoring_query.cpp @@ -0,0 +1,710 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/scoring_query.h" + +#include +#include +#include +#include +#include +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/reader/windowed_posting.h" + +namespace doris::snii::query { + +using format::DictEntry; +using format::DictEntryEnc; +using format::DictEntryKind; +using format::FrqPreludeReader; +using format::WindowMeta; +using reader::LogicalIndexReader; + +namespace { + +// One scored posting for one term in one doc. +struct TermPosting { + uint32_t docid = 0; + double score = 0.0; +}; + +// One window's block-max upper bound and the docid range it covers. block_max is +// true when max_score came from the frq_prelude columns (vs the exact-score +// fallback); both are valid upper bounds, so it is informational only. +struct WindowBound { + uint32_t first_docid = 0; // inclusive + uint32_t last_docid = 0; // inclusive + double max_score = 0.0; // block-max upper bound for any doc in this window + bool block_max = false; +}; + +// All scored postings of one query term plus its block-max metadata. +struct TermCursor { + std::vector postings; // ascending docid, exact per-doc scores + std::vector windows; // ascending, covering all postings + size_t pos = 0; // DAAT cursor into postings +}; + +uint32_t CurrentDoc(const TermCursor& c) { + return c.pos < c.postings.size() ? c.postings[c.pos].docid + : std::numeric_limits::max(); +} + +// Reads one slim .frq window's bytes for a slim pod_ref/inline entry (prelude +// stripped). Windowed entries are handled separately via the prelude decode. +Status FetchSlimWindowBytes(const LogicalIndexReader& idx, const DictEntry& entry, + uint64_t frq_base, std::vector* window_owned, Slice* window) { + if (entry.kind == DictEntryKind::kInline) { + *window = Slice(entry.frq_bytes); + return Status::OK(); + } + uint64_t win_abs = 0; + uint64_t win_len = 0; + RETURN_IF_ERROR(idx.resolve_frq_window(entry, frq_base, &win_abs, &win_len)); + io::BatchRangeFetcher fetcher(idx.reader()); + const size_t h = fetcher.add(win_abs, win_len); + RETURN_IF_ERROR(fetcher.fetch()); + Slice got = fetcher.get(h); + window_owned->assign(got.data(), got.data() + got.size()); + *window = Slice(*window_owned); + return Status::OK(); +} + +// Reads a windowed entry's frq_prelude (block-max columns live here). +Status FetchPrelude(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t frq_base, + FrqPreludeReader* out) { + const auto& region = idx.section_refs().posting_region; + const uint64_t prelude_abs = region.offset + frq_base + entry.frq_off_delta; + io::BatchRangeFetcher fetcher(idx.reader()); + const size_t h = fetcher.add(prelude_abs, entry.prelude_len); + RETURN_IF_ERROR(fetcher.fetch()); + return FrqPreludeReader::open(fetcher.get(h), out); +} + +// Builds per-window block-max bounds from a windowed entry's prelude. Each +// WindowMeta carries the window's max_freq / max_norm and its covered docid +// range (win_base+1 .. last_docid), so bounds come straight from the directory. +Status BuildWindowBounds(const FrqPreludeReader& prelude, const ScorerContext& ctx, double avgdl, + const Bm25Params& params, std::vector* windows) { + const uint32_t n = prelude.n_windows(); + for (uint32_t w = 0; w < n; ++w) { + WindowMeta m; + RETURN_IF_ERROR(prelude.window(w, &m)); + if (m.doc_count == 0) continue; + WindowBound wb; + wb.first_docid = static_cast(m.win_base) + (w == 0 ? 0u : 1u); + wb.last_docid = m.last_docid; + wb.max_score = ctx.max_score(m.max_freq, m.max_norm, avgdl, params); + wb.block_max = true; + windows->push_back(wb); + } + return Status::OK(); +} + +// Fallback single window covering all postings, bounded by the exact max score +// (always a valid upper bound, so pruning stays correct). +void SingleWindowFallback(const std::vector& postings, + std::vector* windows) { + if (postings.empty()) return; + WindowBound wb; + wb.first_docid = postings.front().docid; + wb.last_docid = postings.back().docid; + wb.block_max = false; + for (const auto& p : postings) wb.max_score = std::max(wb.max_score, p.score); + windows->push_back(wb); +} + +// Computes exact per-doc BM25 scores from decoded (docid, freq) vectors. +Status ScoreDecoded(const stats::SniiStatsProvider& stats, const ScorerContext& ctx, + const Bm25Params& params, const std::vector& docids, + const std::vector& freqs, std::vector* out) { + const double avgdl = stats.avgdl(); + out->reserve(docids.size()); + for (size_t i = 0; i < docids.size(); ++i) { + uint8_t norm = 0; + RETURN_IF_ERROR(stats.encoded_norm(docids[i], &norm)); + const uint32_t tf = i < freqs.size() ? freqs[i] : 1; + out->push_back({docids[i], ctx.score(tf, norm, avgdl, params)}); + } + return Status::OK(); +} + +// Decodes a slim/inline term's single .frq window ([dd_region][freq_region]) into +// docids/freqs using the entry's region metadata. +Status DecodeSlim(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t frq_base, + std::vector* docids, std::vector* freqs) { + std::vector owned; + Slice window; + RETURN_IF_ERROR(FetchSlimWindowBytes(idx, entry, frq_base, &owned, &window)); + const uint64_t dd_len = entry.dd_meta.disk_len; + if (dd_len > window.size()) { + return Status::Error( + "scoring_query: slim dd region exceeds window"); + } + Slice dd_region = window.subslice(0, static_cast(dd_len)); + RETURN_IF_ERROR(format::decode_dd_region(dd_region, entry.dd_meta, + /*win_base=*/0, docids)); + // G16-c freq-dropped segments (write_freq == false) carry a zero-length + // freq region on slim/inline entries. Fail with the SEMANTIC error and NOT + // with INVERTED_INDEX_FILE_CORRUPTED: the Doris segment iterator silently + // downgrades that code to a non-index evaluation, which would mask a + // by-design layout as data corruption once BM25 runs over mixed segments. + if (window.size() == static_cast(dd_len) && !docids->empty()) { + return Status::Error( + "scoring_query: freqs requested but the slim entry has no freq region " + "(freq-dropped positions index)"); + } + Slice freq_region = window.subslice(static_cast(dd_len), + window.size() - static_cast(dd_len)); + return format::decode_freq_region(freq_region, entry.freq_meta, docids->size(), freqs); +} + +// Builds the cursor for a windowed term: tiles all windows for exact scores and +// reads the prelude once for true per-window block-max bounds. +Status BuildWindowedCursor(const LogicalIndexReader& idx, const stats::SniiStatsProvider& stats, + const ScorerContext& ctx, const DictEntry& entry, uint64_t frq_base, + uint64_t prx_base, const Bm25Params& params, TermCursor* cursor) { + reader::DecodedPosting posting; + // Scoring needs freqs for BM25: fetch the FULL windows (want_freq=true). + RETURN_IF_ERROR(reader::read_windowed_posting(idx, entry, frq_base, prx_base, + /*want_positions=*/false, + /*want_freq=*/true, &posting)); + RETURN_IF_ERROR( + ScoreDecoded(stats, ctx, params, posting.docids, posting.freqs, &cursor->postings)); + FrqPreludeReader prelude; + if (FetchPrelude(idx, entry, frq_base, &prelude).ok()) { + RETURN_IF_ERROR(BuildWindowBounds(prelude, ctx, stats.avgdl(), params, &cursor->windows)); + } + return Status::OK(); +} + +// Builds the cursor for one term: postings with exact scores + window bounds. +Status BuildCursor(const LogicalIndexReader& idx, const stats::SniiStatsProvider& stats, + const std::string& term, const Bm25Params& params, bool* found, + TermCursor* cursor) { + DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + RETURN_IF_ERROR(idx.lookup(term, found, &entry, &frq_base, &prx_base)); + if (!*found) return Status::OK(); + + const ScorerContext ctx = ScorerContext::make(stats.indexed_doc_count(), entry.df); + + const bool windowed = + entry.kind == DictEntryKind::kPodRef && entry.enc == DictEntryEnc::kWindowed; + if (windowed) { + RETURN_IF_ERROR( + BuildWindowedCursor(idx, stats, ctx, entry, frq_base, prx_base, params, cursor)); + } else { + std::vector docids; + std::vector freqs; + RETURN_IF_ERROR(DecodeSlim(idx, entry, frq_base, &docids, &freqs)); + RETURN_IF_ERROR(ScoreDecoded(stats, ctx, params, docids, freqs, &cursor->postings)); + } + if (cursor->windows.empty()) { + SingleWindowFallback(cursor->postings, &cursor->windows); + } + return Status::OK(); +} + +// Block-max upper bound for a term at a given docid: the max_score of the window +// covering docid (windows are ascending and contiguous). Beyond the last window +// the bound is 0 (the term cannot contribute). +double TermBoundAt(const TermCursor& c, uint32_t docid) { + // Windows are ascending and contiguous; the first window whose last_docid is + // >= docid covers it. Its block-max is a valid upper bound for any contained + // doc, so it also bounds gaps between windows. + for (const auto& w : c.windows) { + if (docid <= w.last_docid) return w.max_score; + } + return 0.0; +} + +// Min-heap keyed on score (smallest at top) maintaining the top-K. +struct TopK { + explicit TopK(uint32_t k) : k_(k) {} + void offer(uint32_t docid, double score) { + if (heap_.size() < k_) { + heap_.push({score, docid}); + return; + } + if (heap_.empty()) return; + const Entry& worst = heap_.top(); // lowest score; ties: largest docid + const bool better = score > worst.first || (score == worst.first && docid < worst.second); + if (better) { + heap_.pop(); + heap_.push({score, docid}); + } + } + double threshold() const { return heap_.size() < k_ ? -1.0 : heap_.top().first; } + + using Entry = std::pair; + struct Cmp { + bool operator()(const Entry& a, const Entry& b) const { + if (a.first != b.first) return a.first > b.first; // min-score at top + return a.second < b.second; // for ties, largest docid at top (evictable) + } + }; + uint32_t k_; + std::priority_queue, Cmp> heap_; +}; + +void DrainSorted(TopK* topk, std::vector* out) { + std::vector all; + while (!topk->heap_.empty()) { + all.push_back({topk->heap_.top().second, topk->heap_.top().first}); + topk->heap_.pop(); + } + std::sort(all.begin(), all.end(), [](const ScoredDoc& a, const ScoredDoc& b) { + if (a.score != b.score) return a.score > b.score; + return a.docid < b.docid; + }); + *out = std::move(all); +} + +Status BuildCursors(const LogicalIndexReader& idx, const stats::SniiStatsProvider& stats, + const std::vector& terms, const Bm25Params& params, + std::vector* cursors) { + for (const auto& term : terms) { + bool found = false; + TermCursor c; + RETURN_IF_ERROR(BuildCursor(idx, stats, term, params, &found, &c)); + if (found && !c.postings.empty()) cursors->push_back(std::move(c)); + } + return Status::OK(); +} + +} // namespace + +Status scoring_query_exhaustive(const LogicalIndexReader& idx, + const stats::SniiStatsProvider& stats, + const std::vector& terms, uint32_t k, + const Bm25Params& params, std::vector* out) { + if (out == nullptr) + return Status::Error("scoring_query: null out"); + out->clear(); + if (k == 0) return Status::OK(); + + std::vector cursors; + RETURN_IF_ERROR(BuildCursors(idx, stats, terms, params, &cursors)); + + std::unordered_map scores; + for (const auto& c : cursors) + for (const auto& p : c.postings) scores[p.docid] += p.score; + + std::vector all; + all.reserve(scores.size()); + for (const auto& [docid, score] : scores) all.push_back({docid, score}); + std::sort(all.begin(), all.end(), [](const ScoredDoc& a, const ScoredDoc& b) { + if (a.score != b.score) return a.score > b.score; + return a.docid < b.docid; + }); + if (all.size() > k) all.resize(k); + *out = std::move(all); + return Status::OK(); +} + +namespace { + +// --- Phase C: selective-fetch (lazy window) WAND ----------------------------- +// +// A LazyTermCursor knows its per-window block-max bounds + docid ranges from the +// frq_prelude WITHOUT fetching any .frq window. Each window's exact (docid,score) +// postings are decoded on first access and cached, so a window is fetched at most +// once and ONLY when the WAND control flow touches a posting in it. Combined with +// window-level SkipTo (advance past whole windows whose last_docid < target via +// the prelude, never fetching them), the offer sequence is byte-identical to the +// eager scoring_query_wand path -- only the bytes read differ. +// +// Soundness: a window is fetched only when LazyCurrentDoc/LazySkipTo land the +// cursor inside it, i.e. it covers a candidate the WAND pivot already proved can +// reach the running theta (bound >= theta). LazySkipTo jumps the cursor to the +// SAME posting (first docid >= target) the eager per-doc walk would, so pivots, +// alignments and offers are identical to the eager path; only windows the eager +// path read-through-but-never-offered-from are skipped. Windows whose block-max +// bound never reaches theta are never the pivot, so never fetched. + +// One query term's lazily-fetched scoring state. +struct LazyTermCursor { + const LogicalIndexReader* idx = nullptr; + const stats::SniiStatsProvider* stats = nullptr; + ScorerContext ctx = ScorerContext::make(1, 1); + Bm25Params params; + DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + FrqPreludeReader prelude; + bool windowed = false; // false => slim/inline single block already materialized + + std::vector windows; // ascending; from prelude (or slim fallback) + std::vector postings; // sparse: only fetched windows are filled + std::vector win_start; // prefix offsets, size = windows.size()+1 + std::vector fetched; // size = windows.size() + size_t pos = 0; // virtual cursor over all windows' postings +}; + +// Total posting count across all windows (the virtual stream length). +uint32_t TotalPostings(const LazyTermCursor& c) { + return c.win_start.empty() ? 0 : c.win_start.back(); +} + +// Index of the window whose virtual range contains posting index p (p < total). +uint32_t WindowOf(const LazyTermCursor& c, uint32_t p) { + const auto it = std::upper_bound(c.win_start.begin(), c.win_start.end(), p); + return static_cast((it - c.win_start.begin()) - 1); +} + +// Fetches + decodes window w into the cursor's posting cache (idempotent). Only +// reached when the WAND proves window w can still contribute to the top-K. +Status MaterializeWindow(LazyTermCursor* c, uint32_t w) { + if (c->fetched[w]) return Status::OK(); + WindowMeta meta; + RETURN_IF_ERROR(c->prelude.window(w, &meta)); + reader::WindowAbsRange r; + RETURN_IF_ERROR(reader::windowed_window_range( + *c->idx, c->entry, c->frq_base, c->prx_base, c->prelude, w, + /*want_positions=*/false, /*want_freq=*/true, &r)); + // Scoring needs docids + freqs: fetch the window's dd sub-range AND freq sub-range. + io::BatchRangeFetcher fetcher(c->idx->reader(), reader::kSameTermCoalesceGap); + const size_t dh = fetcher.add(r.dd_off, r.dd_len); + const size_t fh = fetcher.add(r.freq_off, r.freq_len); + RETURN_IF_ERROR(fetcher.fetch()); + std::vector docids; + std::vector freqs; + std::vector> pos; + RETURN_IF_ERROR(reader::decode_window_slices(meta, fetcher.get(dh), fetcher.get(fh), Slice(), + /*want_positions=*/false, + /*want_freq=*/true, &docids, &freqs, &pos)); + if (docids.size() != c->win_start[w + 1] - c->win_start[w]) { + return Status::Error( + "scoring_query: selective window doc-count drift"); + } + std::vector scored; + RETURN_IF_ERROR(ScoreDecoded(*c->stats, c->ctx, c->params, docids, freqs, &scored)); + std::copy(scored.begin(), scored.end(), c->postings.begin() + c->win_start[w]); + c->fetched[w] = 1; + return Status::OK(); +} + +// Current docid at the cursor, fetching the covering window if needed. Exhausted +// cursor -> UINT32_MAX. +Status LazyCurrentDoc(LazyTermCursor* c, uint32_t* docid) { + if (c->pos >= TotalPostings(*c)) { + *docid = std::numeric_limits::max(); + return Status::OK(); + } + const uint32_t w = WindowOf(*c, static_cast(c->pos)); + RETURN_IF_ERROR(MaterializeWindow(c, w)); + *docid = c->postings[c->pos].docid; + return Status::OK(); +} + +// Advances pos to the first posting with docid >= target, skipping ENTIRE windows +// whose last_docid < target WITHOUT fetching them (prelude-only), then fetching +// just the landing window. Lands on the same posting the eager per-doc walk would. +Status LazySkipTo(LazyTermCursor* c, uint32_t target) { + const uint32_t total = TotalPostings(*c); + while (c->pos < total) { + const uint32_t w = WindowOf(*c, static_cast(c->pos)); + if (c->windows[w].last_docid >= target) break; + c->pos = c->win_start[w + 1]; // skip this window entirely (no fetch) + } + if (c->pos >= total) return Status::OK(); + const uint32_t w = WindowOf(*c, static_cast(c->pos)); + RETURN_IF_ERROR(MaterializeWindow(c, w)); + while (c->pos < total && c->postings[c->pos].docid < target) ++c->pos; + return Status::OK(); +} + +// Initializes a lazy windowed cursor from the prelude alone: per-window block-max +// bounds + ranges + cache slots, with NO .frq window fetched. +Status BuildLazyWindowed(LazyTermCursor* c) { + RETURN_IF_ERROR(reader::fetch_windowed_prelude(*c->idx, c->entry, c->frq_base, &c->prelude)); + RETURN_IF_ERROR( + BuildWindowBounds(c->prelude, c->ctx, c->stats->avgdl(), c->params, &c->windows)); + // BuildWindowBounds keeps only non-empty windows, in window order. Build the + // matching prefix-sum of doc_counts over those same non-empty windows so the + // bound list, win_start and fetched stay 1:1. + const uint32_t nb = static_cast(c->windows.size()); + c->win_start.assign(nb + 1, 0); + c->fetched.assign(nb, 0); + uint32_t bi = 0; + uint32_t acc = 0; + for (uint32_t w = 0; w < c->prelude.n_windows() && bi < nb; ++w) { + WindowMeta meta; + RETURN_IF_ERROR(c->prelude.window(w, &meta)); + if (meta.doc_count == 0) continue; + acc += meta.doc_count; + c->win_start[++bi] = acc; + } + c->postings.assign(acc, TermPosting {}); + return Status::OK(); +} + +// Initializes a slim/inline cursor: its single window is small, so fetch + score +// it eagerly (exactly as the existing path). One bound covers all its postings. +Status BuildLazySlim(LazyTermCursor* c) { + std::vector docids; + std::vector freqs; + RETURN_IF_ERROR(DecodeSlim(*c->idx, c->entry, c->frq_base, &docids, &freqs)); + RETURN_IF_ERROR(ScoreDecoded(*c->stats, c->ctx, c->params, docids, freqs, &c->postings)); + SingleWindowFallback(c->postings, &c->windows); + c->win_start = {0, static_cast(c->postings.size())}; + c->fetched.assign(1, 1); // already materialized + return Status::OK(); +} + +// Builds a LazyTermCursor for one term: prelude-only for windowed terms (no .frq +// fetched), fully-materialized single window for slim/inline (small). +Status BuildLazyCursor(const LogicalIndexReader& idx, const stats::SniiStatsProvider& stats, + const std::string& term, const Bm25Params& params, bool* found, + LazyTermCursor* c) { + uint64_t prx_base = 0; + RETURN_IF_ERROR(idx.lookup(term, found, &c->entry, &c->frq_base, &prx_base)); + if (!*found) return Status::OK(); + c->idx = &idx; + c->stats = &stats; + c->params = params; + c->prx_base = prx_base; + c->ctx = ScorerContext::make(stats.indexed_doc_count(), c->entry.df); + c->windowed = + c->entry.kind == DictEntryKind::kPodRef && c->entry.enc == DictEntryEnc::kWindowed; + return c->windowed ? BuildLazyWindowed(c) : BuildLazySlim(c); +} + +Status SelectiveBuildCursors(const LogicalIndexReader& idx, const stats::SniiStatsProvider& stats, + const std::vector& terms, const Bm25Params& params, + std::vector* cursors) { + for (const auto& term : terms) { + bool found = false; + LazyTermCursor c; + RETURN_IF_ERROR(BuildLazyCursor(idx, stats, term, params, &found, &c)); + if (found && TotalPostings(c) > 0) cursors->push_back(std::move(c)); + } + return Status::OK(); +} + +// Block-max upper bound for a lazy cursor at docid: block_max of the window +// covering docid (ascending, contiguous). Beyond the last window -> 0. Same +// semantics as TermBoundAt over the eager cursor's window list. +double LazyTermBoundAt(const LazyTermCursor& c, uint32_t docid) { + for (const auto& w : c.windows) { + if (docid <= w.last_docid) return w.max_score; + } + return 0.0; +} + +// Sorts cursors ascending by current docid (materializing each cursor's current +// covering window), returning the smallest current docid via *front. +Status SelectiveSortByDoc(std::vector* cursors, uint32_t* front) { + std::vector cur(cursors->size()); + for (size_t i = 0; i < cursors->size(); ++i) { + RETURN_IF_ERROR(LazyCurrentDoc(&(*cursors)[i], &cur[i])); + } + std::vector order(cursors->size()); + for (size_t i = 0; i < order.size(); ++i) order[i] = i; + std::sort(order.begin(), order.end(), [&](size_t a, size_t b) { return cur[a] < cur[b]; }); + std::vector sorted; + sorted.reserve(cursors->size()); + for (size_t i : order) sorted.push_back(std::move((*cursors)[i])); + *cursors = std::move(sorted); + *front = order.empty() ? std::numeric_limits::max() : cur[order.front()]; + return Status::OK(); +} + +// Finds the pivot term: the first cursor (current-docid order) at which the +// accumulated block-max bound reaches theta. >= keeps boundary ties (matching the +// exhaustive total order). *found=false when no remaining doc can beat theta. +Status SelectivePivot(std::vector* cursors, double theta, size_t* pivot, + uint32_t* pivot_doc, bool* found) { + double bound = 0.0; + *found = false; + for (size_t i = 0; i < cursors->size(); ++i) { + uint32_t d = 0; + RETURN_IF_ERROR(LazyCurrentDoc(&(*cursors)[i], &d)); + if (d == std::numeric_limits::max()) break; + bound += LazyTermBoundAt((*cursors)[i], d); + if (bound >= theta) { + *pivot = i; + *pivot_doc = d; + *found = true; + return Status::OK(); + } + } + return Status::OK(); +} + +// Scores the aligned pivot doc exactly (summing all cursors AT pivot_doc) and +// advances those cursors by one posting. +Status SelectiveScorePivot(std::vector* cursors, uint32_t pivot_doc, TopK* topk) { + double doc_score = 0.0; + for (auto& c : *cursors) { + uint32_t d = 0; + RETURN_IF_ERROR(LazyCurrentDoc(&c, &d)); + if (d == pivot_doc) { + doc_score += c.postings[c.pos].score; // window already materialized + ++c.pos; + } + } + topk->offer(pivot_doc, doc_score); + return Status::OK(); +} + +// Advances the first lagging cursor (current doc < pivot_doc) up to pivot_doc. +Status SelectiveAdvanceLagging(std::vector* cursors, uint32_t pivot_doc) { + for (auto& c : *cursors) { + uint32_t d = 0; + RETURN_IF_ERROR(LazyCurrentDoc(&c, &d)); + if (d < pivot_doc) { + RETURN_IF_ERROR(LazySkipTo(&c, pivot_doc)); + return Status::OK(); + } + } + return Status::OK(); +} + +// One WAND iteration body: sort, pick pivot, then either score (aligned) or skip +// a lagging cursor forward. *done=true ends the loop. +Status SelectiveStep(std::vector* cursors, TopK* topk, bool* done) { + uint32_t front = 0; + RETURN_IF_ERROR(SelectiveSortByDoc(cursors, &front)); + if (cursors->empty() || front == std::numeric_limits::max()) { + *done = true; + return Status::OK(); + } + size_t pivot = 0; + uint32_t pivot_doc = 0; + bool found_pivot = false; + RETURN_IF_ERROR(SelectivePivot(cursors, topk->threshold(), &pivot, &pivot_doc, &found_pivot)); + if (!found_pivot) { + *done = true; + return Status::OK(); + } + if (front == pivot_doc) { + return SelectiveScorePivot(cursors, pivot_doc, topk); + } + return SelectiveAdvanceLagging(cursors, pivot_doc); +} + +Status SelectiveWandLoop(std::vector* cursors, TopK* topk) { + bool done = false; + while (!done) { + RETURN_IF_ERROR(SelectiveStep(cursors, topk, &done)); + } + return Status::OK(); +} + +} // namespace + +Status scoring_query_wand_selective(const LogicalIndexReader& idx, + const stats::SniiStatsProvider& stats, + const std::vector& terms, uint32_t k, + const Bm25Params& params, std::vector* out) { + if (out == nullptr) + return Status::Error("scoring_query: null out"); + out->clear(); + if (k == 0) return Status::OK(); + + std::vector cursors; + RETURN_IF_ERROR(SelectiveBuildCursors(idx, stats, terms, params, &cursors)); + + TopK topk(k); + RETURN_IF_ERROR(SelectiveWandLoop(&cursors, &topk)); + DrainSorted(&topk, out); + return Status::OK(); +} + +Status scoring_query_wand(const LogicalIndexReader& idx, const stats::SniiStatsProvider& stats, + const std::vector& terms, uint32_t k, + const Bm25Params& params, std::vector* out) { + if (out == nullptr) + return Status::Error("scoring_query: null out"); + out->clear(); + if (k == 0) return Status::OK(); + + std::vector cursors; + RETURN_IF_ERROR(BuildCursors(idx, stats, terms, params, &cursors)); + + TopK topk(k); + // Document-at-a-time WAND with block-max bounds. + while (true) { + // Sort cursors by current docid (ascending; exhausted cursors sink). + std::sort(cursors.begin(), cursors.end(), [](const TermCursor& a, const TermCursor& b) { + return CurrentDoc(a) < CurrentDoc(b); + }); + if (cursors.empty() || + CurrentDoc(cursors.front()) == std::numeric_limits::max()) { + break; + } + + const double theta = topk.threshold(); + // Accumulate block-max upper bounds in docid order to find the pivot term. + double bound = 0.0; + size_t pivot = 0; + bool found_pivot = false; + for (size_t i = 0; i < cursors.size(); ++i) { + const uint32_t d = CurrentDoc(cursors[i]); + if (d == std::numeric_limits::max()) break; + bound += TermBoundAt(cursors[i], d); + // Use >= (not >) so a doc whose upper bound only TIES the K-th threshold is + // still explored and exact-scored: under the (score desc, docid asc) total + // order a tie can still evict the current K-th entry (smaller docid wins), + // exactly as the exhaustive path would. Strict > would wrongly prune ties. + if (bound >= theta) { + pivot = i; + found_pivot = true; + break; + } + } + if (!found_pivot) break; // no doc can beat the threshold anymore. + + const uint32_t pivot_doc = CurrentDoc(cursors[pivot]); + if (CurrentDoc(cursors.front()) == pivot_doc) { + // All cursors at the pivot doc are aligned: score it exactly. + double doc_score = 0.0; + for (auto& c : cursors) { + if (CurrentDoc(c) == pivot_doc) { + doc_score += c.postings[c.pos].score; + ++c.pos; + } + } + topk.offer(pivot_doc, doc_score); + } else { + // Advance a lagging cursor toward pivot_doc (skip docs it cannot win on). + for (auto& c : cursors) { + if (CurrentDoc(c) < pivot_doc) { + while (c.pos < c.postings.size() && c.postings[c.pos].docid < pivot_doc) { + ++c.pos; + } + break; + } + } + } + } + DrainSorted(&topk, out); + return Status::OK(); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/scoring_query.h b/be/src/storage/index/snii/query/scoring_query.h new file mode 100644 index 00000000000000..90457146c22435 --- /dev/null +++ b/be/src/storage/index/snii/query/scoring_query.h @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/bm25_scorer.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/stats/snii_stats_provider.h" + +// scoring_query -- top-K BM25 scored retrieval over one logical index for one or +// more query terms. Two entry points produce IDENTICAL rankings: +// - scoring_query_exhaustive(): scores every candidate document (the baseline +// correctness oracle). +// - scoring_query_wand(): a block-max / WAND-style optimization that uses the +// per-window max_freq / max_norm columns from the frq_prelude to bound each +// window's best possible score and SKIP windows that cannot enter the +// current top-K. A window without block-max stats (slim/inline entries or a +// missing prelude) is never pruned, so the result still equals the +// exhaustive ranking. +// +// Results are sorted by score descending; ties are broken by ascending docid so +// the ordering is deterministic and the two paths compare equal. +namespace doris::snii::query { + +// One scored hit. +struct ScoredDoc { + uint32_t docid = 0; + double score = 0.0; +}; + +// Exhaustive baseline: score every doc that contains any query term, return the +// top-k by score. params controls k1/b. Unknown terms are skipped. +Status scoring_query_exhaustive(const reader::LogicalIndexReader& idx, + const stats::SniiStatsProvider& stats, + const std::vector& terms, uint32_t k, + const Bm25Params& params, std::vector* out); + +// WAND-style block-max pruning. MUST return the same top-k as the exhaustive +// path. Windows whose block-max upper bound cannot beat the current k-th score +// are skipped; windows lacking block-max stats are scored fully. +Status scoring_query_wand(const reader::LogicalIndexReader& idx, + const stats::SniiStatsProvider& stats, + const std::vector& terms, uint32_t k, + const Bm25Params& params, std::vector* out); + +// SELECTIVE-FETCH block-max WAND (design spec section 5, "Phase C"). Same WAND / +// theta / >= tie machinery as scoring_query_wand, but it DEFERS the .frq window +// fetch: for each windowed term it first reads ONLY the frq_prelude (block-max +// columns), then fetches a term's .frq window lazily and at most once -- and ONLY +// when the running block-max bound proves a doc in that window can still reach the +// top-K (bound >= theta). A window the bound rules out is never fetched. The +// result (top-K docids AND scores, INCLUDING ties) is byte-identical to +// scoring_query_exhaustive / scoring_query_wand; only the bytes read differ. +// Slim/inline terms (no prelude) are fetched fully, exactly as today. +Status scoring_query_wand_selective(const reader::LogicalIndexReader& idx, + const stats::SniiStatsProvider& stats, + const std::vector& terms, uint32_t k, + const Bm25Params& params, std::vector* out); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/term_expansion.cpp b/be/src/storage/index/snii/query/term_expansion.cpp new file mode 100644 index 00000000000000..9465993b384561 --- /dev/null +++ b/be/src/storage/index/snii/query/term_expansion.cpp @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/internal/term_expansion.h" + +#include +#include + +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/internal/docid_union.h" + +namespace doris::snii::query::internal { + +Status emit_expanded_docid_union(const reader::LogicalIndexReader& idx, + std::string_view enum_prefix, const TermMatcher& matches, + DocIdSink* const sink, int32_t max_expansions) { + if (sink == nullptr) { + return Status::Error("term_expansion: null sink"); + } + + std::vector postings; + int32_t count = 0; + RETURN_IF_ERROR(idx.visit_prefix_terms( + enum_prefix, [&](reader::LogicalIndexReader::PrefixHit&& hit, bool* stop) { + if (format::is_phrase_bigram_term(hit.term)) { + return Status::OK(); + } + if (!matches(hit.term)) { + return Status::OK(); + } + postings.push_back({std::move(hit.entry), hit.frq_base, hit.prx_base}); + ++count; + *stop = max_expansions > 0 && count >= max_expansions; + return Status::OK(); + })); + return emit_docid_union(idx, postings, sink); +} + +} // namespace doris::snii::query::internal diff --git a/be/src/storage/index/snii/query/term_query.cpp b/be/src/storage/index/snii/query/term_query.cpp new file mode 100644 index 00000000000000..633229eab67e97 --- /dev/null +++ b/be/src/storage/index/snii/query/term_query.cpp @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/term_query.h" + +#include + +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" + +namespace doris::snii::query { + +using format::DictEntry; +using reader::LogicalIndexReader; + +Status term_query(const LogicalIndexReader& idx, std::string_view term, + std::vector* docids) { + if (docids == nullptr) + return Status::Error("term_query: null out"); + docids->clear(); + VectorDocIdSink sink(*docids); + return term_query(idx, term, &sink); +} + +Status term_query(const LogicalIndexReader& idx, std::string_view term, DocIdSink* sink) { + if (sink == nullptr) + return Status::Error("term_query: null sink"); + + bool found = false; + DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + RETURN_IF_ERROR(idx.lookup(term, &found, &entry, &frq_base, &prx_base)); + if (!found) return Status::OK(); + return internal::read_docid_posting(idx, entry, frq_base, prx_base, sink); +} + +Status term_query(const LogicalIndexReader& idx, std::string_view term, + std::vector* docids, QueryProfile* profile) { + QueryProfileScope profile_scope(idx.reader(), profile); + return term_query(idx, term, docids); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/term_query.h b/be/src/storage/index/snii/query/term_query.h new file mode 100644 index 00000000000000..7296acd9c60702 --- /dev/null +++ b/be/src/storage/index/snii/query/term_query.h @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/query_profile.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// term_query -- the simplest SNII query: return the sorted docid set that +// contains term. It runs the term lookup on the logical index, then issues a +// single batched .frq range read (one serial round) to decode the postings. +// Absent term -> empty result (OK status). +namespace doris::snii::query { + +Status term_query(const reader::LogicalIndexReader& idx, std::string_view term, + std::vector* docids); +Status term_query(const reader::LogicalIndexReader& idx, std::string_view term, DocIdSink* sink); +Status term_query(const reader::LogicalIndexReader& idx, std::string_view term, + std::vector* docids, QueryProfile* profile); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/wildcard_query.cpp b/be/src/storage/index/snii/query/wildcard_query.cpp new file mode 100644 index 00000000000000..e8313c499797aa --- /dev/null +++ b/be/src/storage/index/snii/query/wildcard_query.cpp @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/wildcard_query.h" + +#include +#include +#include +#include + +#include "storage/index/snii/query/internal/term_expansion.h" +#include "storage/index/snii/query/internal/wildcard_matcher.h" + +namespace doris::snii::query { + +namespace { + +std::string literal_prefix_for_wildcard(std::string_view pattern) { + std::string out; + for (char c : pattern) { + if (c == '*' || c == '?') { + break; + } + out.push_back(c); + } + return out; +} + +} // namespace + +Status wildcard_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, int32_t max_expansions) { + if (docids == nullptr) { + return Status::Error("wildcard_query: null out"); + } + docids->clear(); + VectorDocIdSink sink(*docids); + return wildcard_query(idx, pattern, &sink, max_expansions); +} + +Status wildcard_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions) { + QueryProfileScope profile_scope(idx.reader(), profile); + return wildcard_query(idx, pattern, docids, max_expansions); +} + +Status wildcard_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + DocIdSink* const sink, int32_t max_expansions) { + if (sink == nullptr) { + return Status::Error("wildcard_query: null sink"); + } + const std::string enum_prefix = literal_prefix_for_wildcard(pattern); + // Request-scoped matcher: its two DP scratch rows are reused across every + // visited dictionary term, so the whole-dictionary scan triggered by a + // leading wildcard performs O(1) scratch allocations instead of O(2N). + internal::WildcardMatcher<> matcher(pattern); + return internal::emit_expanded_docid_union( + idx, enum_prefix, [&matcher](std::string_view term) { return matcher(term); }, sink, + max_expansions); +} + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/query/wildcard_query.h b/be/src/storage/index/snii/query/wildcard_query.h new file mode 100644 index 00000000000000..66c08b18ae270e --- /dev/null +++ b/be/src/storage/index/snii/query/wildcard_query.h @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/query_profile.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// wildcard_query -- MATCH_WILDCARD semantics over dictionary terms. `*` matches +// any byte sequence, `?` matches one byte, and all other bytes match literally. +// Matching terms are executed as a sorted deduplicated docid union. +namespace doris::snii::query { + +Status wildcard_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, int32_t max_expansions = 0); +Status wildcard_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + std::vector* const docids, QueryProfile* profile, + int32_t max_expansions = 0); +Status wildcard_query(const reader::LogicalIndexReader& idx, std::string_view pattern, + DocIdSink* const sink, int32_t max_expansions = 0); + +} // namespace doris::snii::query diff --git a/be/src/storage/index/snii/reader/dict_block_cache.h b/be/src/storage/index/snii/reader/dict_block_cache.h new file mode 100644 index 00000000000000..18c65978ee9263 --- /dev/null +++ b/be/src/storage/index/snii/reader/dict_block_cache.h @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/dict_block.h" + +// DictBlockCache -- a REQUEST-SCOPED (per-query) MRU cache of decoded DICT +// blocks, keyed by block ordinal. +// +// Why request-scoped (and not a reader-level shared cache): the same DICT block +// is decoded once per LogicalIndexReader::lookup() today, so a multi-term query +// (phrase / boolean / conjunction) whose terms fall in the same block re-runs +// the zstd decompress + CRC verify + anchor parse for every term. Threading one +// of these caches through a single query's lookups collapses that to a single +// decode per unique block. +// +// CONCURRENCY: this object carries NO shared mutable state and is intentionally +// NOT thread-safe. It is meant to live on one query's stack/context and be used +// by a single thread; concurrent queries each own a separate cache. The shared +// LogicalIndexReader therefore stays const and lock-free -- no lock is ever held +// across a decode/IO. (The cross-query, lock-striped variant that would let +// queries share decoded blocks is deferred to the T26 concurrency work.) +namespace doris::snii::reader { + +// A decoded DICT block with stable backing storage. Heap-allocated and owned by +// a shared_ptr so the embedded DictBlockReader's Slice into `bytes` stays valid +// for the whole lifetime of any pin handed to a caller -- even after the block +// has been evicted from the cache. +struct DecodedDictBlock { + std::vector bytes; // decompressed (or raw) block bytes + format::DictBlockReader reader; // its Slice points into `bytes` +}; + +class DictBlockCache { +public: + // Loads (decodes) the block for an ordinal into a freshly heap-allocated + // DecodedDictBlock. Always invoked OUTSIDE any cache bookkeeping; it performs + // the file read + optional zstd decompress + CRC/anchor parse. + using Loader = std::function*)>; + + // A small fixed bound is enough for a single query: it only needs to keep the + // handful of distinct blocks touched while resolving one query's terms. + static constexpr size_t kDefaultMaxEntries = 8; + + DictBlockCache() = default; + explicit DictBlockCache(size_t max_entries) + : max_entries_(max_entries == 0 ? 1 : max_entries) {} + + // Returns the decoded block for `ordinal`, invoking `loader` only on a miss. + // The returned pin keeps the block alive for the caller's use regardless of + // any later eviction. On a hit, `loader` is not called (no re-decode). + Status get_or_load(uint32_t ordinal, const Loader& loader, + std::shared_ptr* out) { + if (auto it = index_.find(ordinal); it != index_.end()) { + order_.splice(order_.begin(), order_, it->second); // promote to MRU + *out = it->second->block; + return Status::OK(); + } + + std::shared_ptr loaded; + // decode happens here, never under a lock (explicit Status, header-safe: + // RETURN_IF_ERROR would need a bare `Status` in scope). + if (Status st = loader(&loaded); !st.ok()) { + return st; + } + order_.push_front(Entry {.ordinal = ordinal, .block = loaded}); + index_[ordinal] = order_.begin(); + evict_overflow(); + *out = std::move(loaded); + return Status::OK(); + } + + // Number of resident (non-evicted) entries -- bounded by max_entries(). + size_t size() const { return index_.size(); } + size_t max_entries() const { return max_entries_; } + +private: + struct Entry { + uint32_t ordinal = 0; + std::shared_ptr block; + }; + + // Drops least-recently-used entries until the bound holds. Evicting only + // releases the cache's reference; any pin a caller still holds keeps the + // block (and its reader's Slice) alive. + void evict_overflow() { + while (index_.size() > max_entries_) { + const Entry& victim = order_.back(); + index_.erase(victim.ordinal); + order_.pop_back(); + } + } + + size_t max_entries_ = kDefaultMaxEntries; + std::list order_; // front = most recently used + std::unordered_map::iterator> index_; +}; + +} // namespace doris::snii::reader diff --git a/be/src/storage/index/snii/reader/logical_index_reader.cpp b/be/src/storage/index/snii/reader/logical_index_reader.cpp new file mode 100644 index 00000000000000..c929b985a89204 --- /dev/null +++ b/be/src/storage/index/snii/reader/logical_index_reader.cpp @@ -0,0 +1,533 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/reader/logical_index_reader.h" + +#include +#include +#include +#include +#include + +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/encoding/zstd_codec.h" +#include "storage/index/snii/format/dict_block.h" +#include "storage/index/snii/format/dict_block_directory.h" +#include "storage/index/snii/reader/dict_block_cache.h" + +namespace doris::snii::reader { + +using format::BlockRef; +using format::bsbf_hash; +using format::DictBlockDirectoryReader; +using format::DictBlockReader; +using format::DictEntry; +using format::IndexTier; +using format::kBsbfBytesPerBlock; +using format::kBsbfHeaderSize; +using format::PerIndexMetaReader; +using format::RegionRef; +using format::SampledTermIndexReader; + +namespace { +constexpr uint64_t kMaxDictBlockUncompBytes = 256ULL * 1024 * 1024; +constexpr uint64_t kDefaultDictResidentMaxBytes = 256ULL * 1024; + +// L0/L1 tiering threshold (bytes). Defaults to kBsbfResidentMaxBytes; the env +// SNII_BSBF_RESIDENT_MAX overrides it for tuning and for exercising the +// on-demand L1 path in tests without a 250K-term corpus. Read fresh each open. +uint64_t bsbf_resident_max_bytes() { + const char* s = std::getenv("SNII_BSBF_RESIDENT_MAX"); + if (s != nullptr) { + char* end = nullptr; + const unsigned long long v = std::strtoull(s, &end, 10); + if (end != s) { + return v; + } + } + return format::kBsbfResidentMaxBytes; +} + +uint64_t dict_resident_max_bytes() { + const char* s = std::getenv("SNII_DICT_RESIDENT_MAX"); + if (s != nullptr) { + char* end = nullptr; + const unsigned long long v = std::strtoull(s, &end, 10); + if (end != s) { + return v; + } + } + return kDefaultDictResidentMaxBytes; +} + +Status checked_size(uint64_t value, const char* error, size_t* out) { + if (value > std::numeric_limits::max()) { + return Status::Error(error); + } + *out = static_cast(value); + return Status::OK(); +} + +Status dict_block_memory_bytes(const BlockRef& ref, uint64_t* out) { + if ((ref.flags & format::block_ref_flags::kZstd) == 0) { + *out = ref.length; + return Status::OK(); + } + if (ref.uncomp_len == 0 || ref.uncomp_len > kMaxDictBlockUncompBytes) { + return Status::Error( + "dict block: zstd uncomp_len out of range"); + } + *out = ref.uncomp_len; + return Status::OK(); +} + +// Decompresses a zstd dict block from its on-disk bytes into *out. The decode +// buffer in zstd_decompress is resize_uninitialized'd (T19) then fully written. +Status zstd_decompress_dict_block(Slice on_disk, const BlockRef& ref, std::vector* out) { + uint64_t memory_bytes = 0; + RETURN_IF_ERROR(dict_block_memory_bytes(ref, &memory_bytes)); + size_t uncomp_len = 0; + RETURN_IF_ERROR( + checked_size(memory_bytes, "dict block: zstd length out of range", &uncomp_len)); + return zstd_decompress(on_disk, uncomp_len, out); +} + +// Materializes the usable (uncompressed) bytes of a dict block from a view over +// its on-disk bytes -- a raw block is copied, a zstd block is decompressed. Used +// by the resident single-range path, where on_disk is a sub-slice of the shared +// region buffer (so a raw block must be copied, not aliased). +Status decompress_dict_block_payload(Slice on_disk, const BlockRef& ref, + std::vector* out) { + if ((ref.flags & format::block_ref_flags::kZstd) == 0) { + out->assign(on_disk.data(), on_disk.data() + on_disk.size()); + return Status::OK(); + } + return zstd_decompress_dict_block(on_disk, ref, out); +} + +Status read_dict_block_bytes(io::FileReader* reader, const BlockRef& ref, + std::vector* out) { + size_t read_len = 0; + RETURN_IF_ERROR(checked_size(ref.length, "dict block: on-disk length out of range", &read_len)); + + std::vector block_bytes; + RETURN_IF_ERROR(reader->read_at(ref.offset, read_len, &block_bytes)); + if (block_bytes.size() != read_len) { + return Status::Error( + "dict block: short read"); + } + + // Raw on-demand block: move the freshly read buffer in (no copy). + if ((ref.flags & format::block_ref_flags::kZstd) == 0) { + *out = std::move(block_bytes); + return Status::OK(); + } + return zstd_decompress_dict_block(Slice(block_bytes), ref, out); +} + +Status open_dict_block(io::FileReader* reader, const BlockRef& ref, IndexTier tier, + bool has_positions, std::vector* bytes, DictBlockReader* out) { + RETURN_IF_ERROR(read_dict_block_bytes(reader, ref, bytes)); + return DictBlockReader::open(Slice(*bytes), tier, has_positions, out); +} + +// Validates that block `ref` lies fully within dict_region and returns its byte +// range relative to the start of the region. Defends the single-range resident +// read against a corrupt directory ref (offset before the region, or a range +// that runs past it) before it is used to index the region buffer. +Status slice_dict_block_in_region(const BlockRef& ref, const RegionRef& dict_region, + size_t region_len, size_t* rel_off, size_t* len) { + if (ref.offset < dict_region.offset) { + return Status::Error( + "dict block: ref before dict region"); + } + size_t rel = 0; + size_t block_len = 0; + RETURN_IF_ERROR( + checked_size(ref.offset - dict_region.offset, "dict block: ref offset OOR", &rel)); + RETURN_IF_ERROR(checked_size(ref.length, "dict block: ref length OOR", &block_len)); + if (rel > region_len || block_len > region_len - rel) { + return Status::Error( + "dict block: ref past dict region"); + } + *rel_off = rel; + *len = block_len; + return Status::OK(); +} +} // namespace + +Status LogicalIndexReader::load_resident_dict_blocks() { + resident_dict_blocks_.clear(); + + const uint64_t max_bytes = dict_resident_max_bytes(); + if (max_bytes == 0 || dbd_.n_blocks() == 0) { + return Status::OK(); + } + + uint64_t total_bytes = 0; + for (uint32_t ord = 0; ord < dbd_.n_blocks(); ++ord) { + BlockRef ref {}; + RETURN_IF_ERROR(dbd_.get(ord, &ref)); + uint64_t block_bytes = 0; + RETURN_IF_ERROR(dict_block_memory_bytes(ref, &block_bytes)); + if (block_bytes > max_bytes - total_bytes) { + return Status::OK(); + } + total_bytes += block_bytes; + } + + // The resident blocks are physically contiguous within dict_region, so read + // the whole region in a SINGLE range read (was one read_at per block -> up to + // ~4 serial S3 rounds on a cold open) and decode each block from a sub-slice. + // The region buffer is <= the resident byte cap (<=256KB) and freed on return; + // each ResidentDictBlock keeps its own decoded copy. + const RegionRef& dict_region = section_refs().dict_region; + size_t region_len = 0; + RETURN_IF_ERROR( + checked_size(dict_region.length, "dict region: length out of range", ®ion_len)); + std::vector region; + RETURN_IF_ERROR(reader_->read_at(dict_region.offset, region_len, ®ion)); + if (region.size() != region_len) { + return Status::Error( + "dict region: short read"); + } + + resident_dict_blocks_.reserve(dbd_.n_blocks()); + for (uint32_t ord = 0; ord < dbd_.n_blocks(); ++ord) { + BlockRef ref {}; + RETURN_IF_ERROR(dbd_.get(ord, &ref)); + size_t rel_off = 0; + size_t block_len = 0; + RETURN_IF_ERROR( + slice_dict_block_in_region(ref, dict_region, region_len, &rel_off, &block_len)); + const Slice on_disk(region.data() + rel_off, block_len); + ResidentDictBlock block; + RETURN_IF_ERROR(decompress_dict_block_payload(on_disk, ref, &block.bytes)); + RETURN_IF_ERROR( + DictBlockReader::open(Slice(block.bytes), tier_, has_positions_, &block.reader)); + resident_dict_blocks_.push_back(std::move(block)); + } + return Status::OK(); +} + +Status LogicalIndexReader::dict_block_reader_for_ordinal( + uint32_t ordinal, DictBlockCache* cache, std::shared_ptr* pin, + const DictBlockReader** out) const { + pin->reset(); + if (!resident_dict_blocks_.empty()) { + if (resident_dict_blocks_.size() != dbd_.n_blocks() || + ordinal >= resident_dict_blocks_.size()) { + return Status::Error( + "logical_index: incomplete resident dict"); + } + // Resident blocks live for the reader lifetime: no pin needed. + *out = &resident_dict_blocks_[ordinal].reader; + return Status::OK(); + } + + // On-demand: decode into a heap-allocated DecodedDictBlock held by *pin so the + // reader's Slice never dangles. The loader (file read + optional zstd + CRC + + // anchor parse) runs OUTSIDE any cache bookkeeping; on a cache hit it is not + // called, so a block shared by several terms of one query decodes only once. + DictBlockCache::Loader loader = [&](std::shared_ptr* slot) -> Status { + BlockRef ref {}; + RETURN_IF_ERROR(dbd_.get(ordinal, &ref)); + auto block = std::make_shared(); + RETURN_IF_ERROR(open_dict_block(reader_, ref, tier_, has_positions_, &block->bytes, + &block->reader)); + *slot = std::move(block); + return Status::OK(); + }; + if (cache != nullptr) { + RETURN_IF_ERROR(cache->get_or_load(ordinal, loader, pin)); + } else { + RETURN_IF_ERROR(loader(pin)); + } + *out = &(*pin)->reader; + return Status::OK(); +} + +Status LogicalIndexReader::open(io::FileReader* file_reader, IndexTier tier, bool has_positions, + Slice meta_block, LogicalIndexReader* out) { + if (file_reader == nullptr) { + return Status::Error("logical_index: null file reader"); + } + if (out == nullptr) { + return Status::Error("logical_index: null out"); + } + if (meta_block.empty()) { + return Status::Error( + "logical_index: empty meta block"); + } + *out = LogicalIndexReader {}; + + out->reader_ = file_reader; + out->tier_ = tier; + out->has_positions_ = has_positions; + out->meta_block_.assign(meta_block.data(), meta_block.data() + meta_block.size()); + const Slice owned_meta(out->meta_block_); + + RETURN_IF_ERROR(PerIndexMetaReader::open(owned_meta, &out->meta_)); + // G13: large sti/dbd frames are stored zstd-compressed in the meta blob; the + // frame accessors materialize the original frames (raw frames alias the meta + // block and leave the scratch untouched). The scratches are transient: + // SampledTermIndexReader / DictBlockDirectoryReader fully materialize their + // decoded state on open and retain no view into the input frame. + { + std::vector scratch; + Slice frame; + RETURN_IF_ERROR(out->meta_.sampled_term_index_frame(&scratch, &frame)); + RETURN_IF_ERROR(SampledTermIndexReader::open(frame, &out->sti_)); + } + { + std::vector scratch; + Slice frame; + RETURN_IF_ERROR(out->meta_.dict_block_directory_frame(&scratch, &frame)); + RETURN_IF_ERROR(DictBlockDirectoryReader::open(frame, &out->dbd_)); + } + RETURN_IF_ERROR(out->load_resident_dict_blocks()); + + // Block-split bloom XFilter -- gated on RESIDENCY (P1 cold-read fix, see + // docs/perf/P1-cold-read-amplification.md). The bloom is set up and used ONLY + // when the whole (small) filter fits under the resident cap: it is read in + // full, verified, and kept in memory so probes are in-memory and enter the + // Doris searcher cache with the rest of the logical-index metadata. + // + // When NON-resident (the common case for a real text column, where the filter + // is many MB) the bloom is skipped ENTIRELY: not even the 28B header is read, + // and has_bsbf_ stays false. Every term then falls through to sti -> dict, + // which yields the true found/absent. At 1 MiB cache-block granularity a + // non-resident bloom never saves a physical block (an absent term still costs + // one dict block either way), so its 28B header + per-term 32B probes were pure + // cold read amplification. + const RegionRef& bsbf = out->meta_.section_refs().bsbf; + if (bsbf.length > 0 && bsbf.length <= bsbf_resident_max_bytes()) { + if (bsbf.length <= kBsbfHeaderSize) { + return Status::Error( + "logical_index: bsbf section too small"); + } + const uint64_t num_bytes = bsbf.length - kBsbfHeaderSize; + std::vector head; + RETURN_IF_ERROR(file_reader->read_at(bsbf.offset, bsbf.length, &head)); + if (head.size() < bsbf.length) { + return Status::Error( + "logical_index: short bsbf resident read"); + } + RETURN_IF_ERROR(format::BsbfHeader::parse(Slice(head.data(), kBsbfHeaderSize), bsbf.offset, + &out->bsbf_header_)); + // Cross-check the header geometry against the section ref. + if (out->bsbf_header_.num_bytes != num_bytes) { + return Status::Error( + "logical_index: bsbf header/section size mismatch"); + } + const Slice bitset(head.data() + kBsbfHeaderSize, out->bsbf_header_.num_bytes); + if (crc32c(bitset) != out->bsbf_header_.bitset_crc) { + return Status::Error( + "logical_index: bsbf bitset crc mismatch"); + } + out->bsbf_resident_bitset_.assign(bitset.data(), bitset.data() + bitset.size()); + out->has_bsbf_ = true; + out->bsbf_resident_ = true; + } + return Status::OK(); +} + +size_t LogicalIndexReader::memory_usage() const { + // meta_block_ retains the raw per-index meta bytes that sti_/dbd_ were decoded + // from, so it ALREADY (conservatively) double-counts their ENCODED size. The + // sti_/dbd_ heap_bytes() added below charge the DECODED resident copies -- an + // over-count, never an under-count. Do NOT drop meta_block_ to "de-dup" this: + // the charge feeds InvertedIndexSearcherCache and must not under-report RSS + // (which was the pre-fix bug: the derived sti_/dbd_/anchor heap was omitted, so + // the cache under-charged and over-committed). + size_t bytes = sizeof(*this) + meta_block_.capacity() + bsbf_resident_bitset_.capacity(); + bytes += sti_.heap_bytes(); + bytes += dbd_.heap_bytes(); + for (const auto& block : resident_dict_blocks_) { + bytes += sizeof(block) + block.bytes.capacity() + block.reader.heap_bytes(); + } + return bytes; +} + +Status LogicalIndexReader::lookup(std::string_view term, bool* found, DictEntry* entry, + uint64_t* frq_base, uint64_t* prx_base, + DictBlockCache* cache) const { + *found = false; + if (reader_ == nullptr) { + return Status::Error("logical_index: not opened"); + } + + // 1. XFilter fast rejection. A DEFINITELY-ABSENT term returns empty without the + // DICT read. The bloom is consulted ONLY when resident (loaded in memory at + // open) -- the in-memory bitset probe adds no disk read. A non-resident bloom is + // never set up (has_bsbf_ == false, P1 cold-read fix), so the lookup falls + // through to sti -> dict, which yields the true found/absent for every term. + if (has_bsbf_) { + const uint64_t h = bsbf_hash(term); + bool maybe = true; + if (bsbf_resident_) { + const uint32_t blk = format::bsbf_block_index(h, bsbf_header_.num_blocks); + maybe = format::bsbf_block_contains( + h, + bsbf_resident_bitset_.data() + static_cast(blk) * kBsbfBytesPerBlock); + } + if (!maybe) { + return Status::OK(); + } + } + + // 2. SampledTermIndex -> candidate block ordinal. + bool maybe = false; + uint32_t ordinal = 0; + RETURN_IF_ERROR(sti_.locate(term, &maybe, &ordinal)); + if (!maybe) { + return Status::OK(); + } + + // 3. Use a resident small-DICT block when present; otherwise read the DICT + // block on demand and parse it with the same validation path used at open. + // `pin` keeps an on-demand block alive through find_term (resident: null). + const DictBlockReader* br = nullptr; + std::shared_ptr pin; + RETURN_IF_ERROR(dict_block_reader_for_ordinal(ordinal, cache, &pin, &br)); + + bool hit = false; + RETURN_IF_ERROR(br->find_term(term, &hit, entry)); + if (!hit) { + return Status::OK(); + } + + *found = true; + *frq_base = br->frq_base(); + *prx_base = br->prx_base(); + return Status::OK(); +} + +Status LogicalIndexReader::visit_prefix_terms(std::string_view prefix, + const PrefixHitVisitor& visitor, + DictBlockCache* cache) const { + if (!visitor) { + return Status::Error( + "logical_index: null prefix visitor"); + } + if (reader_ == nullptr) { + return Status::Error("logical_index: not opened"); + } + + // Seek the start block: the SampledTermIndex block whose first term <= prefix + // (terms with `prefix` are >= prefix, so they begin in that block or later). + // If the prefix sorts before every sample (or is empty), start at block 0. + uint32_t start = 0; + if (!prefix.empty()) { + bool maybe = false; + uint32_t ordinal = 0; + RETURN_IF_ERROR(sti_.locate(prefix, &maybe, &ordinal)); + if (maybe) { + start = ordinal; + } + } + + for (uint32_t ord = start; ord < dbd_.n_blocks(); ++ord) { + const DictBlockReader* br = nullptr; + std::shared_ptr pin; + RETURN_IF_ERROR(dict_block_reader_for_ordinal(ord, cache, &pin, &br)); + + // Stream this block's prefix range: anchor-jump past pre-prefix segments, + // decode only the bodies we keep, and stop at the first term past the + // range (decode_all materialized every entry of every scanned block). + // The visitor still owns final term acceptance, so results are identical; + // `br`/`pin` stay alive across this synchronous call. + bool prefix_exhausted = false; + bool visitor_stopped = false; + RETURN_IF_ERROR(br->visit_prefix_range( + prefix, /*accept_key=*/ {}, + [&](DictEntry&& e, bool* stop) -> Status { + PrefixHit hit; + hit.term = e.term; + hit.entry = std::move(e); + hit.frq_base = br->frq_base(); + hit.prx_base = br->prx_base(); + RETURN_IF_ERROR(visitor(std::move(hit), stop)); + visitor_stopped = *stop; + return Status::OK(); + }, + &prefix_exhausted)); + if (visitor_stopped || prefix_exhausted) { + return Status::OK(); + } + } + return Status::OK(); +} + +Status LogicalIndexReader::prefix_terms(std::string_view prefix, std::vector* const out, + int32_t max_terms, DictBlockCache* cache) const { + if (out == nullptr) { + return Status::Error("logical_index: null out"); + } + out->clear(); + return visit_prefix_terms( + prefix, + [&](PrefixHit&& hit, bool* stop) { + out->push_back(std::move(hit)); + *stop = max_terms > 0 && out->size() >= static_cast(max_terms); + return Status::OK(); + }, + cache); +} + +namespace { + +// Validates a pod_ref window locator against the posting region and returns the +// absolute window range (after the prelude). Rejects corrupt locators rather +// than letting size_t underflow / uint64 overflow reach read_at. +Status resolve_window(const format::RegionRef& section, uint64_t base, uint64_t off_delta, + uint64_t total_len, uint64_t prelude_len, uint64_t* abs_off, uint64_t* len) { + if (prelude_len > total_len) { + return Status::Error( + "logical_index: prelude_len exceeds window len"); + } + const uint64_t in_region = base + off_delta; + if (in_region < base) { + return Status::Error( + "logical_index: locator overflow"); + } + if (in_region > section.length || total_len > section.length - in_region) { + return Status::Error( + "logical_index: window past posting region"); + } + *abs_off = section.offset + in_region + prelude_len; + *len = total_len - prelude_len; + return Status::OK(); +} + +} // namespace + +Status LogicalIndexReader::resolve_frq_window(const format::DictEntry& entry, uint64_t frq_base, + uint64_t* abs_off, uint64_t* len) const { + return resolve_window(section_refs().posting_region, frq_base, entry.frq_off_delta, + entry.frq_len, entry.prelude_len, abs_off, len); +} + +Status LogicalIndexReader::resolve_prx_window(const format::DictEntry& entry, uint64_t prx_base, + uint64_t* abs_off, uint64_t* len) const { + // .prx windows carry no prelude (prelude_len = 0); both spans live in the + // same posting region (prx span precedes frq span for the same term). + return resolve_window(section_refs().posting_region, prx_base, entry.prx_off_delta, + entry.prx_len, 0, abs_off, len); +} + +} // namespace doris::snii::reader diff --git a/be/src/storage/index/snii/reader/logical_index_reader.h b/be/src/storage/index/snii/reader/logical_index_reader.h new file mode 100644 index 00000000000000..7e1b8c327786a9 --- /dev/null +++ b/be/src/storage/index/snii/reader/logical_index_reader.h @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/format/bsbf.h" +#include "storage/index/snii/format/dict_block.h" +#include "storage/index/snii/format/dict_block_directory.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/per_index_meta.h" +#include "storage/index/snii/format/sampled_term_index.h" +#include "storage/index/snii/format/stats_block.h" +#include "storage/index/snii/io/file_reader.h" + +// LogicalIndexReader -- read-side counterpart of LogicalIndexWriter for one +// logical index. It owns the resident per-index meta sub-readers (XFilter, +// SampledTermIndex, DICT block directory, StatsBlock, SectionRefs) parsed from +// the per-index meta block, and resolves a query term to its DictEntry through +// the documented lookup flow: +// XFilter (reject absent) -> SampledTermIndex (candidate block ordinal) -> +// DICT block directory (block range) -> resident small-DICT block or one +// range read of the DICT block -> DictBlockReader::find_term. +// +// lookup() also returns the block's frq_base/prx_base (captured by the +// DictBlockReader) so callers can resolve a pod_ref entry's absolute .frq/.prx +// offsets via the writer's contract. Both deltas index into the SAME +// interleaved posting region (prx_base == frq_base; the prx span precedes the +// frq span): +// abs_frq = posting_region.offset + frq_base + entry.frq_off_delta +// abs_prx = posting_region.offset + prx_base + entry.prx_off_delta +// +// The reader copies the meta block bytes at open so every parsed sub-reader has +// stable backing storage for the reader lifetime. +namespace doris::snii::reader { + +// Forward-declared: this widely-included header only names DictBlockCache* and +// shared_ptr*; the full definitions are pulled into the +// .cpp and into tests that construct a cache. Keeps the request-scoped cache +// header out of the ~500 TUs that transitively include this one. +struct DecodedDictBlock; +class DictBlockCache; + +class LogicalIndexReader { +public: + LogicalIndexReader() = default; + + // Parses the per-index meta block and binds the reader to file_reader. + // file_reader / meta_block must outlive this reader. + static Status open(io::FileReader* file_reader, format::IndexTier tier, bool has_positions, + Slice meta_block, LogicalIndexReader* out); + + // Resolves term to a DictEntry. *found=false when the term is absent (XFilter + // rejection, out-of-range sample, or DICT-block miss). On a hit, *entry is + // filled and *frq_base / *prx_base carry the candidate block's bases. + // + // `cache` is an optional REQUEST-SCOPED DictBlockCache: when a single query + // threads one cache through its per-term lookups, an on-demand DICT block hit + // by several terms is decoded once instead of once per term. nullptr keeps the + // pre-existing behavior (each lookup materializes its own block). The cache is + // caller-owned, single-threaded, and never mutates this (const) reader. + Status lookup(std::string_view term, bool* found, format::DictEntry* entry, uint64_t* frq_base, + uint64_t* prx_base, DictBlockCache* cache = nullptr) const; + + // One enumerated term whose key has the requested prefix, with its DictEntry + // and the owning DICT block's frq/prx bases (for posting resolution). + struct PrefixHit { + std::string term; + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + }; + + using PrefixHitVisitor = std::function; + + // Ordered term enumeration: every term with `prefix`, in lexicographic order, + // by seeking the start DICT block via the SampledTermIndex and scanning + // forward across contiguous blocks until the terms pass the prefix range. + // Empty prefix enumerates all terms. This is the contiguous-DICT-block design + // the term-anchor layout was built for (MATCH_PHRASE_PREFIX / prefix / range + // queries). The visitor form avoids materializing all hits when callers only + // need a bounded expansion. + Status visit_prefix_terms(std::string_view prefix, const PrefixHitVisitor& visitor, + DictBlockCache* cache = nullptr) const; + Status prefix_terms(std::string_view prefix, std::vector* const out, + int32_t max_terms = 0, DictBlockCache* cache = nullptr) const; + + // Resolves a pod_ref entry's absolute .frq / .prx window byte range, + // validating the locator against the posting_region length (defends against + // corrupt entries: prelude_len > frq_len underflow, or off_delta+len past the + // region). Both windows resolve against the single posting_region. *abs_off + // is the absolute file offset of the window (after prelude); *len its byte + // length. + Status resolve_frq_window(const format::DictEntry& entry, uint64_t frq_base, uint64_t* abs_off, + uint64_t* len) const; + Status resolve_prx_window(const format::DictEntry& entry, uint64_t prx_base, uint64_t* abs_off, + uint64_t* len) const; + + const format::SectionRefs& section_refs() const { return meta_.section_refs(); } + const format::StatsBlock& stats() const { return meta_.stats(); } + format::IndexTier tier() const { return tier_; } + bool has_positions() const { return has_positions_; } + // Effective phrase-bigram df-prune thresholds the writer applied to this + // index (G01 min / G15 max), from the optional kBigramPruneInfo meta + // section. Both 0 == absent == legacy segment: every adjacent pair was + // materialized, so a bigram dict miss means "no adjacency" (empty phrase + // result). Either non-zero: pairs were df-pruned at build (low-df below + // min, near-ubiquitous above max), so the 2-term phrase bigram path must + // treat a dict miss as "fall back to generic positions verification". + // max is 0 on every pre-G15 segment (single-varint section payload). + uint64_t bigram_prune_min_df() const { return meta_.bigram_prune_min_df(); } + uint64_t bigram_prune_max_df() const { return meta_.bigram_prune_max_df(); } + bool phrase_bigrams_deferred() const { return meta_.phrase_bigrams_deferred(); } + io::FileReader* reader() const { return reader_; } + size_t memory_usage() const; + +private: + io::FileReader* reader_ = nullptr; + format::IndexTier tier_ = format::IndexTier::kT1; + bool has_positions_ = false; + std::vector meta_block_; + format::PerIndexMetaReader meta_; + format::SampledTermIndexReader sti_; + format::DictBlockDirectoryReader dbd_; + format::BsbfHeader bsbf_header_; // resident header (from section ref) + bool has_bsbf_ = false; + // L0 tiering: when the bsbf section is small (<= kBsbfResidentMaxBytes) its + // whole bitset is loaded here at open -> in-memory probe, no per-lookup + // round. Larger filters keep only the parsed header here, so the small + // header enters Doris searcher cache and lookup reads just one 32-byte body + // block for an L1 probe. + bool bsbf_resident_ = false; + std::vector bsbf_resident_bitset_; + + // Small DICT blocks are opened once with the index so exact lookups avoid an + // otherwise serial S3 round for the term dictionary. Empty means the + // dictionary exceeded the resident threshold and lookup/prefix enumeration + // read blocks on demand. Each DictBlockReader holds a Slice into the owning + // bytes. + struct ResidentDictBlock { + std::vector bytes; + format::DictBlockReader reader; + }; + Status load_resident_dict_blocks(); + // Resolves the DictBlockReader for `ordinal`. Resident blocks return a pointer + // into the reader-owned resident set with *pin left null (stable for the reader + // lifetime). On-demand blocks are decoded (optionally via the request-scoped + // `cache`) into a heap-allocated DecodedDictBlock; *pin holds it alive so *out + // never dangles under a later cache eviction. Callers must keep *pin alive for + // as long as they use *out. + Status dict_block_reader_for_ordinal(uint32_t ordinal, DictBlockCache* cache, + std::shared_ptr* pin, + const format::DictBlockReader** out) const; + std::vector resident_dict_blocks_; +}; + +} // namespace doris::snii::reader diff --git a/be/src/storage/index/snii/reader/snii_segment_reader.cpp b/be/src/storage/index/snii/reader/snii_segment_reader.cpp new file mode 100644 index 00000000000000..5d7d4486012822 --- /dev/null +++ b/be/src/storage/index/snii/reader/snii_segment_reader.cpp @@ -0,0 +1,210 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/reader/snii_segment_reader.h" + +#include +#include + +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/per_index_meta.h" +#include "storage/index/snii/format/stats_block.h" +#include "storage/index/snii/format/tail_pointer.h" + +namespace doris::snii::reader { + +using format::IndexTier; +using format::PerIndexMetaReader; +using format::StatsBlock; +using format::TailMetaRegionReader; +using format::TailPointer; + +namespace { + +// Reads the fixed tail pointer (last tail_pointer_size() bytes) of the file. +Status ReadTailPointer(io::FileReader* reader, TailPointer* tp) { + const size_t tp_size = format::tail_pointer_size(); + const uint64_t total = reader->size(); + if (total < tp_size) { + return Status::Error( + "segment: file smaller than tail pointer"); + } + std::vector buf; + RETURN_IF_ERROR(reader->read_at(total - tp_size, tp_size, &buf)); + return format::decode_tail_pointer(Slice(buf), tp); +} + +Status ReadTailMetaHeader(io::FileReader* reader, const TailPointer& tp, + format::TailMetaRegionHeader* header) { + const size_t header_size = format::tail_meta_header_size(); + if (tp.meta_region_length < header_size) { + return Status::Error( + "segment: tail meta region smaller than header"); + } + std::vector buf; + RETURN_IF_ERROR(reader->read_at(tp.meta_region_offset, header_size, &buf)); + return format::TailMetaRegionReader::parse_header(Slice(buf), header); +} + +} // namespace + +Status SniiSegmentReader::open(io::FileReader* const reader, SniiSegmentReader* const out) { + if (reader == nullptr) { + return Status::Error("segment: null reader"); + } + if (out == nullptr) { + return Status::Error("segment: null out"); + } + + // NOTE: the per-segment bootstrap header (offset 0) is intentionally NOT read + // here. It is still WRITTEN on disk for inspect tooling, but its only runtime + // role -- gating the container format_version -- is already covered (more + // strictly, exact-match) by ReadTailPointer below, which validates the 'TAIL' + // magic, the embedded format_version == kFormatVersion, and the tail crc. The + // open() path reads only the file tail, avoiding a redundant offset-0 cache + // block / remote round-trip per segment on cold queries. + // + // The bootstrap header's min_reader_version forward-compat gate is intentionally + // retired together with this read: the tail pointer carries format_version but not + // min_reader_version, so future format evolution must bump format_version (which the + // tail rejects on exact-match) rather than rely on a min_reader_version bump under a + // stable format_version. + TailPointer tp; + RETURN_IF_ERROR(ReadTailPointer(reader, &tp)); + if (tp.meta_region_length == 0) { + return Status::Error( + "segment: empty tail meta region"); + } + + format::TailMetaRegionHeader meta_header; + RETURN_IF_ERROR(ReadTailMetaHeader(reader, tp, &meta_header)); + if (meta_header.meta_region_len != tp.meta_region_length) { + return Status::Error( + "segment: tail meta length mismatch"); + } + + std::vector directory; + if (meta_header.directory_offset > UINT64_MAX - tp.meta_region_offset) { + return Status::Error( + "segment: tail meta directory file offset overflow"); + } + RETURN_IF_ERROR(reader->read_at(tp.meta_region_offset + meta_header.directory_offset, + meta_header.directory_length, &directory)); + + out->reader_ = reader; + out->meta_region_offset_ = tp.meta_region_offset; + out->meta_region_length_ = tp.meta_region_length; + return TailMetaRegionReader::open_directory(meta_header, Slice(directory), + &out->region_reader_); +} + +Status SniiSegmentReader::read_index_meta(uint64_t index_id, std::string_view suffix, + std::vector* const out) const { + if (out == nullptr) { + return Status::Error("segment: null meta out"); + } + if (reader_ == nullptr) { + return Status::Error("segment: not opened"); + } + + bool found = false; + format::LogicalIndexRef ref; + RETURN_IF_ERROR(region_reader_.find_ref(index_id, suffix, &found, &ref)); + if (!found) { + return Status::Error( + "segment: logical index not found"); + } + if (ref.meta_off > meta_region_length_ || ref.meta_len > meta_region_length_ - ref.meta_off) { + return Status::Error( + "segment: logical index meta out of tail region"); + } + if (ref.meta_off > UINT64_MAX - meta_region_offset_) { + return Status::Error( + "segment: logical index meta file offset overflow"); + } + RETURN_IF_ERROR(reader_->read_at(meta_region_offset_ + ref.meta_off, ref.meta_len, out)); + return Status::OK(); +} + +Status SniiSegmentReader::index_exists(uint64_t index_id, std::string_view suffix, + bool* const exists) const { + if (exists == nullptr) { + return Status::Error("segment: null exists out"); + } + if (reader_ == nullptr) { + return Status::Error("segment: not opened"); + } + + format::LogicalIndexRef ref; + return region_reader_.find_ref(index_id, suffix, exists, &ref); +} + +Status SniiSegmentReader::open_index_from_meta(Slice meta_bytes, + LogicalIndexReader* const out) const { + if (out == nullptr) { + return Status::Error("segment: null index out"); + } + if (reader_ == nullptr) { + return Status::Error("segment: not opened"); + } + // Determine tier / positions capability from the per-index meta. Positions + // capability is read from the PERSISTED header flag (kHasPositions), NOT from + // any region length: after the frq/prx merge, posting_region.length is non-zero + // for ANY index with a pod_ref term -- docs-only included -- so a region-length + // heuristic would mis-classify a docs-only index as positional and make + // DictBlockReader::check_flags hard-fail. The "|| has_norms" is kept only as a + // defensive belt-and-suspenders (a scoring index always has positions). + PerIndexMetaReader meta; + RETURN_IF_ERROR(PerIndexMetaReader::open(meta_bytes, &meta)); + const bool has_norms = meta.section_refs().norms.length > 0; + const bool has_positions = meta.has_positions() || has_norms; + IndexTier tier = IndexTier::kT1; + if (has_norms) { + tier = IndexTier::kT3; + } else if (has_positions) { + tier = IndexTier::kT2; + } + + return LogicalIndexReader::open(reader_, tier, has_positions, meta_bytes, out); +} + +Status SniiSegmentReader::open_index(uint64_t index_id, std::string_view suffix, + LogicalIndexReader* const out) const { + std::vector meta_bytes; + RETURN_IF_ERROR(read_index_meta(index_id, suffix, &meta_bytes)); + return open_index_from_meta(Slice(meta_bytes), out); +} + +Status SniiSegmentReader::section_refs_for_index(uint64_t index_id, std::string_view suffix, + format::SectionRefs* const out) const { + if (out == nullptr) { + return Status::Error("segment: null section refs out"); + } + if (reader_ == nullptr) { + return Status::Error("segment: not opened"); + } + + std::vector meta_bytes; + RETURN_IF_ERROR(read_index_meta(index_id, suffix, &meta_bytes)); + PerIndexMetaReader meta; + RETURN_IF_ERROR(PerIndexMetaReader::open(Slice(meta_bytes), &meta)); + *out = meta.section_refs(); + return Status::OK(); +} + +} // namespace doris::snii::reader diff --git a/be/src/storage/index/snii/reader/snii_segment_reader.h b/be/src/storage/index/snii/reader/snii_segment_reader.h new file mode 100644 index 00000000000000..d8233bd6d085d2 --- /dev/null +++ b/be/src/storage/index/snii/reader/snii_segment_reader.h @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/format/per_index_meta.h" +#include "storage/index/snii/format/tail_meta_region.h" +#include "storage/index/snii/io/file_reader.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// SniiSegmentReader -- entry point for the SNII segment read path. It opens a +// single .idx container through a (possibly metered) io::FileReader and exposes +// its logical indexes. open() reads only the file tail: +// 1. the fixed tail pointer (last tail_pointer_size() bytes), which also gates +// the container format_version ('TAIL' magic + format_version exact-match + +// tail crc), and +// 2. the tail meta header + logical-index directory. +// The bootstrap header at offset 0 is still WRITTEN on disk (for inspect tooling) +// but is intentionally NOT read at open: its only runtime role (the container +// version gate) is already covered, more strictly, by the tail pointer, so +// skipping it avoids a redundant offset-0 cache block / remote round-trip per +// segment on cold queries. +// Per-index meta blocks are read lazily by open_index() so opening one logical +// index does not read every other logical index's metadata. +// +// open_index() then materializes one LogicalIndexReader from the per-index meta +// block of a given (index_id, suffix); query functions operate on that reader. +namespace doris::snii::reader { + +class SniiSegmentReader { +public: + SniiSegmentReader() = default; + + // Reads the tail pointer + tail meta region from reader (the offset-0 + // bootstrap header is not read; the tail pointer gates the container version). + // reader must outlive the returned SniiSegmentReader and every + // LogicalIndexReader opened from it. reader == nullptr / out == nullptr -> + // InvalidArgument; structural problems -> Corruption / Unsupported. + static Status open(io::FileReader* const reader, SniiSegmentReader* const out); + + uint32_t n_logical_indexes() const { return region_reader_.n_logical_indexes(); } + + // Reads the per-index meta block bytes for (index_id, suffix). The returned + // vector owns the exact meta block and may be passed to open_index_from_meta(). + Status read_index_meta(uint64_t index_id, std::string_view suffix, + std::vector* const out) const; + Status index_exists(uint64_t index_id, std::string_view suffix, bool* const exists) const; + + Status open_index_from_meta(Slice meta_bytes, LogicalIndexReader* const out) const; + + // Loads the per-index meta block for (index_id, suffix) and builds a + // LogicalIndexReader bound to the same FileReader. Absent index -> NotFound. + Status open_index(uint64_t index_id, std::string_view suffix, + LogicalIndexReader* const out) const; + Status section_refs_for_index(uint64_t index_id, std::string_view suffix, + format::SectionRefs* const out) const; + + io::FileReader* reader() const { return reader_; } + +private: + io::FileReader* reader_ = nullptr; + uint64_t meta_region_offset_ = 0; + uint64_t meta_region_length_ = 0; + format::TailMetaRegionReader region_reader_; +}; + +} // namespace doris::snii::reader diff --git a/be/src/storage/index/snii/reader/windowed_posting.cpp b/be/src/storage/index/snii/reader/windowed_posting.cpp new file mode 100644 index 00000000000000..8e824571ce7377 --- /dev/null +++ b/be/src/storage/index/snii/reader/windowed_posting.cpp @@ -0,0 +1,295 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/reader/windowed_posting.h" + +#include +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/io/batch_range_fetcher.h" + +namespace doris::snii::reader { + +using format::DictEntry; +using format::FrqPreludeReader; +using format::FrqRegionMeta; +using format::WindowMeta; + +namespace { + +// Resolves the absolute file offset of the prelude bytes for a windowed entry. +// The frq span lives in the interleaved posting region (after the term's prx span). +uint64_t PreludeAbs(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t frq_base) { + const auto& region = idx.section_refs().posting_region; + return region.offset + frq_base + entry.frq_off_delta; +} + +// Validates that [off, off+len) fits within [0, total). +Status InBounds(uint64_t off, uint64_t len, uint64_t total) { + if (off > total || len > total - off) { + return Status::Error( + "windowed_posting: range out of section"); + } + return Status::OK(); +} + +// Block geometry of a windowed entry's grouped .frq payload (all offsets absolute). +struct BlockGeometry { + uint64_t dd_block_off = 0; // absolute start of the dd-block + uint64_t dd_block_len = 0; + uint64_t freq_block_off = 0; // absolute start of the freq-block + uint64_t freq_block_len = 0; + uint64_t frq_region_len = 0; // entry.frq_len - prelude_len (dd-block + freq-block) +}; + +// Derives the dd-block / freq-block absolute ranges from the entry + prelude, +// validating they tile the post-prelude .frq region exactly. +Status ResolveBlocks(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t frq_base, + const FrqPreludeReader& prelude, BlockGeometry* g) { + if (entry.prelude_len > entry.frq_len) { + return Status::Error( + "windowed_posting: prelude_len exceeds frq_len"); + } + const uint64_t frq_window_start = PreludeAbs(idx, entry, frq_base) + entry.prelude_len; + g->frq_region_len = entry.frq_len - entry.prelude_len; + g->dd_block_len = prelude.dd_block_len(); + g->freq_block_len = prelude.freq_block_len(); + // dd-block + freq-block must fit exactly within the post-prelude region. + if (g->dd_block_len > g->frq_region_len || + g->freq_block_len > g->frq_region_len - g->dd_block_len) { + return Status::Error( + "windowed_posting: blocks exceed frq region"); + } + g->dd_block_off = frq_window_start; + g->freq_block_off = frq_window_start + g->dd_block_len; + return Status::OK(); +} + +// Per-window decode state for the full-posting path. +struct WindowSlices { + WindowMeta meta; + Slice dd_region; + Slice freq_region; + Slice prx_window; +}; + +// Carves window w's dd (and freq when want_freq) sub-slices out of the fetched +// blocks, validating each locator against its block length. +Status CarveRegionSlices(const WindowMeta& m, Slice dd_block, Slice freq_block, bool want_freq, + WindowSlices* out) { + RETURN_IF_ERROR(InBounds(m.dd_off, m.dd_disk_len, dd_block.size())); + out->dd_region = + dd_block.subslice(static_cast(m.dd_off), static_cast(m.dd_disk_len)); + if (!want_freq) return Status::OK(); + RETURN_IF_ERROR(InBounds(m.freq_off, m.freq_disk_len, freq_block.size())); + out->freq_region = freq_block.subslice(static_cast(m.freq_off), + static_cast(m.freq_disk_len)); + return Status::OK(); +} + +// Decodes window w from the fetched blocks (+ optional prx slice) and appends to out. +Status AppendWindow(const WindowSlices& ws, bool want_positions, bool want_freq, + DecodedPosting* out) { + std::vector docids, freqs; + std::vector> pos; + RETURN_IF_ERROR(decode_window_slices(ws.meta, ws.dd_region, ws.freq_region, ws.prx_window, + want_positions, want_freq, &docids, &freqs, &pos)); + out->docids.insert(out->docids.end(), docids.begin(), docids.end()); + out->freqs.insert(out->freqs.end(), freqs.begin(), freqs.end()); + if (want_positions) { + for (auto& v : pos) out->positions.push_back(std::move(v)); + } + return Status::OK(); +} + +} // namespace + +Status fetch_windowed_prelude(const LogicalIndexReader& idx, const DictEntry& entry, + uint64_t frq_base, FrqPreludeReader* prelude) { + if (entry.prelude_len == 0) { + return Status::Error( + "windowed_posting: windowed entry has no prelude"); + } + if (entry.prelude_len > entry.frq_len) { + return Status::Error( + "windowed_posting: prelude_len exceeds frq_len"); + } + const uint64_t prelude_abs = PreludeAbs(idx, entry, frq_base); + io::BatchRangeFetcher fetcher(idx.reader()); + const size_t h = fetcher.add(prelude_abs, entry.prelude_len); + RETURN_IF_ERROR(fetcher.fetch()); + return FrqPreludeReader::open(fetcher.get(h), prelude); +} + +Status windowed_window_range(const LogicalIndexReader& idx, const DictEntry& entry, + uint64_t frq_base, uint64_t prx_base, const FrqPreludeReader& prelude, + uint32_t w, bool want_positions, bool want_freq, WindowAbsRange* out) { + if (out == nullptr) + return Status::Error("windowed_posting: null range"); + *out = WindowAbsRange {}; + BlockGeometry g; + RETURN_IF_ERROR(ResolveBlocks(idx, entry, frq_base, prelude, &g)); + WindowMeta meta; + RETURN_IF_ERROR(prelude.window(w, &meta)); + + // dd sub-range within the dd-block. + RETURN_IF_ERROR(InBounds(meta.dd_off, meta.dd_disk_len, g.dd_block_len)); + out->dd_off = g.dd_block_off + meta.dd_off; + out->dd_len = meta.dd_disk_len; + + if (want_freq) { + // Symmetric to the positions guard below: a G16 freq-elided posting + // (freq-dropped index or prune-mode bigram) declares has_freq=false in + // its prelude flags. INVALID_ARGUMENT and not FILE_CORRUPTED: the + // Doris segment iterator silently downgrades the corruption code to a + // non-index evaluation, which would mask this by-design layout. + if (!prelude.has_freq()) { + return Status::Error( + "windowed_posting: freqs requested but prelude has none"); + } + RETURN_IF_ERROR(InBounds(meta.freq_off, meta.freq_disk_len, g.freq_block_len)); + out->freq_off = g.freq_block_off + meta.freq_off; + out->freq_len = meta.freq_disk_len; + } + + if (!want_positions) return Status::OK(); + if (!prelude.has_prx()) { + return Status::Error( + "windowed_posting: positions requested but prelude has none"); + } + const uint64_t prx_region_start = + idx.section_refs().posting_region.offset + prx_base + entry.prx_off_delta; + RETURN_IF_ERROR(InBounds(meta.prx_off, meta.prx_len, entry.prx_len)); + out->prx_off = prx_region_start + meta.prx_off; + out->prx_len = meta.prx_len; + return Status::OK(); +} + +Status decode_window_slices(const WindowMeta& meta, Slice dd_region, Slice freq_region, + Slice prx_window, bool want_positions, bool want_freq, + std::vector* docids, std::vector* freqs, + std::vector>* positions) { + FrqRegionMeta dd_meta; + dd_meta.zstd = meta.dd_zstd; + dd_meta.uncomp_len = meta.dd_uncomp_len; + dd_meta.disk_len = meta.dd_disk_len; + dd_meta.crc = meta.crc_dd; + dd_meta.verify_crc = meta.verify_crc; + RETURN_IF_ERROR(format::decode_dd_region(dd_region, dd_meta, meta.win_base, docids)); + if (docids->size() != meta.doc_count) { + return Status::Error( + "windowed_posting: frq doc_count mismatch"); + } + if (want_freq) { + FrqRegionMeta freq_meta; + freq_meta.zstd = meta.freq_zstd; + freq_meta.uncomp_len = meta.freq_uncomp_len; + freq_meta.disk_len = meta.freq_disk_len; + freq_meta.crc = meta.crc_freq; + freq_meta.verify_crc = meta.verify_crc; + RETURN_IF_ERROR(format::decode_freq_region(freq_region, freq_meta, meta.doc_count, freqs)); + } else { + freqs->clear(); + } + if (!want_positions) return Status::OK(); + + ByteSource psrc(prx_window); + RETURN_IF_ERROR(format::read_prx_window(&psrc, positions)); + if (positions->size() != docids->size()) { + return Status::Error( + "windowed_posting: prx/frq doc-count mismatch"); + } + return Status::OK(); +} + +namespace { + +// Fetches the dd-block (always), the freq-block (when want_freq) and the whole .prx +// region (when want_positions) of a windowed entry in ONE batch and returns the +// in-memory block slices. The dd-block is a single contiguous range -> the +// docid-only / phrase path reads it as one Range GET (the byte-saving core). +Status FetchBlocks(const LogicalIndexReader& idx, const DictEntry& entry, uint64_t prx_base, + const BlockGeometry& g, bool want_positions, bool want_freq, + io::BatchRangeFetcher* fetcher, size_t* dd_h, size_t* freq_h, size_t* prx_h) { + *dd_h = fetcher->add(g.dd_block_off, g.dd_block_len); + if (want_freq) { + *freq_h = fetcher->add(g.freq_block_off, g.freq_block_len); + } + if (want_positions) { + const uint64_t prx_region_start = + idx.section_refs().posting_region.offset + prx_base + entry.prx_off_delta; + *prx_h = fetcher->add(prx_region_start, entry.prx_len); + } + return fetcher->fetch(); +} + +} // namespace + +Status read_windowed_posting(const LogicalIndexReader& idx, const DictEntry& entry, + uint64_t frq_base, uint64_t prx_base, bool want_positions, + bool want_freq, DecodedPosting* out) { + if (out == nullptr) { + return Status::Error("windowed_posting: null out"); + } + *out = DecodedPosting {}; + + FrqPreludeReader prelude; + RETURN_IF_ERROR(fetch_windowed_prelude(idx, entry, frq_base, &prelude)); + if (want_positions && !prelude.has_prx()) { + return Status::Error( + "windowed_posting: positions requested but prelude has none"); + } + // G16 freq-elided postings (freq-dropped index or prune-mode bigram) + // declare has_freq=false; fail with the semantic error instead of a deep + // region-decode corruption. INVALID_ARGUMENT so the Doris segment + // iterator's corruption downgrade never masks this by-design layout. + if (want_freq && !prelude.has_freq()) { + return Status::Error( + "windowed_posting: freqs requested but prelude has none"); + } + BlockGeometry g; + RETURN_IF_ERROR(ResolveBlocks(idx, entry, frq_base, prelude, &g)); + + io::BatchRangeFetcher fetcher(idx.reader()); + size_t dd_h = 0, freq_h = 0, prx_h = 0; + RETURN_IF_ERROR(FetchBlocks(idx, entry, prx_base, g, want_positions, want_freq, &fetcher, &dd_h, + &freq_h, &prx_h)); + const Slice dd_block = fetcher.get(dd_h); + const Slice freq_block = want_freq ? fetcher.get(freq_h) : Slice(); + const Slice prx_region = want_positions ? fetcher.get(prx_h) : Slice(); + + const uint32_t n = prelude.n_windows(); + for (uint32_t w = 0; w < n; ++w) { + WindowSlices ws; + RETURN_IF_ERROR(prelude.window(w, &ws.meta)); + RETURN_IF_ERROR(CarveRegionSlices(ws.meta, dd_block, freq_block, want_freq, &ws)); + if (want_positions) { + RETURN_IF_ERROR(InBounds(ws.meta.prx_off, ws.meta.prx_len, prx_region.size())); + ws.prx_window = prx_region.subslice(static_cast(ws.meta.prx_off), + static_cast(ws.meta.prx_len)); + } + RETURN_IF_ERROR(AppendWindow(ws, want_positions, want_freq, out)); + } + return Status::OK(); +} + +} // namespace doris::snii::reader diff --git a/be/src/storage/index/snii/reader/windowed_posting.h b/be/src/storage/index/snii/reader/windowed_posting.h new file mode 100644 index 00000000000000..dee527430aff08 --- /dev/null +++ b/be/src/storage/index/snii/reader/windowed_posting.h @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// WindowedPostingReader -- shared read-side decode of a windowed term's posting +// from its two-level frq_prelude + GROUPED dd-block / freq-block (design 1.6). +// +// A windowed pod_ref entry's .frq payload is laid out +// [prelude][dd-block][freq-block] +// where the dd-block concatenates every window's dd_region and the freq-block +// every window's freq_region. The docs-only prefix [prelude][dd-block] is ONE +// contiguous run. This helper: +// 1. range-fetches the prelude (prelude_len bytes) and parses the directory, +// 2. range-fetches the WHOLE dd-block in ONE contiguous range (and, for +// scoring, +// the whole freq-block in one more range), +// 3. decodes each window's dd region (and freq region) from the in-memory +// blocks +// via the prelude metadata (dd_off/dd_disk_len, freq_off/freq_disk_len), +// and concatenates the per-window docids / freqs / positions. +// +// The slim/inline single-window path is handled by the term/phrase/scoring +// callers directly; this helper is for enc=windowed entries only. +namespace doris::snii::reader { + +// Coalesce gap (bytes) used when batch-fetching MULTIPLE dd sub-ranges of the +// SAME term (the phrase window-skip path): dd regions of one term are +// contiguous in the dd-block, so merging reads separated by <= this gap into +// one physical Range GET trades a little over-read for fewer remote GETs (the +// design's higher-priority metric). Only applied to same-term multi-window +// batches, never to cross-term. +inline constexpr uint64_t kSameTermCoalesceGap = 16 * 1024; + +// Full decoded posting for one windowed term (docids ascending across windows). +struct DecodedPosting { + std::vector docids; + std::vector freqs; // aligned with docids + std::vector> positions; // aligned; empty when no prx +}; + +// Decodes the entire windowed posting. want_positions requires the index to +// have positions (and the entry to carry prx). want_freq selects whether the +// freq-block is fetched + decoded: when false ONLY the contiguous +// [prelude][dd-block] prefix is fetched (docid-only / phrase callers) and +// DecodedPosting.freqs stays empty; when true the freq-block is additionally +// fetched (scoring). Returns Corruption on any prelude/block inconsistency +// (doc-count mismatch, out-of-range offsets). +Status read_windowed_posting(const LogicalIndexReader& idx, const format::DictEntry& entry, + uint64_t frq_base, uint64_t prx_base, bool want_positions, + bool want_freq, DecodedPosting* out); + +// --- Sub-block (window) skipping helpers (shared with phrase / selective WAND) +// -- +// +// These expose the per-window dd/freq/prx addressing within the grouped blocks +// so the skip path can fetch ONLY the windows covering candidate docids (their +// dd sub-ranges within the dd-block, near-contiguous and coalesce-friendly) +// instead of the whole posting, without duplicating the offset arithmetic. + +// Absolute file byte ranges of one window's regions. dd is always valid; freq +// is valid only when want_freq; prx is valid only when want_positions (and +// has_prx). +struct WindowAbsRange { + uint64_t dd_off = 0; + uint64_t dd_len = 0; + uint64_t freq_off = 0; + uint64_t freq_len = 0; + uint64_t prx_off = 0; + uint64_t prx_len = 0; +}; + +// Fetches + parses the two-level prelude of a windowed entry (one batched +// read). +Status fetch_windowed_prelude(const LogicalIndexReader& idx, const format::DictEntry& entry, + uint64_t frq_base, format::FrqPreludeReader* prelude); + +// Computes the absolute file ranges of window w's dd region (and freq region +// when want_freq, and .prx window when want_positions), fully validated against +// the POD sections (anti-DoS: rejects out-of-range offsets and overflowing +// locators). +Status windowed_window_range(const LogicalIndexReader& idx, const format::DictEntry& entry, + uint64_t frq_base, uint64_t prx_base, + const format::FrqPreludeReader& prelude, uint32_t w, + bool want_positions, bool want_freq, WindowAbsRange* out); + +// Decodes one window's docids (and per-doc positions when want_positions, and +// per-doc freqs when want_freq) from already-fetched byte slices: dd_region is +// the window's dd sub-slice; freq_region its freq sub-slice (ignored when +// !want_freq); prx_window its .prx bytes. The decoded docids are absolute +// (win_base applied). Returns Corruption on any doc-count mismatch between the +// prelude, dd/freq and prx. +Status decode_window_slices(const format::WindowMeta& meta, Slice dd_region, Slice freq_region, + Slice prx_window, bool want_positions, bool want_freq, + std::vector* docids, std::vector* freqs, + std::vector>* positions); + +} // namespace doris::snii::reader diff --git a/be/src/storage/index/snii/snii_doris_adapter.cpp b/be/src/storage/index/snii/snii_doris_adapter.cpp new file mode 100644 index 00000000000000..c593953f6a05de --- /dev/null +++ b/be/src/storage/index/snii/snii_doris_adapter.cpp @@ -0,0 +1,387 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/snii_doris_adapter.h" + +#include + +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "runtime/exec_env.h" +#include "storage/index/snii/common/uninitialized_buffer.h" +#include "util/countdown_latch.h" +#include "util/threadpool.h" + +namespace doris::segment_v2::snii_doris { +namespace { +// Per-call cap on concurrently dispatched physical segment reads. A coalesced +// batch of at most this many segments is served as a single concurrent round +// (mirrors ::doris::snii::io::MeteredFileReader's "at most one serial round" contract and +// the S3 standalone reader's 16-way fan-out). +constexpr size_t kMaxConcurrentReads = 16; +} // namespace + +thread_local const io::IOContext* DorisSniiFileReader::_scoped_io_ctx = nullptr; +ThreadPool* DorisSniiFileReader::_io_pool_for_test = nullptr; + +Status DorisSniiFileWriter::append(::doris::snii::Slice data) { + if (_writer == nullptr) { + return Status::Error("doris writer is null"); + } + return _writer->append(Slice(reinterpret_cast(data.data()), data.size())); +} + +Status DorisSniiFileWriter::finalize() { + if (_writer == nullptr) { + return Status::Error("doris writer is null"); + } + return Status::OK(); +} + +uint64_t DorisSniiFileWriter::bytes_written() const { + return _writer == nullptr ? 0 : _writer->bytes_appended(); +} + +DorisSniiFileReader::DorisSniiFileReader(io::FileReaderSPtr reader, const io::IOContext* io_ctx) + : _reader(std::move(reader)), _default_io_ctx(_make_index_io_context(io_ctx)) {} + +io::IOContext DorisSniiFileReader::_make_index_io_context(const io::IOContext* io_ctx) { + io::IOContext index_io_ctx; + if (io_ctx != nullptr) { + index_io_ctx = *io_ctx; + } + index_io_ctx.is_inverted_index = true; + // is_index_data is inherited from io_ctx: META scopes set it true at the source + // (index_file_reader), non-meta reads default to false. + return index_io_ctx; +} + +DorisSniiFileReader::ScopedIOContext::ScopedIOContext(const io::IOContext* io_ctx) + : _previous(_scoped_io_ctx), _io_ctx(DorisSniiFileReader::_make_index_io_context(io_ctx)) { + _scoped_io_ctx = &_io_ctx; +} + +DorisSniiFileReader::ScopedIOContext::~ScopedIOContext() { + _scoped_io_ctx = _previous; +} + +Status DorisSniiFileReader::read_at(uint64_t offset, size_t len, std::vector* out) { + RETURN_IF_ERROR(_check_read_range(offset, len)); + RETURN_IF_ERROR(_read_at(offset, len, out, _current_io_ctx())); + if (len > 0) { + _record_read_stats(cast_set(len), cast_set(len), 1, 1); + } + return Status::OK(); +} + +// NOLINTNEXTLINE(readability-non-const-parameter): out is the SNII read output buffer. +Status DorisSniiFileReader::_read_at(uint64_t offset, size_t len, std::vector* out, + const io::IOContext* io_ctx) const { + if (_reader == nullptr) { + return Status::Error("doris reader is null"); + } + if (out == nullptr) { + return Status::Error("output buffer is null"); + } + RETURN_IF_ERROR(_check_read_range(offset, len)); + if (len == 0) { + out->clear(); + return Status::OK(); + } + out->resize(len); + size_t bytes_read = 0; + auto status = _reader->read_at(offset, Slice(out->data(), len), &bytes_read, io_ctx); + if (!status.ok()) { + return status; + } + if (bytes_read != len) { + return Status::Error( + fmt::format("short read at offset {}, expect {}, got {}", offset, len, bytes_read)); + } + return Status::OK(); +} + +// NOLINTBEGIN(readability-non-const-parameter): outs is the SNII batch read output buffer. +Status DorisSniiFileReader::read_batch(const std::vector<::doris::snii::io::Range>& ranges, + std::vector>* outs) { + if (outs == nullptr) { + return Status::Error("output buffers is null"); + } + outs->clear(); + outs->resize(ranges.size()); + if (ranges.empty()) { + return Status::OK(); + } + + // ----- Phase 1: plan (serial, lock-free) ----- + // No section-classification lock exists on this reader, so the whole plan is + // a plain in-memory scan; the NO-IO-UNDER-LOCK red line holds trivially (no + // lock is taken anywhere in read_batch). + struct IndexedRange { + uint64_t offset = 0; + size_t len = 0; + size_t index = 0; + }; + int64_t request_bytes = 0; + std::vector sorted; + sorted.reserve(ranges.size()); + for (size_t i = 0; i < ranges.size(); ++i) { + RETURN_IF_ERROR(_check_read_range(ranges[i].offset, ranges[i].len)); + request_bytes += cast_set(ranges[i].len); + if (ranges[i].len == 0) { + continue; + } + sorted.push_back({ranges[i].offset, ranges[i].len, i}); + } + if (sorted.empty()) { + return Status::OK(); + } + // F27: callers (BatchRangeFetcher::fetch) already pass offset-sorted ranges; + // only pay for a sort when the input is actually out of order. + auto by_offset = [](const IndexedRange& lhs, const IndexedRange& rhs) { + return lhs.offset < rhs.offset; + }; + if (!std::ranges::is_sorted(sorted, by_offset)) { + std::ranges::sort(sorted, by_offset); + } + + // Coalesce the sorted ranges into disjoint physical segments. + struct Seg { + uint64_t offset = 0; + size_t len = 0; + size_t begin = 0; // first index into `sorted` covered by this segment + size_t end = 0; // one-past-last index into `sorted` + bool single = false; + }; + constexpr uint64_t max_coalesced_gap = 4096; + constexpr uint64_t max_coalesced_read = 1ULL << 20; + std::vector segs; + for (size_t begin = 0; begin < sorted.size();) { + uint64_t read_offset = sorted[begin].offset; + uint64_t read_end = sorted[begin].offset + sorted[begin].len; + size_t end = begin + 1; + while (end < sorted.size()) { + const uint64_t next_end = sorted[end].offset + sorted[end].len; + if ((sorted[end].offset > read_end && + sorted[end].offset - read_end > max_coalesced_gap) || + next_end - read_offset > max_coalesced_read) { + break; + } + read_end = std::max(read_end, next_end); + ++end; + } + Seg seg; + seg.offset = read_offset; + seg.len = cast_set(read_end - read_offset); + seg.begin = begin; + seg.end = end; + // A single-range group exactly covers its segment, so it can be read + // straight into the caller's output slot with no temp + no second copy. + seg.single = (end == begin + 1); + segs.push_back(seg); + begin = end; + } + + // Resolve per-segment target buffers, io contexts and the shared sink on the + // calling thread: workers (which run on tracker-less pool threads) must not + // allocate, and per-segment private cache-stat slots keep disjoint physical + // reads from racing on the shared FileCacheStatistics. + const size_t num_segs = segs.size(); + const io::IOContext* base_io_ctx = _current_io_ctx(); + std::vector> tmp_bufs(num_segs); + std::vector*> targets(num_segs); + std::vector seg_stats(num_segs); + std::vector seg_io_ctx(num_segs); + std::vector seg_status(num_segs); + int64_t read_bytes = 0; + for (size_t s = 0; s < num_segs; ++s) { + const Seg& seg = segs[s]; + std::vector* target = + seg.single ? &(*outs)[sorted[seg.begin].index] : &tmp_bufs[s]; + ::doris::snii::resize_uninitialized(*target, seg.len); + targets[s] = target; + seg_io_ctx[s] = *base_io_ctx; + seg_io_ctx[s].file_cache_stats = + base_io_ctx->file_cache_stats != nullptr ? &seg_stats[s] : nullptr; + read_bytes += cast_set(seg.len); + } + + // ----- Phase 2: physical reads (lock-free; concurrent when a pool exists) ----- + auto run_segment = [&](size_t s) { + seg_status[s] = _read_at(segs[s].offset, segs[s].len, targets[s], &seg_io_ctx[s]); + }; + ThreadPool* pool = _select_io_pool(); + if (pool != nullptr && num_segs > 1) { + for (size_t base = 0; base < num_segs; base += kMaxConcurrentReads) { + const size_t wave_end = std::min(base + kMaxConcurrentReads, num_segs); + ::doris::CountDownLatch latch(cast_set(wave_end - base)); + for (size_t s = base; s < wave_end; ++s) { + Status submit_st = pool->submit_func([&run_segment, &latch, s]() { + run_segment(s); + latch.count_down(); + }); + if (!submit_st.ok()) { + // Pool full/shut down: read this segment inline; never skip + // the count_down or the latch would not drain. + run_segment(s); + latch.count_down(); + } + } + latch.wait(); + } + } else { + // Serial fallback: no executor (e.g. tools without ExecEnv) or a single + // segment (avoids micro-batch scheduling overhead). + for (size_t s = 0; s < num_segs; ++s) { + run_segment(s); + } + } + + // ----- Phase 3: first-error, scatter, merge stats, account (serial) ----- + for (size_t s = 0; s < num_segs; ++s) { + RETURN_IF_ERROR(seg_status[s]); + } + for (size_t s = 0; s < num_segs; ++s) { + const Seg& seg = segs[s]; + if (seg.single) { + continue; // already read in place + } + const std::vector& bytes = tmp_bufs[s]; + for (size_t i = seg.begin; i < seg.end; ++i) { + const uint64_t pos = sorted[i].offset - seg.offset; + auto& out = (*outs)[sorted[i].index]; + out.assign(bytes.begin() + cast_set(pos), + bytes.begin() + cast_set(pos + sorted[i].len)); + } + } + if (base_io_ctx->file_cache_stats != nullptr) { + for (size_t s = 0; s < num_segs; ++s) { + _merge_file_cache_statistics(base_io_ctx->file_cache_stats, seg_stats[s]); + } + } + _record_read_stats(request_bytes, read_bytes, cast_set(num_segs), + cast_set(_compute_num_waves(num_segs))); + return Status::OK(); +} +// NOLINTEND(readability-non-const-parameter) + +uint64_t DorisSniiFileReader::size() const { + return _reader == nullptr ? 0 : _reader->size(); +} + +const io::IOContext* DorisSniiFileReader::_current_io_ctx() const { + return _scoped_io_ctx != nullptr ? _scoped_io_ctx : &_default_io_ctx; +} + +void DorisSniiFileReader::_record_read_stats(int64_t request_bytes, int64_t read_bytes, + int64_t range_read_count, + int64_t serial_read_rounds) const { + const auto* io_ctx = _current_io_ctx(); + if (io_ctx->file_cache_stats == nullptr) { + return; + } + auto* stats = io_ctx->file_cache_stats; + stats->inverted_index_request_bytes += request_bytes; + stats->inverted_index_read_bytes += read_bytes; + stats->inverted_index_range_read_count += range_read_count; + stats->inverted_index_serial_read_rounds += serial_read_rounds; +} + +void DorisSniiFileReader::set_io_thread_pool_for_test(ThreadPool* pool) { + _io_pool_for_test = pool; +} + +ThreadPool* DorisSniiFileReader::_select_io_pool() { + if (_io_pool_for_test != nullptr) { + return _io_pool_for_test; + } + if (ExecEnv::ready()) { + return ExecEnv::GetInstance()->buffered_reader_prefetch_thread_pool(); + } + return nullptr; +} + +size_t DorisSniiFileReader::_compute_num_waves(size_t seg_count) { + if (seg_count == 0) { + return 0; + } + return (seg_count + kMaxConcurrentReads - 1) / kMaxConcurrentReads; +} + +void DorisSniiFileReader::_merge_file_cache_statistics(io::FileCacheStatistics* dst, + const io::FileCacheStatistics& src) { + if (dst == nullptr) { + return; + } + dst->num_local_io_total += src.num_local_io_total; + dst->num_remote_io_total += src.num_remote_io_total; + dst->num_peer_io_total += src.num_peer_io_total; + dst->local_io_timer += src.local_io_timer; + dst->bytes_read_from_local += src.bytes_read_from_local; + dst->bytes_read_from_remote += src.bytes_read_from_remote; + dst->bytes_read_from_peer += src.bytes_read_from_peer; + dst->remote_io_timer += src.remote_io_timer; + dst->peer_io_timer += src.peer_io_timer; + dst->remote_wait_timer += src.remote_wait_timer; + dst->write_cache_io_timer += src.write_cache_io_timer; + dst->bytes_write_into_cache += src.bytes_write_into_cache; + dst->num_skip_cache_io_total += src.num_skip_cache_io_total; + dst->read_cache_file_directly_timer += src.read_cache_file_directly_timer; + dst->cache_get_or_set_timer += src.cache_get_or_set_timer; + dst->lock_wait_timer += src.lock_wait_timer; + dst->get_timer += src.get_timer; + dst->set_timer += src.set_timer; + dst->inverted_index_num_local_io_total += src.inverted_index_num_local_io_total; + dst->inverted_index_num_remote_io_total += src.inverted_index_num_remote_io_total; + dst->inverted_index_num_peer_io_total += src.inverted_index_num_peer_io_total; + dst->inverted_index_bytes_read_from_local += src.inverted_index_bytes_read_from_local; + dst->inverted_index_bytes_read_from_remote += src.inverted_index_bytes_read_from_remote; + dst->inverted_index_bytes_read_from_peer += src.inverted_index_bytes_read_from_peer; + dst->inverted_index_remote_physical_read_bytes += src.inverted_index_remote_physical_read_bytes; + dst->inverted_index_bytes_write_into_cache += src.inverted_index_bytes_write_into_cache; + dst->inverted_index_local_io_timer += src.inverted_index_local_io_timer; + dst->inverted_index_remote_io_timer += src.inverted_index_remote_io_timer; + dst->inverted_index_peer_io_timer += src.inverted_index_peer_io_timer; + dst->inverted_index_io_timer += src.inverted_index_io_timer; + dst->inverted_index_request_bytes += src.inverted_index_request_bytes; + dst->inverted_index_read_bytes += src.inverted_index_read_bytes; + dst->inverted_index_range_read_count += src.inverted_index_range_read_count; + dst->inverted_index_serial_read_rounds += src.inverted_index_serial_read_rounds; +} + +Status DorisSniiFileReader::_check_read_range(uint64_t offset, size_t len) const { + if (_reader == nullptr) { + return Status::Error("doris reader is null"); + } + if (offset > std::numeric_limits::max() - len) { + return Status::Error( + fmt::format("read range overflows: offset {}, len {}", offset, len)); + } + const uint64_t end = offset + len; + if (end > _reader->size()) { + return Status::Error( + fmt::format("read range exceeds file size: offset {}, len {}, file size {}", offset, + len, _reader->size())); + } + return Status::OK(); +} + +} // namespace doris::segment_v2::snii_doris diff --git a/be/src/storage/index/snii/snii_doris_adapter.h b/be/src/storage/index/snii/snii_doris_adapter.h new file mode 100644 index 00000000000000..a783a3a0a07c14 --- /dev/null +++ b/be/src/storage/index/snii/snii_doris_adapter.h @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "io/fs/file_reader.h" +#include "io/fs/file_writer.h" +#include "io/io_common.h" +#include "storage/index/snii/io/file_reader.h" +#include "storage/index/snii/io/file_writer.h" +#include "util/slice.h" + +namespace doris { +class ThreadPool; +} // namespace doris + +namespace doris::segment_v2::snii_doris { + +class DorisSniiFileWriter final : public ::doris::snii::io::FileWriter { +public: + explicit DorisSniiFileWriter(io::FileWriter* writer) : _writer(writer) {} + + Status append(::doris::snii::Slice data) override; + Status finalize() override; + uint64_t bytes_written() const override; + +private: + io::FileWriter* _writer = nullptr; +}; + +class DorisSniiFileReader final : public ::doris::snii::io::FileReader { +public: + class ScopedIOContext { + public: + explicit ScopedIOContext(const io::IOContext* io_ctx); + ~ScopedIOContext(); + + ScopedIOContext(const ScopedIOContext&) = delete; + ScopedIOContext& operator=(const ScopedIOContext&) = delete; + + private: + const io::IOContext* _previous = nullptr; + io::IOContext _io_ctx; + }; + + explicit DorisSniiFileReader(io::FileReaderSPtr reader, const io::IOContext* io_ctx = nullptr); + + Status read_at(uint64_t offset, size_t len, std::vector* out) override; + Status read_batch(const std::vector<::doris::snii::io::Range>& ranges, + std::vector>* outs) override; + uint64_t size() const override; + + // Test-only: inject (or clear with nullptr) the thread pool used to fan out + // batch segment reads. nullptr (default) routes to the BE buffered-reader + // prefetch pool when ExecEnv is ready, otherwise a serial fallback. + static void set_io_thread_pool_for_test(ThreadPool* pool); + +private: + static io::IOContext _make_index_io_context(const io::IOContext* io_ctx); + Status _check_read_range(uint64_t offset, size_t len) const; + Status _read_at(uint64_t offset, size_t len, std::vector* out, + const io::IOContext* io_ctx) const; + const io::IOContext* _current_io_ctx() const; + void _record_read_stats(int64_t request_bytes, int64_t read_bytes, int64_t range_read_count, + int64_t serial_read_rounds) const; + + // Selects the executor for parallel batch segment reads: the test seam if + // set, else the BE prefetch pool when ExecEnv is ready, else nullptr (the + // caller then reads segments serially). + static ThreadPool* _select_io_pool(); + // Number of dependent serial dispatch waves for `seg_count` physical + // segments given the per-call concurrency cap. This is the F19 metric: + // a coalesced batch of <= cap segments is a single concurrent round. + static size_t _compute_num_waves(size_t seg_count); + // Sums every counter of `src` into `dst`. Per-segment reads accumulate file + // cache statistics into private slots (so disjoint physical reads never race + // on the shared sink); this folds them back. MUST stay in sync with + // io::FileCacheStatistics (be/src/io/io_common.h) when fields are added. + static void _merge_file_cache_statistics(io::FileCacheStatistics* dst, + const io::FileCacheStatistics& src); + + io::FileReaderSPtr _reader; + io::IOContext _default_io_ctx; + static thread_local const io::IOContext* _scoped_io_ctx; + // Test seam for the batch-read executor; see set_io_thread_pool_for_test. + static ThreadPool* _io_pool_for_test; +}; + +} // namespace doris::segment_v2::snii_doris diff --git a/be/src/storage/index/snii/snii_index_reader.cpp b/be/src/storage/index/snii/snii_index_reader.cpp new file mode 100644 index 00000000000000..85be88f1dd97ab --- /dev/null +++ b/be/src/storage/index/snii/snii_index_reader.cpp @@ -0,0 +1,670 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/snii_index_reader.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" +#include "storage/index/index_file_reader.h" +#include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/index/inverted/inverted_index_cache.h" +#include "storage/index/inverted/inverted_index_iterator.h" +#include "storage/index/snii/common/single_flight.h" +#include "storage/index/snii/format/null_bitmap.h" +#include "storage/index/snii/query/boolean_query.h" +#include "storage/index/snii/query/count_query.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/regexp_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/query/wildcard_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/snii_doris_adapter.h" +#include "util/defer_op.h" +#include "util/time.h" + +namespace doris::segment_v2 { + +namespace { + +class RoaringDocIdSink final : public ::doris::snii::query::DocIdSink { +public: + explicit RoaringDocIdSink(roaring::Roaring* bitmap) : _bitmap(bitmap) { + DCHECK(_bitmap != nullptr); + } + + Status append_sorted(std::span docids) override { + if (!docids.empty()) { + _bitmap->addMany(docids.size(), docids.data()); + } + return Status::OK(); + } + + Status append_range(uint32_t first, uint64_t last_exclusive) override { + if (last_exclusive > first) { + _bitmap->addRange(first, last_exclusive); + } + return Status::OK(); + } + + // Roaring addMany/addRange deduplicate and order natively, so multi-term OR + // can stream each posting straight into the bitmap (no per-term vector + merge). + bool dedups() const override { return true; } + +private: + roaring::Roaring* _bitmap; +}; + +struct SniiQueryExecutionResult { + std::shared_ptr bitmap; +}; + +std::vector to_terms(const InvertedIndexQueryInfo& query_info) { + std::vector terms; + terms.reserve(query_info.term_infos.size()); + for (const auto& term_info : query_info.term_infos) { + DCHECK(term_info.is_single_term()); + terms.push_back(term_info.get_single_term()); + } + return terms; +} + +void parse_phrase_slop(std::string* query, InvertedIndexQueryInfo* query_info) { + DCHECK(query != nullptr); + DCHECK(query_info != nullptr); + const auto is_digits = [](std::string_view str) { + return std::all_of(str.begin(), str.end(), [](unsigned char c) { return std::isdigit(c); }); + }; + + const size_t last_space_pos = query->find_last_of(' '); + if (last_space_pos == std::string::npos) { + return; + } + const size_t tilde_pos = last_space_pos + 1; + if (tilde_pos >= query->size() - 1 || (*query)[tilde_pos] != '~') { + return; + } + + const size_t slop_pos = tilde_pos + 1; + std::string_view slop_str(query->data() + slop_pos, query->size() - slop_pos); + if (slop_str.empty()) { + return; + } + + bool ordered = false; + if (slop_str.size() == 1) { + if (!std::isdigit(static_cast(slop_str[0]))) { + return; + } + } else if (slop_str.back() == '+') { + ordered = true; + slop_str.remove_suffix(1); + } + + if (!is_digits(slop_str)) { + return; + } + auto result = std::from_chars(slop_str.begin(), slop_str.end(), query_info->slop); + if (result.ec != std::errc()) { + return; + } + query_info->ordered = ordered; + *query = query->substr(0, last_space_pos); +} + +std::string build_snii_query_cache_value(const InvertedIndexQueryInfo& query_info) { + std::string cache_value; + for (const auto& term_info : query_info.term_infos) { + DCHECK(term_info.is_single_term()); + const auto& term = term_info.get_single_term(); + cache_value.append(std::to_string(term.size())); + cache_value.push_back(':'); + cache_value.append(term); + cache_value.push_back('@'); + cache_value.append(std::to_string(term_info.position)); + cache_value.push_back(';'); + } + return cache_value; +} + +std::shared_ptr docids_to_bitmap(const std::vector& docids) { + auto result = std::make_shared(); + if (!docids.empty()) { + result->addMany(docids.size(), docids.data()); + } + result->runOptimize(); + return result; +} + +// Runs `compute` under single-flight keyed by `key`: concurrent identical queries collapse to a +// single execution and the followers reuse the leader's bitmap. `compute(out)` fills *out and +// returns its Status; on overall success *result receives the bitmap. See SingleFlight for why +// this matters under a cold cache with parallel scanners hitting the same segment. +template +Status run_query_single_flight( + ::doris::snii::SingleFlight>>& flight, + const std::string& key, std::shared_ptr* result, Compute&& compute) { + auto follower = flight.join_or_lead(key); + if (follower.has_value()) { + auto [leader_status, leader_bitmap] = follower->get(); + if (leader_status.ok() && leader_bitmap != nullptr) { + *result = std::move(leader_bitmap); + return Status::OK(); + } + // Leader failed; fall through and compute independently (rare error path). + } + const bool is_leader = !follower.has_value(); + + Status status = Status::OK(); + std::shared_ptr bitmap; + { + // Publish to any waiting followers on every exit path (including errors). + DEFER(if (is_leader) { flight.publish(key, std::make_pair(status, bitmap)); }); + status = compute(&bitmap); + } + RETURN_IF_ERROR(status); + *result = std::move(bitmap); + return Status::OK(); +} + +Status execute_snii_query(const ::doris::snii::reader::LogicalIndexReader& logical_reader, + InvertedIndexQueryType query_type, + const InvertedIndexQueryInfo& query_info, std::string_view search_str, + const std::vector& terms, int32_t max_expansions, + SniiQueryExecutionResult* result) { + result->bitmap = std::make_shared(); + RoaringDocIdSink sink(result->bitmap.get()); + std::vector docids; + bool emitted_to_sink = false; + Status status; + switch (query_type) { + case InvertedIndexQueryType::EQUAL_QUERY: + case InvertedIndexQueryType::MATCH_ANY_QUERY: + status = terms.size() == 1 + ? ::doris::snii::query::term_query(logical_reader, terms.front(), &sink) + : ::doris::snii::query::boolean_or(logical_reader, terms, &sink); + emitted_to_sink = true; + break; + case InvertedIndexQueryType::MATCH_ALL_QUERY: + if (terms.size() == 1) { + status = ::doris::snii::query::term_query(logical_reader, terms.front(), &sink); + emitted_to_sink = true; + } else { + status = ::doris::snii::query::boolean_and(logical_reader, terms, &docids); + } + break; + case InvertedIndexQueryType::MATCH_PHRASE_QUERY: + if (query_info.slop != 0) { + return Status::Error( + "SNII does not support sloppy phrase query yet"); + } + if (terms.size() == 1) { + status = ::doris::snii::query::term_query(logical_reader, terms.front(), &sink); + emitted_to_sink = true; + } else { + status = ::doris::snii::query::phrase_query(logical_reader, terms, &docids); + } + break; + case InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY: + if (terms.size() == 1) { + status = ::doris::snii::query::prefix_query(logical_reader, terms.front(), &sink, + max_expansions); + emitted_to_sink = true; + } else { + status = ::doris::snii::query::phrase_prefix_query(logical_reader, terms, &docids, + max_expansions); + } + break; + case InvertedIndexQueryType::MATCH_REGEXP_QUERY: + status = ::doris::snii::query::regexp_query(logical_reader, search_str, &sink, + max_expansions); + emitted_to_sink = true; + break; + case InvertedIndexQueryType::WILDCARD_QUERY: + status = ::doris::snii::query::wildcard_query(logical_reader, search_str, &sink, + max_expansions); + emitted_to_sink = true; + break; + case InvertedIndexQueryType::LESS_THAN_QUERY: + case InvertedIndexQueryType::LESS_EQUAL_QUERY: + case InvertedIndexQueryType::GREATER_THAN_QUERY: + case InvertedIndexQueryType::GREATER_EQUAL_QUERY: + case InvertedIndexQueryType::RANGE_QUERY: + return Status::Error( + "SNII inverted index storage format does not support BKD/range query"); + default: + return Status::Error( + "SNII unsupported inverted index query type {}", query_type_to_string(query_type)); + } + RETURN_IF_ERROR(status); + if (emitted_to_sink) { + result->bitmap->runOptimize(); + } else { + result->bitmap = docids_to_bitmap(docids); + } + return Status::OK(); +} + +} // namespace + +Status SniiIndexReader::new_iterator(std::unique_ptr* iterator) { + if (*iterator == nullptr) { + *iterator = InvertedIndexIterator::create_unique(); + } + dynamic_cast(iterator->get()) + ->add_reader(_reader_type, + dynamic_pointer_cast(shared_from_this())); + return Status::OK(); +} + +Status SniiIndexReader::_parse_query_terms(const IndexQueryContextPtr& context, + std::string search_str, + InvertedIndexQueryType query_type, + const InvertedIndexAnalyzerCtx* analyzer_ctx, + InvertedIndexQueryInfo* query_info) { + DCHECK(query_info != nullptr); + if (query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY || + query_type == InvertedIndexQueryType::WILDCARD_QUERY) { + query_info->term_infos.emplace_back(search_str, 0); + return Status::OK(); + } + if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY || + query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY) { + // parse_phrase_slop MUST run first: it strips a trailing ~N slop suffix off + // search_str and records it in query_info before tokenization. + parse_phrase_slop(&search_str, query_info); + SCOPED_RAW_TIMER(&context->stats->inverted_index_analyzer_timer); + try { + // Mirror the non-phrase branch below: reuse the per-query-expr + // analyzer_ctx (built once and shared across segments) instead of + // rebuilding a CLucene analyzer per segment from properties. Falls back + // to the properties path when analyzer_ctx is absent (internal callers). + if (analyzer_ctx != nullptr && !analyzer_ctx->should_tokenize()) { + query_info->term_infos.emplace_back(search_str); + } else if (analyzer_ctx != nullptr && analyzer_ctx->analyzer != nullptr) { + auto reader = inverted_index::InvertedIndexAnalyzer::create_reader( + analyzer_ctx->char_filter_map); + reader->init(search_str.data(), static_cast(search_str.size()), true); + query_info->term_infos = inverted_index::InvertedIndexAnalyzer::get_analyse_result( + reader, analyzer_ctx->analyzer.get()); + } else { + query_info->term_infos = inverted_index::InvertedIndexAnalyzer::get_analyse_result( + search_str, _index_meta.properties()); + } + } catch (const CLuceneError& e) { + return Status::Error( + "SNII analyze query failed: {}", e.what()); + } catch (const Exception& e) { + return Status::Error( + "SNII analyze query failed: {}", e.what()); + } + return Status::OK(); + } + + SCOPED_RAW_TIMER(&context->stats->inverted_index_analyzer_timer); + try { + if (analyzer_ctx != nullptr && !analyzer_ctx->should_tokenize()) { + query_info->term_infos.emplace_back(search_str); + } else if (analyzer_ctx != nullptr && analyzer_ctx->analyzer != nullptr) { + auto reader = inverted_index::InvertedIndexAnalyzer::create_reader( + analyzer_ctx->char_filter_map); + reader->init(search_str.data(), static_cast(search_str.size()), true); + query_info->term_infos = inverted_index::InvertedIndexAnalyzer::get_analyse_result( + reader, analyzer_ctx->analyzer.get()); + } else { + query_info->term_infos = inverted_index::InvertedIndexAnalyzer::get_analyse_result( + search_str, _index_meta.properties()); + } + } catch (const CLuceneError& e) { + return Status::Error( + "SNII analyze query failed: {}", e.what()); + } catch (const Exception& e) { + return Status::Error( + "SNII analyze query failed: {}", e.what()); + } + return Status::OK(); +} + +Status SniiIndexReader::_get_logical_reader( + const IndexQueryContextPtr& context, InvertedIndexCacheHandle* searcher_cache_handle, + std::unique_ptr<::doris::snii::reader::LogicalIndexReader>* uncached_reader, + const ::doris::snii::reader::LogicalIndexReader** logical_reader) { + DCHECK(searcher_cache_handle != nullptr); + DCHECK(uncached_reader != nullptr); + DCHECK(logical_reader != nullptr); + + const bool enable_searcher_cache = + context->runtime_state != nullptr && + context->runtime_state->query_options().enable_inverted_index_searcher_cache; + const auto index_file_key = _index_file_reader->get_index_file_cache_key(&_index_meta); + InvertedIndexSearcherCache::CacheKey searcher_cache_key(index_file_key); + + bool cache_hit = false; + if (enable_searcher_cache) { + SCOPED_RAW_TIMER(&context->stats->inverted_index_lookup_timer); + cache_hit = InvertedIndexSearcherCache::instance()->lookup(searcher_cache_key, + searcher_cache_handle); + } + + if (cache_hit) { + context->stats->inverted_index_searcher_cache_hit++; + *logical_reader = searcher_cache_handle->get_snii_logical_reader(); + if (*logical_reader == nullptr) { + return Status::InternalError("SNII searcher cache entry has no logical reader"); + } + return Status::OK(); + } + + SCOPED_RAW_TIMER(&context->stats->inverted_index_searcher_open_timer); + context->stats->inverted_index_searcher_cache_miss++; + RETURN_IF_ERROR( + _index_file_reader->init(config::inverted_index_read_buffer_size, context->io_ctx)); + auto opened_reader = + DORIS_TRY(_index_file_reader->open_snii_index(&_index_meta, context->io_ctx)); + + if (!enable_searcher_cache) { + *logical_reader = opened_reader.get(); + *uncached_reader = std::move(opened_reader); + return Status::OK(); + } + + const size_t reader_size = std::max(opened_reader->memory_usage(), 1); + auto* cache_value = new InvertedIndexSearcherCache::CacheValue( + std::move(opened_reader), reader_size, UnixMillis(), _index_file_reader); + InvertedIndexSearcherCache::instance()->insert(searcher_cache_key, cache_value, + searcher_cache_handle); + *logical_reader = searcher_cache_handle->get_snii_logical_reader(); + if (*logical_reader == nullptr) { + return Status::InternalError("SNII searcher cache insert produced empty logical reader"); + } + return Status::OK(); +} + +Status SniiIndexReader::query(const IndexQueryContextPtr& context, const std::string& column_name, + const Field& query_value, InvertedIndexQueryType query_type, + std::shared_ptr& bit_map, + const InvertedIndexAnalyzerCtx* analyzer_ctx) { + SCOPED_RAW_TIMER(&context->stats->inverted_index_query_timer); + std::string search_str = query_value.get(); + + if (int ignore_above = + std::stoi(get_parser_ignore_above_value_from_properties(_index_meta.properties())); + _reader_type == InvertedIndexReaderType::STRING_TYPE && search_str.size() > ignore_above) { + return Status::Error( + "query value is too long, evaluate skipped."); + } + + InvertedIndexQueryInfo query_info; + RETURN_IF_ERROR(_parse_query_terms(context, search_str, query_type, analyzer_ctx, &query_info)); + if (query_info.term_infos.empty()) { + auto msg = fmt::format("token parser result is empty for SNII query '{}'", search_str); + if (is_match_query(query_type)) { + LOG(WARNING) << msg; + bit_map = std::make_shared(); + return Status::OK(); + } + return Status::Error(msg); + } + + auto terms = to_terms(query_info); + const int32_t max_expansions = + context->runtime_state == nullptr + ? 50 + : context->runtime_state->query_options().inverted_index_max_expansions; + std::string cache_value = build_snii_query_cache_value(query_info); + if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) { + cache_value += " " + std::to_string(query_info.slop); + cache_value += " " + std::to_string(query_info.ordered); + } else if (query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY || + query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY || + query_type == InvertedIndexQueryType::WILDCARD_QUERY) { + cache_value += " " + std::to_string(max_expansions); + } + auto index_file_key = _index_file_reader->get_index_file_cache_key(&_index_meta); + // Single-flight key: identifies this exact (segment file, column, query type, terms) so + // concurrent identical queries share one execution. Built before cache_value is moved + // into cache_key below; mirrors the query-cache key components. + std::string single_flight_key = index_file_key; + single_flight_key.push_back('\x01'); + single_flight_key.append(column_name); + single_flight_key.push_back('\x01'); + single_flight_key.append(std::to_string(static_cast(query_type))); + single_flight_key.push_back('\x01'); + single_flight_key.append(cache_value); + + InvertedIndexQueryCache::CacheKey cache_key {index_file_key, column_name, query_type, + std::move(cache_value)}; + auto* cache = InvertedIndexQueryCache::instance(); + InvertedIndexQueryCacheHandle cache_handler; + if (handle_query_cache(context, cache, cache_key, &cache_handler, bit_map)) { + return Status::OK(); + } + + // G02 count-only fast path: the SegmentIterator asserted (via the context + // flag) that only the match COUNT of this predicate matters, so eligible + // shapes are answered from dict-entry df without decoding postings. Placed + // AFTER the query-cache lookup (a cached row-accurate bitmap is free and + // counts correctly) and BEFORE single-flight; the fabricated [0, df) bitmap + // is returned early and NEVER inserted into the query cache or published to + // single-flight followers -- both are keyed identically to row-accurate + // queries and must only ever serve real row ids. + if (context->count_on_index_fastpath) { + bool count_handled = false; + std::shared_ptr count_bitmap; + RETURN_IF_ERROR(_try_count_only_fastpath(context, query_type, query_info, terms, + &count_handled, &count_bitmap)); + if (count_handled) { + bit_map = std::move(count_bitmap); + // G03 reply: tell the SegmentIterator the bitmap is count-shaped + // (cardinality exact, row ids fabricated) so it may short-circuit + // row emission. Deliberately NOT set on the cache-hit return above + // or on the decode path below -- those bitmaps are row-accurate + // and keep today's emission. + context->count_on_index_fastpath_hit = true; + return Status::OK(); + } + } + + // Under a cold cache, parallel scanners _lazy_init the same segment concurrently and each + // would otherwise miss the searcher/query caches and redundantly open + decode this segment's + // index. Collapse identical concurrent queries into one shared execution (see SingleFlight). + static ::doris::snii::SingleFlight>> + query_single_flight; + std::shared_ptr result_bitmap; + RETURN_IF_ERROR(run_query_single_flight(query_single_flight, single_flight_key, &result_bitmap, + [&](std::shared_ptr* out) { + return _compute_query_bitmap( + context, query_type, query_info, search_str, + terms, max_expansions, out); + })); + bit_map = result_bitmap; + cache->insert(cache_key, bit_map, &cache_handler); + return Status::OK(); +} + +Status SniiIndexReader::_compute_query_bitmap(const IndexQueryContextPtr& context, + InvertedIndexQueryType query_type, + const InvertedIndexQueryInfo& query_info, + std::string_view search_str, + const std::vector& terms, + int32_t max_expansions, + std::shared_ptr* out) { + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(context->io_ctx); + InvertedIndexCacheHandle searcher_cache_handle; + std::unique_ptr<::doris::snii::reader::LogicalIndexReader> uncached_reader; + const ::doris::snii::reader::LogicalIndexReader* logical_reader = nullptr; + RETURN_IF_ERROR(_get_logical_reader(context, &searcher_cache_handle, &uncached_reader, + &logical_reader)); + SniiQueryExecutionResult query_result; + RETURN_IF_ERROR(execute_snii_query(*logical_reader, query_type, query_info, search_str, terms, + max_expansions, &query_result)); + *out = std::move(query_result.bitmap); + return Status::OK(); +} + +Status SniiIndexReader::_try_count_only_fastpath(const IndexQueryContextPtr& context, + InvertedIndexQueryType query_type, + const InvertedIndexQueryInfo& query_info, + const std::vector& terms, + bool* handled, + std::shared_ptr* out) { + *handled = false; + // Shape guard: only exact-term query types. Prefix/regexp/wildcard/ + // phrase-prefix expand the term set, so no single dict entry carries the + // count; range types never reach SNII anyway. + switch (query_type) { + case InvertedIndexQueryType::EQUAL_QUERY: + case InvertedIndexQueryType::MATCH_ANY_QUERY: + case InvertedIndexQueryType::MATCH_ALL_QUERY: + case InvertedIndexQueryType::MATCH_PHRASE_QUERY: + break; + default: + return Status::OK(); + } + // The normal path rejects sloppy phrases with a BYPASS (downgrade to raw + // evaluation); the count path must fall through so the downgrade happens. + if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY && query_info.slop != 0) { + return Status::OK(); + } + const bool single_term = terms.size() == 1; + const bool two_term_phrase = + terms.size() == 2 && query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY; + if (!single_term && !two_term_phrase) { + // Multi-term MATCH_ANY (OR) / MATCH_ALL (AND) counts are not derivable + // from per-term dfs (overlap unknown) -> normal decode path. + return Status::OK(); + } + + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(context->io_ctx); + InvertedIndexCacheHandle searcher_cache_handle; + std::unique_ptr<::doris::snii::reader::LogicalIndexReader> uncached_reader; + const ::doris::snii::reader::LogicalIndexReader* logical_reader = nullptr; + RETURN_IF_ERROR(_get_logical_reader(context, &searcher_cache_handle, &uncached_reader, + &logical_reader)); + + uint64_t count = 0; + if (single_term) { + RETURN_IF_ERROR( + ::doris::snii::query::count_only_term_df(*logical_reader, terms.front(), &count)); + } else { + bool bigram_handled = false; + RETURN_IF_ERROR(::doris::snii::query::count_only_two_term_phrase_bigram_df( + *logical_reader, terms[0], terms[1], &bigram_handled, &count)); + if (!bigram_handled) { + // Pruned or absent bigram (or positionless index): the generic + // phrase path owns those semantics. + return Status::OK(); + } + } + + // Null handling. df is the exact match count REGARDLESS of nulls: the + // writer adds no tokens for a null doc (scalar add_nulls; a NULL array row + // is an empty range), so postings -- and therefore df -- never include + // null rows, exactly matching MATCH's "null never matches" semantics. The + // fabricated bitmap however flows through FunctionMatchBase -> + // InvertedIndexResultBitmap::mask_out_null, which subtracts the segment's + // REAL null bitmap from it; a dense [0, df) range colliding with null row + // ids would be shrunk below df. So on a segment WITH a null bitmap, load + // it (query-cache backed, the same read the normal MATCH path performs) + // and fabricate df ids DISJOINT from it, making that subtraction a + // provable no-op. Segments without a null section (the writer omits it + // when no row is null) keep the trivial [0, df) range. + auto result = std::make_shared(); + if (count > 0 && logical_reader->section_refs().null_bitmap.length > 0) { + InvertedIndexQueryCacheHandle null_bitmap_cache_handle; + RETURN_IF_ERROR(read_null_bitmap(context, &null_bitmap_cache_handle)); + std::shared_ptr nulls = null_bitmap_cache_handle.get_bitmap(); + // Fall through on a missing bitmap behind the cache handle or a + // fabrication failure (df + null count breaching the docid domain): + // both mean a corrupt index, and the row-accurate decode -- which + // intersects real ids -- must own the answer rather than a blind + // fabrication. The count_fastpath_hits test seam already counted the + // dict lookup above; production correctness is unaffected. + if (nulls == nullptr) { + return Status::OK(); + } + if (!nulls->isEmpty()) { + if (!::doris::snii::query::fabricate_null_disjoint_count_bitmap(count, *nulls, + result.get()) + .ok()) { + return Status::OK(); + } + } else { + result->addRange(0, count); + } + } else if (count > 0) { + result->addRange(0, count); + } + *out = std::move(result); + *handled = true; + return Status::OK(); +} + +Status SniiIndexReader::try_query(const IndexQueryContextPtr& /*context*/, + const std::string& /*column_name*/, const Field& /*query_value*/, + InvertedIndexQueryType /*query_type*/, size_t* /*count*/) { + return Status::Error("SNII does not support try_query"); +} + +Status SniiIndexReader::read_null_bitmap(const IndexQueryContextPtr& context, + InvertedIndexQueryCacheHandle* cache_handle, + lucene::store::Directory* /*dir*/) { + SCOPED_RAW_TIMER(&context->stats->inverted_index_query_null_bitmap_timer); + auto index_file_key = _index_file_reader->get_index_file_cache_key(&_index_meta); + InvertedIndexQueryCache::CacheKey cache_key { + index_file_key, "", InvertedIndexQueryType::UNKNOWN_QUERY, "null_bitmap"}; + auto* cache = InvertedIndexQueryCache::instance(); + if (cache->lookup(cache_key, cache_handle)) { + return Status::OK(); + } + + snii_doris::DorisSniiFileReader::ScopedIOContext io_context_scope(context->io_ctx); + InvertedIndexCacheHandle searcher_cache_handle; + std::unique_ptr<::doris::snii::reader::LogicalIndexReader> uncached_reader; + const ::doris::snii::reader::LogicalIndexReader* logical_reader = nullptr; + RETURN_IF_ERROR(_get_logical_reader(context, &searcher_cache_handle, &uncached_reader, + &logical_reader)); + auto null_bitmap = std::make_shared(); + const auto& ref = logical_reader->section_refs().null_bitmap; + if (ref.length > 0) { + std::vector bytes; + RETURN_IF_ERROR(logical_reader->reader()->read_at(ref.offset, ref.length, &bytes)); + ::doris::snii::format::NullBitmapReader reader; + RETURN_IF_ERROR(::doris::snii::format::NullBitmapReader::open(::doris::snii::Slice(bytes), + &reader)); + reader.copy_to(null_bitmap.get()); + null_bitmap->runOptimize(); + } + cache->insert(cache_key, null_bitmap, cache_handle); + return Status::OK(); +} + +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/snii/snii_index_reader.h b/be/src/storage/index/snii/snii_index_reader.h new file mode 100644 index 00000000000000..577ff6304dec6a --- /dev/null +++ b/be/src/storage/index/snii/snii_index_reader.h @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "storage/index/inverted/inverted_index_query_type.h" +#include "storage/index/inverted/inverted_index_reader.h" + +namespace doris::snii::reader { +class LogicalIndexReader; +} // namespace doris::snii::reader + +namespace doris::segment_v2 { + +class SniiIndexReader final : public InvertedIndexReader { + ENABLE_FACTORY_CREATOR(SniiIndexReader); + +public: + SniiIndexReader(const TabletIndex* index_meta, + const std::shared_ptr& index_file_reader, + InvertedIndexReaderType reader_type) + : InvertedIndexReader(index_meta, index_file_reader), _reader_type(reader_type) {} + + Status new_iterator(std::unique_ptr* iterator) override; + Status query(const IndexQueryContextPtr& context, const std::string& column_name, + const Field& query_value, InvertedIndexQueryType query_type, + std::shared_ptr& bit_map, + const InvertedIndexAnalyzerCtx* analyzer_ctx = nullptr) override; + Status try_query(const IndexQueryContextPtr& context, const std::string& column_name, + const Field& query_value, InvertedIndexQueryType query_type, + size_t* count) override; + Status read_null_bitmap(const IndexQueryContextPtr& context, + InvertedIndexQueryCacheHandle* cache_handle, + lucene::store::Directory* dir = nullptr) override; + InvertedIndexReaderType type() override { return _reader_type; } + +private: + Status _parse_query_terms(const IndexQueryContextPtr& context, std::string search_str, + InvertedIndexQueryType query_type, + const InvertedIndexAnalyzerCtx* analyzer_ctx, + InvertedIndexQueryInfo* query_info); + Status _get_logical_reader( + const IndexQueryContextPtr& context, InvertedIndexCacheHandle* searcher_cache_handle, + std::unique_ptr<::doris::snii::reader::LogicalIndexReader>* uncached_reader, + const ::doris::snii::reader::LogicalIndexReader** logical_reader); + // Opens the segment index and runs the query, producing the result bitmap. Invoked as the + // single-flight "compute" step by query(); see SingleFlight for the concurrency rationale. + Status _compute_query_bitmap(const IndexQueryContextPtr& context, + InvertedIndexQueryType query_type, + const InvertedIndexQueryInfo& query_info, + std::string_view search_str, const std::vector& terms, + int32_t max_expansions, std::shared_ptr* out); + // G02 count-only fast path. Only called when the caller (SegmentIterator) + // set context->count_on_index_fastpath, i.e. the match count alone decides + // the scan result. On *handled = true, *out is a bitmap of cardinality df + // (row ids NOT real) built from dict entries WITHOUT decoding postings: + // single exact term -> dict-entry df; 2-term MATCH_PHRASE with a surviving + // (dict-HIT) G01 bigram -> bigram df. On a segment without a null bitmap + // the fabricated ids are the dense range [0, df); on a segment WITH one + // they are the first df NON-NULL row ids (see + // fabricate_null_disjoint_count_bitmap) so that the unconditional + // FunctionMatchBase -> mask_out_null subtraction of the real null bitmap + // is a no-op and the cardinality stays df -- which is already the exact + // match count, because postings never contain null docs. Falls through + // (*handled = false) for every other shape: multi-term OR/AND, + // prefix/regexp/wildcard expansion, sloppy phrase, pruned/absent bigram. + // On *handled = true, query() also raises + // context->count_on_index_fastpath_hit (G03) so the SegmentIterator may + // short-circuit row emission for the count-shaped bitmap. + Status _try_count_only_fastpath(const IndexQueryContextPtr& context, + InvertedIndexQueryType query_type, + const InvertedIndexQueryInfo& query_info, + const std::vector& terms, bool* handled, + std::shared_ptr* out); + + InvertedIndexReaderType _reader_type; +}; + +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/snii/snii_index_writer.cpp b/be/src/storage/index/snii/snii_index_writer.cpp new file mode 100644 index 00000000000000..da4747490fba3c --- /dev/null +++ b/be/src/storage/index/snii/snii_index_writer.cpp @@ -0,0 +1,397 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/snii_index_writer.h" + +#include + +#include +#include + +#include "common/cast_set.h" +#include "common/config.h" +#include "common/logging.h" +#include "storage/index/index_file_writer.h" +#include "storage/index/inverted/analyzer/analyzer.h" +#include "storage/index/inverted/query/query_info.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/writer/global_memory_limiter.h" +#include "storage/tablet/tablet_schema.h" + +namespace doris::segment_v2 { + +SniiIndexColumnWriter::SniiIndexColumnWriter(IndexFileWriter* index_file_writer, + const TabletIndex* index_meta, bool single_field) + : _index_file_writer(index_file_writer), + _index_meta(index_meta), + _single_field(single_field) {} + +Status SniiIndexColumnWriter::init() { + _should_analyzer = + inverted_index::InvertedIndexAnalyzer::should_analyzer(_index_meta->properties()); + _has_positions = get_parser_phrase_support_string_from_properties(_index_meta->properties()) == + INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES; + _config = _has_positions ? ::doris::snii::format::IndexConfig::kDocsPositions + : ::doris::snii::format::IndexConfig::kDocsOnly; + auto ignore_above_value = + get_parser_ignore_above_value_from_properties(_index_meta->properties()); + _ignore_above = cast_set(std::stoul(ignore_above_value)); + const auto spill_threshold = + static_cast(config::inverted_index_ram_buffer_size * 1024 * 1024); + _memory_reporter = + std::make_unique<::doris::snii::writer::MemoryReporter>(nullptr, spill_threshold); + _term_buffer = std::make_unique<::doris::snii::writer::SpimiTermBuffer>( + _has_positions, spill_threshold, _memory_reporter.get()); + // G09: join the PROCESS-WIDE build-RAM limiter. The per-writer cap above + // bounds one writer; a load keeps (tablets x concurrency) writers alive at + // once, none of which may ever reach it -- the global registry bounds their + // SUM by asking the largest buffers to spill early (advisory flags honored + // on each writer's own thread; byte-identical output). Budget refreshed + // from the mutable config at every writer init; 0 disables (no + // registration, zero per-token overhead beyond the G08 path). + // G09 anti-storm knobs (see the config comments): the forced-spill floor + // gates both the owner-side honor (a request is a pending no-op until the + // reclaimable arena regrows past it) and the limiter's victim eligibility, + // and the run-file cap merge-compacts a writer's spill runs so the final + // k-way merge's fd fan-in stays bounded. Applied unconditionally -- the + // floor also protects test-seam requests, and the cap also bounds + // per-writer gate-2 runs when the global limiter is off. + _term_buffer->set_forced_spill_min_arena_bytes( + static_cast(std::max(config::snii_forced_spill_min_arena_bytes, 0))); + _term_buffer->set_max_run_files( + static_cast(std::max(config::snii_spill_max_run_files_per_buffer, 0))); + const int64_t global_budget = config::snii_index_writer_global_memory_bytes; + if (global_budget > 0) { + auto* global_limiter = ::doris::snii::writer::GlobalMemoryLimiter::instance(); + global_limiter->set_budget_bytes(global_budget); + global_limiter->set_min_victim_arena_bytes(config::snii_forced_spill_min_arena_bytes); + _term_buffer->attach_global_limiter(global_limiter); + } + // G04 bigram diet: whenever the G01 flush-time bigram df-prune WILL be + // active (config != 0; <0 auto and >0 fixed both resolve to a threshold + // >= 1 at flush), bigram positions are dead bytes and the intern vocabulary + // may be capped -- enable docs+freq-only bigram buffering plus, when + // snii_bigram_vocab_cap_bytes > 0, incremental df==1 eviction with the + // ever-dropped bloom. Captured ONCE here; a mid-import flip of the prune + // config to 0 is rejected at flush (see IndexFileWriter::add_snii_index). + if (_has_positions && config::snii_bigram_prune_min_df != 0) { + const int64_t cap = config::snii_bigram_vocab_cap_bytes; + // G06: mid-feed spill drains need a bigram df gate BEFORE the final doc + // count (and thus the exact flush threshold) exists. A fixed positive + // config IS the flush threshold; the auto formula (< 0) is monotonic in + // doc_count, so its 0-doc floor is a safe LOWER BOUND -- every mid-feed + // drop it allows is one the flush gate would repeat for a + // never-reappearing pair, and each is bloom-recorded anyway, so a + // reappearing pair stays correct (dropped at flush, reader falls back). + // The EXACT effective threshold is re-plumbed per flush by + // LogicalIndexWriter::build_blocks before the final drain. + const int32_t prune_conf = config::snii_bigram_prune_min_df; + const uint32_t drain_min_df = + prune_conf > 0 ? static_cast(prune_conf) + : ::doris::snii::format::default_phrase_bigram_prune_min_df(0); + _term_buffer->configure_bigram_diet(cap > 0 ? static_cast(cap) : 0, drain_min_df); + } + _analyzer_config.analyzer_name = get_analyzer_name_from_properties(_index_meta->properties()); + _analyzer_config.parser_type = get_inverted_index_parser_type_from_string( + get_parser_string_from_properties(_index_meta->properties())); + _analyzer_config.parser_mode = + get_parser_mode_string_from_properties(_index_meta->properties()); + _analyzer_config.char_filter_map = + get_parser_char_filter_map_from_properties(_index_meta->properties()); + _analyzer_config.lower_case = + get_parser_lowercase_from_properties(_index_meta->properties()); + _analyzer_config.stop_words = get_parser_stopwords_from_properties(_index_meta->properties()); + try { + _char_string_reader = inverted_index::InvertedIndexAnalyzer::create_reader( + _analyzer_config.char_filter_map); + if (_should_analyzer) { + _analyzer = inverted_index::InvertedIndexAnalyzer::create_analyzer(&_analyzer_config); + } + } catch (const CLuceneError& e) { + return Status::Error( + "SNII create analyzer failed: {}", e.what()); + } catch (const Exception& e) { + return Status::Error( + "SNII create analyzer failed: {}", e.what()); + } + return Status::OK(); +} + +void SniiIndexColumnWriter::set_direct_load(bool is_direct_load) { + // Feed/sentinel coherence rests on this running before the first row: a + // post-feed flip could emit pair postings and then drop the sentinel that + // vouches for them (or vice versa) within one segment. Latched by an + // explicit marker (in release too): the FIRST call wins, a repeat or late + // call keeps the first-captured decision instead of desyncing the pair + // feed from the sentinel decision at finish(). The late-call drop is + // logged: silently losing the hint would hide a wiring regression as a + // mere perf cliff (segment full-builds despite the config). + DCHECK(!_direct_load_marked && _rid == 0); + if (_direct_load_marked || _rid != 0) { + LOG_EVERY_N(WARNING, 100) << "SNII set_direct_load(" << is_direct_load + << ") ignored (already_marked=" << _direct_load_marked + << ", rows_fed=" << _rid << ") for index " + << (_index_meta != nullptr ? _index_meta->index_id() : -1) + << "; keeping the first-captured defer decision"; + return; + } + _direct_load_marked = true; + _is_direct_load = is_direct_load; + // Bigram-defer decision, captured ONCE by the latch above (mirrors init()'s + // G04 "Captured ONCE here" capture discipline): IndexColumnWriter::create() + // has already run init() when the segment writer calls this, and no row has + // been fed yet, so _has_positions is final and the decision precedes every + // token. The bigram diet that init() may have armed on the term buffer + // stays armed but INERT: with zero pair feeds the pair map is never + // populated (the per-new-unigram pos_suppressed probe is the same two loads + // as the non-defer path) and prepare_pair_terms_for_drain early-returns on + // the empty map. finish() persists this same decision as resident per-index + // metadata; new readers skip impossible pair probes and enter positions + // verification directly, while the omitted sentinel keeps older readers on + // their existing fallback contract. + _phrase_bigrams_deferred = _single_field && is_direct_load && + config::snii_bigram_defer_build_to_compaction && _has_positions; +} + +Status SniiIndexColumnWriter::_add_phrase_bigram_tokens(uint32_t docid) { + if (_bigram_positioned.size() < 2) { + return Status::OK(); + } + + // G05 pair-keyed bigram add: every adjacent pair is fed to the buffer as the + // two UNIGRAM TERM-IDS _add_value_tokens captured when it interned the row's + // tokens, so the per-pair hot path is one integer pair-map probe -- no term + // bytes are hashed, compared or composed during accumulation. The composed + // on-disk term string is materialized inside the buffer only at spill/flush, + // and only for terms actually written. + const bool did_sort = emit_adjacent_phrase_bigrams( + _bigram_positioned, [&](uint32_t left_id, uint32_t right_id, uint32_t position) { + _term_buffer->add_bigram_token(left_id, right_id, docid, position); + }); + // Analyzer token positions are monotonic non-decreasing, so the filtered + // positioned terms are already position-ordered and the guard never sorts. + // The emitted pair SET (and thus the posting bytes after SpimiTermBuffer + // re-sorts on finish) is identical to the pre-refactor primary+secondary-key + // sort. + DCHECK(!did_sort); + return Status::OK(); +} + +Status SniiIndexColumnWriter::_add_value_tokens(const Slice& value, uint32_t docid, + uint32_t position_base, uint32_t* max_position) { + DCHECK(max_position != nullptr); + *max_position = position_base; + if ((!_should_analyzer && value.size > _ignore_above) || (_should_analyzer && value.empty())) { + return Status::OK(); + } + + // clear() keeps the backing capacity across rows so the per-row bigram build + // stops allocating a fresh positioned-term vector every text value / array + // element. + _bigram_positioned.clear(); + // T1a: tokens STREAM from the analyzer straight into the SPIMI buffer as + // string_views (the buffer interns the bytes into its own storage) -- no + // per-row vector and no per-token std::string materialization + // (the old get_analyse_result lane; profile: 3.4-4.7% of import CPU burned + // in token realloc). Golden-byte pins (snii_writer_golden_bytes_test.cpp) + // hold this path byte-identical to the materializing one it replaced. + const bool collect_bigrams = _has_positions && !_phrase_bigrams_deferred; + auto consume_token = [&](std::string_view term, int32_t token_position) { + const uint32_t position = + _has_positions ? position_base + cast_set(token_position) : 0; + // G05: capture the unigram's SPIMI term-id as it is interned; the + // bigram-indexable ones seed the id-keyed pair adds below. Unigram ids + // are stable for the buffer's lifetime (only hidden bigram terms are + // evicted/recycled by the G04 diet), so the pair keys built from them + // stay resolvable until flush materialization. + const uint32_t term_id = _term_buffer->add_token_returning_id(term, docid, position); + *max_position = std::max(*max_position, position); + // Deferred-bigram segments skip the pair capture entirely (compaction + // rebuilds the bigrams later); the unigram add above -- including its + // term-id-returning path -- is deliberately untouched. + if (collect_bigrams && term_id != ::doris::snii::writer::SpimiTermBuffer::kInvalidTermId && + ::doris::snii::format::is_phrase_bigram_indexable_term(term)) { + _bigram_positioned.push_back( + {term_id, position_base + cast_set(token_position)}); + } + }; + + if (!_should_analyzer) { + // Keyword lane: the whole value is one exact-match token at position 0 + // (an EMPTY value is a valid keyword token, mirrored from the old lane). + consume_token(std::string_view(value.data, value.size), 0); + } else { + try { + _char_string_reader->init(value.data, cast_set(value.size), false); + std::unique_ptr token_stream( + _analyzer->tokenStream(L"", _char_string_reader)); + // EXACT InvertedIndexAnalyzer::get_analyse_result semantics, + // including the subtle one: an empty token's position increment is + // dropped WITH the token (not accumulated into the next). + lucene::analysis::Token token; + int32_t position = 0; + while (token_stream->next(&token)) { + if (token.termLength() != 0) { + position += token.getPositionIncrement(); + consume_token( + std::string_view(token.termBuffer(), token.termLength()), + position); + } + } + token_stream->close(); + } catch (const CLuceneError& e) { + return Status::Error( + "SNII analyze value failed: {}", e.what()); + } catch (const Exception& e) { + return Status::Error( + "SNII analyze value failed: {}", e.what()); + } + } + if (!_phrase_bigrams_deferred) { + RETURN_IF_ERROR(_add_phrase_bigram_tokens(docid)); + } + return Status::OK(); +} + +Status SniiIndexColumnWriter::add_values(const std::string /*name*/, const void* values, + size_t count) { + const auto* v = reinterpret_cast(values); + for (size_t i = 0; i < count; ++i) { + uint32_t max_position = 0; + RETURN_IF_ERROR(_add_value_tokens(*v, _rid, 0, &max_position)); + ++v; + ++_rid; + } + return Status::OK(); +} + +Status SniiIndexColumnWriter::add_array_values(size_t field_size, const void* value_ptr, + const uint8_t* nested_null_map, + const uint8_t* offsets_ptr, size_t count) { + if (count == 0) { + return Status::OK(); + } + const auto* offsets = reinterpret_cast(offsets_ptr); + size_t start_off = 0; + for (size_t i = 0; i < count; ++i) { + auto array_elem_size = offsets[i + 1] - offsets[i]; + uint32_t position_base = 0; + for (auto j = start_off; j < start_off + array_elem_size; ++j) { + if (nested_null_map != nullptr && nested_null_map[j] == 1) { + continue; + } + const auto* value = reinterpret_cast( + reinterpret_cast(value_ptr) + j * field_size); + uint32_t max_position = position_base; + RETURN_IF_ERROR(_add_value_tokens(*value, _rid, position_base, &max_position)); + position_base = max_position + 1; + } + start_off += array_elem_size; + ++_rid; + } + return Status::OK(); +} + +void SniiIndexColumnWriter::_report_null_docids_capacity(bool release_all) { + if (_memory_reporter == nullptr) { + return; + } + const int64_t now = + release_all ? 0 : static_cast(_null_docids.capacity() * sizeof(uint32_t)); + if (now != _null_docids_charged_bytes) { + _memory_reporter->report(now - _null_docids_charged_bytes); + _null_docids_charged_bytes = now; + } +} + +Status SniiIndexColumnWriter::add_nulls(uint32_t count) { + // GEOMETRIC BULK reserve -- never an exact one: append_nullable calls + // add_nulls once per NULL RUN (thousands to millions of calls on a large + // interleaved-null segment), and an exact reserve(size()+count) caps + // capacity at "just enough" -- the NEXT call then reallocates and memcpys + // the WHOLE array, defeating geometric growth and turning total memcpy + // quadratic: O(runs x array_bytes). On an agentlogs full-compaction segment + // (12.4M rows, 22% interleaved nulls) that was TBs of memcpy per tablet -- + // the compaction ran 8+x slower than V3 (whose add_nulls is a roaring + // addRange). Doubling on overflow keeps the O(count) amortization AND makes + // one large run pay at most one reallocation. + const size_t need = _null_docids.size() + count; + if (need > _null_docids.capacity()) { + _null_docids.reserve(std::max(need, _null_docids.capacity() * 2)); + } + for (uint32_t i = 0; i < count; ++i) { + _null_docids.push_back(_rid + i); + } + _rid += count; + _report_null_docids_capacity(); + return Status::OK(); +} + +Status SniiIndexColumnWriter::add_array_nulls(const uint8_t* null_map, size_t num_rows) { + DCHECK(_rid >= num_rows); + if (num_rows == 0 || null_map == nullptr) { + return Status::OK(); + } + const auto first_row = _rid - num_rows; + for (size_t i = 0; i < num_rows; ++i) { + if (null_map[i] == 1) { + _null_docids.push_back(cast_set(first_row + i)); + } + } + _report_null_docids_capacity(); + return Status::OK(); +} + +Status SniiIndexColumnWriter::finish() { + DCHECK(_term_buffer != nullptr); + // The same captured decision is persisted with the index below. New readers + // use it to skip impossible hidden-pair probes and enter positions + // verification; the omitted sentinel preserves the existing fallback for + // older readers. _phrase_bigrams_deferred gates both pair feed and + // sentinel, so they can never disagree within one segment. + if (_has_positions && _rid > 0 && !_phrase_bigrams_deferred) { + _term_buffer->add_token(::doris::snii::format::make_phrase_bigram_sentinel_term(), 0, 0); + } + auto status = _term_buffer->status(); + if (!status.ok()) { + return Status::InternalError("SNII term buffer error: {}", status.to_string()); + } + // Ownership of _null_docids hands off to the flush below (transient, + // flush-scoped); release the accumulation-phase charge so the retained + // reporter (and the LOAD MemTracker behind it) balances to zero. + _report_null_docids_capacity(/*release_all=*/true); + const IndexFileWriter::SniiAddIndexOptions options { + .phrase_bigrams_deferred = _phrase_bigrams_deferred, + .is_direct_load = _is_direct_load, + }; + RETURN_IF_ERROR(_index_file_writer->add_snii_index(_index_meta, cast_set(_rid), + std::move(_null_docids), _term_buffer.get(), + _config, options, _memory_reporter.get())); + _index_file_writer->retain_snii_memory_reporter(std::move(_memory_reporter)); + _term_buffer.reset(); + return Status::OK(); +} + +void SniiIndexColumnWriter::close_on_error() { + _term_buffer.reset(); + // Balance the LOAD MemTracker mirror before dropping the reporter. + _report_null_docids_capacity(/*release_all=*/true); + _memory_reporter.reset(); + _null_docids.clear(); +} + +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/snii/snii_index_writer.h b/be/src/storage/index/snii/snii_index_writer.h new file mode 100644 index 00000000000000..ae51c24b93a04c --- /dev/null +++ b/be/src/storage/index/snii/snii_index_writer.h @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "storage/index/index_writer.h" +#include "storage/index/inverted/inverted_index_parser.h" +#include "storage/index/inverted/query/query_info.h" +#include "storage/index/inverted/util/reader.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/snii_phrase_bigram_build.h" +#include "storage/index/snii/writer/memory_reporter.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" +#include "util/slice.h" + +namespace lucene::analysis { +class Analyzer; +} + +namespace doris::segment_v2 { + +class SniiIndexColumnWriter final : public IndexColumnWriter { +public: + SniiIndexColumnWriter(IndexFileWriter* index_file_writer, const TabletIndex* index_meta, + bool single_field); + ~SniiIndexColumnWriter() override = default; + + Status init() override; + void set_direct_load(bool is_direct_load) override; + Status add_values(const std::string name, const void* values, size_t count) override; + Status add_array_values(size_t field_size, const void* value_ptr, + const uint8_t* nested_null_map, const uint8_t* offsets_ptr, + size_t count) override; + Status add_nulls(uint32_t count) override; + Status add_array_nulls(const uint8_t* null_map, size_t num_rows) override; + Status finish() override; + int64_t size() const override { return 0; } + void close_on_error() override; + +#ifdef BE_TEST + // TEST-ONLY view of the accumulated null docids: the growth-policy + // regression pin asserts add_nulls keeps geometric growth (an exact + // reserve(size+count) per null RUN made total memcpy quadratic -- the + // agentlogs full-compaction pathology). + const std::vector& null_docids_for_test() const { return _null_docids; } +#endif + +private: + Status _add_value_tokens(const Slice& value, uint32_t docid, uint32_t position_base, + uint32_t* max_position); + // Emits the hidden phrase-bigram tokens for the row whose bigram-indexable + // unigrams (as SPIMI term-ids + positions) _add_value_tokens collected into + // _bigram_positioned. + Status _add_phrase_bigram_tokens(uint32_t docid); + // Mirrors _null_docids' capacity into _memory_reporter (delta-charged); + // release_all zeroes the charge (finish() handoff / close_on_error()). + void _report_null_docids_capacity(bool release_all = false); + + IndexFileWriter* _index_file_writer = nullptr; + const TabletIndex* _index_meta = nullptr; + bool _should_analyzer = false; + bool _has_positions = false; + // IndexColumnWriter::create() marks ARRAY item indexes as multi-field. + // They retain the hidden bigram layout to preserve existing SNII full-build + // phrase behavior at element boundaries. + bool _single_field = true; + // Latch: set_direct_load() ran. The first call wins; a repeat or late call + // is ignored (and logged) so the pair feed can never desync from the + // sentinel decision at finish(). + bool _direct_load_marked = false; + // Captured by set_direct_load() under the same latch: this writer serves a + // stream/broker load (DataWriteType::TYPE_DIRECT). Consumed at finish() to + // route the prx region to the load-tier zstd level (patch C, + // config::snii_prx_zstd_level_direct_load). + bool _is_direct_load = false; + // Defer the hidden phrase-bigram build to compaction for single-field + // indexes only: no pair tokens, no sentinel, and a resident per-index + // capability flag (kPhraseBigramsDeferred) routes fresh-segment phrases + // directly to positions verification. ARRAY indexes retain their full build + // to preserve existing SNII full-build phrase behavior at element + // boundaries. Captured ONCE in set_direct_load() -- create() runs init() + // first and the segment writer marks direct load before any row is fed -- + // so a mid-load flip of config::snii_bigram_defer_build_to_compaction can + // never desync the pair feed from the sentinel decision at finish(). + bool _phrase_bigrams_deferred = false; + uint32_t _ignore_above = 0; + uint32_t _rid = 0; + ::doris::snii::format::IndexConfig _config = ::doris::snii::format::IndexConfig::kDocsOnly; + InvertedIndexAnalyzerConfig _analyzer_config; + inverted_index::ReaderPtr _char_string_reader; + std::shared_ptr _analyzer; + std::unique_ptr<::doris::snii::writer::MemoryReporter> _memory_reporter; + std::unique_ptr<::doris::snii::writer::SpimiTermBuffer> _term_buffer; + std::vector _null_docids; + // Bytes of _null_docids capacity currently mirrored into _memory_reporter + // (and through it Doris's LOAD MemTracker). Re-charged on growth in + // add_nulls / add_array_nulls, released in finish() / close_on_error() -- + // without it a large interleaved-null segment accumulates untracked RSS the + // G09 limiter cannot see. + int64_t _null_docids_charged_bytes = 0; + // Reused across every _add_value_tokens call: clear() keeps the backing + // capacity so the per-row phrase-bigram build stops re-allocating a fresh + // positioned-term vector on each text row/array element. G05: carries the + // SPIMI unigram TERM-IDS captured while adding the row's unigram tokens (not + // term bytes) -- the bigram emission feeds the id-keyed + // SpimiTermBuffer::add_bigram_token pair path. Single-threaded per-column + // build state (see _add_phrase_bigram_tokens). + std::vector _bigram_positioned; +}; + +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/snii/snii_phrase_bigram_build.h b/be/src/storage/index/snii/snii_phrase_bigram_build.h new file mode 100644 index 00000000000000..9d168fe8dfd43c --- /dev/null +++ b/be/src/storage/index/snii/snii_phrase_bigram_build.h @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +// Phrase-bigram emission for the SNII index writer, extracted here as a +// dependency-free (no Doris/CLucene types) template so it can be unit tested in +// isolation. The writer feeds it the row's indexable analyzer tokens and turns +// every emitted pair into a hidden `make_phrase_bigram_term` posting. +namespace doris::segment_v2 { + +// One analyzer token positioned within a row, for phrase-bigram emission. `term` +// is a NON-owning view into the caller's token storage (the row's TermInfo +// vector); it is valid only for the duration of the emit call. +struct PhrasePositionedTerm { + std::string_view term; + uint32_t position = 0; +}; + +// G05 id-keyed variant: the token is carried as the SPIMI unigram TERM-ID the +// writer captured from add_token_returning_id when it interned the token, so +// the emitted pair feeds SpimiTermBuffer::add_bigram_token(left_id, right_id, +// ...) -- no term bytes flow through the per-pair hot path at all. Unigram ids +// are stable for the buffer's lifetime (only hidden bigram terms are ever +// evicted/recycled), so holding them across the row is safe. The member is +// deliberately named `term` too: emit_adjacent_phrase_bigrams below is generic +// over the element type and forwards `.term` / `.position` unchanged. +struct PhrasePositionedTermId { + uint32_t term = 0; + uint32_t position = 0; +}; + +// Emits every adjacent phrase-bigram pair (left@p, right@p+1) drawn from `terms`. +// +// Contract: `terms` is expected to be ordered by ascending position. Analyzer +// output already satisfies this -- token position is monotonic non-decreasing +// (analyzer.cpp advances `position += getPositionIncrement()` with increment >= +// 0) and the per-array `position_base` is a uniform constant offset -- so the +// guard below is a COLD PATH that only fires when the invariant is violated +// (e.g. a hand-shuffled unit-test input). The return value reports whether the +// guard sorted: the writer asserts `!did_sort` to document the invariant, while +// an out-of-order caller still gets the correct pair set (the guard sorts first). +// +// The pre-refactor sort used a secondary term key; it is intentionally DROPPED. +// Only the SET of emitted (left, right, position) triples is defined -- the +// order of `emit` calls within a position group is unspecified. That is exactly +// what the downstream SpimiTermBuffer needs: it dedups per term and re-sorts on +// finish, so emission order never reaches the on-disk posting bytes. +// +// Generic over the positioned-term element `PT` (PhrasePositionedTerm or +// PhrasePositionedTermId -- anything with `.term` and a uint32_t `.position`). +// `emit` has signature void(decltype(PT::term) left, decltype(PT::term) right, +// uint32_t position); `position` is the left token's position. +template +bool emit_adjacent_phrase_bigrams(std::vector& terms, Emit&& emit) { + bool did_sort = false; + if (!std::ranges::is_sorted(terms, {}, &PT::position)) { + std::ranges::sort(terms, {}, &PT::position); + did_sort = true; + } + + size_t left_begin = 0; + while (left_begin < terms.size()) { + size_t left_end = left_begin + 1; + while (left_end < terms.size() && terms[left_end].position == terms[left_begin].position) { + ++left_end; + } + + size_t right_begin = left_end; + while (right_begin < terms.size() && + terms[right_begin].position <= terms[left_begin].position) { + ++right_begin; + } + if (right_begin == terms.size() || + terms[right_begin].position != terms[left_begin].position + 1) { + left_begin = left_end; + continue; + } + size_t right_end = right_begin + 1; + while (right_end < terms.size() && + terms[right_end].position == terms[right_begin].position) { + ++right_end; + } + + for (size_t l = left_begin; l < left_end; ++l) { + for (size_t r = right_begin; r < right_end; ++r) { + emit(terms[l].term, terms[r].term, terms[l].position); + } + } + left_begin = left_end; + } + return did_sort; +} + +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/snii/stats/snii_stats_provider.cpp b/be/src/storage/index/snii/stats/snii_stats_provider.cpp new file mode 100644 index 00000000000000..521f3fedac2c0b --- /dev/null +++ b/be/src/storage/index/snii/stats/snii_stats_provider.cpp @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/stats/snii_stats_provider.h" + +#include +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/stats_block.h" +#include "storage/index/snii/io/batch_range_fetcher.h" + +namespace doris::snii::stats { + +using format::DictEntry; +using format::NormsPodReader; +using format::RegionRef; + +namespace { + +// Resolves a term's DictEntry. *found=false for an absent term (OK status). +Status LookupEntry(const reader::LogicalIndexReader& idx, std::string_view term, bool* found, + DictEntry* entry) { + uint64_t frq_base = 0; + uint64_t prx_base = 0; + return idx.lookup(term, found, entry, &frq_base, &prx_base); +} + +} // namespace + +Status SniiStatsProvider::open(const reader::LogicalIndexReader* idx, SniiStatsProvider* out) { + if (idx == nullptr || out == nullptr) { + return Status::Error("stats_provider: null argument"); + } + out->idx_ = idx; + const auto& sb = idx->stats(); + out->doc_count_ = sb.doc_count; + out->indexed_doc_count_ = sb.indexed_doc_count; + out->sum_total_term_freq_ = sb.sum_total_term_freq; + + const RegionRef& norms = idx->section_refs().norms; + if (norms.length == 0) { + out->has_norms_ = false; + return Status::OK(); + } + + io::BatchRangeFetcher fetcher(idx->reader()); + const size_t h = fetcher.add(norms.offset, norms.length); + RETURN_IF_ERROR(fetcher.fetch()); + Slice framed = fetcher.get(h); + out->norms_bytes_.assign(framed.data(), framed.data() + framed.size()); + RETURN_IF_ERROR(NormsPodReader::open(Slice(out->norms_bytes_), &out->norms_reader_)); + out->has_norms_ = true; + return Status::OK(); +} + +double SniiStatsProvider::avgdl() const { + const uint64_t denom = std::max(1, indexed_doc_count_); + return static_cast(sum_total_term_freq_) / static_cast(denom); +} + +Status SniiStatsProvider::doc_freq(std::string_view term, uint64_t* df) const { + if (df == nullptr) + return Status::Error("stats_provider: null df"); + *df = 0; + bool found = false; + DictEntry entry; + RETURN_IF_ERROR(LookupEntry(*idx_, term, &found, &entry)); + if (found) *df = entry.df; + return Status::OK(); +} + +Status SniiStatsProvider::total_term_freq(std::string_view term, uint64_t* ttf) const { + if (ttf == nullptr) + return Status::Error("stats_provider: null ttf"); + *ttf = 0; + bool found = false; + DictEntry entry; + RETURN_IF_ERROR(LookupEntry(*idx_, term, &found, &entry)); + if (!found) return Status::OK(); + // tier>=T2 entries carry the total term frequency directly in ttf_delta (the + // LogicalIndexWriter stores ttf there, not a delta from df). G16-f blocks + // (kNoTermStats: freq-dropped index) omit it -- fail with the semantic + // error instead of silently returning the default 0 (mixed old/new + // segments would otherwise disagree on the same term). + if (!entry.term_stats_present) { + return Status::Error( + "snii_stats: ttf requested but the dict block carries no term stats " + "(freq-dropped index)"); + } + *ttf = entry.ttf_delta; + return Status::OK(); +} + +Status SniiStatsProvider::encoded_norm(uint32_t docid, uint8_t* out) const { + if (out == nullptr) + return Status::Error("stats_provider: null out"); + if (!has_norms_) { + return Status::Error( + "stats_provider: index has no norms"); + } + return norms_reader_.try_encoded_norm(docid, out); +} + +} // namespace doris::snii::stats diff --git a/be/src/storage/index/snii/stats/snii_stats_provider.h b/be/src/storage/index/snii/stats/snii_stats_provider.h new file mode 100644 index 00000000000000..66a7a155f82a39 --- /dev/null +++ b/be/src/storage/index/snii/stats/snii_stats_provider.h @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/norms_pod.h" +#include "storage/index/snii/reader/logical_index_reader.h" + +// SniiStatsProvider -- exposes the native SNII scoring statistics required by +// BM25, sourced directly from the on-disk structures of one logical index: +// - segment-level counts (doc_count, indexed_doc_count, sum_total_term_freq) +// from the StatsBlock embedded in the per-index meta block. +// - per-term df / ttf from the term's DictEntry (resolved through the reader's +// lookup flow). The LogicalIndexWriter stores ttf directly in ttf_delta for +// tier>=T2 entries, so total_term_freq returns entry.ttf_delta. +// - per-doc length normalization byte (encoded_norm) from the norms POD, +// range-read once at open via section_refs().norms and parsed with +// NormsPodReader. +// +// avgdl() = sum_total_term_freq / max(1, indexed_doc_count): the average document +// length used by BM25 length normalization. The provider performs no scoring; it +// only surfaces the statistics so query::Bm25Scorer can combine them. +namespace doris::snii::stats { + +class SniiStatsProvider { +public: + SniiStatsProvider() = default; + + // Binds to idx and materializes the norms POD (one range read) when the index + // carries scoring norms. idx must outlive this provider. A scoring index + // without a norms section, or a corrupt norms POD, returns a non-OK Status. + static Status open(const reader::LogicalIndexReader* idx, SniiStatsProvider* out); + + // Segment-level counts (direct StatsBlock fields). + uint64_t doc_count() const { return doc_count_; } + uint64_t indexed_doc_count() const { return indexed_doc_count_; } + uint64_t sum_total_term_freq() const { return sum_total_term_freq_; } + + // Average document length: sum_total_term_freq / max(1, indexed_doc_count). + double avgdl() const; + + // Per-term document frequency. Absent term -> *df = 0 (OK status). + Status doc_freq(std::string_view term, uint64_t* df) const; + + // Per-term total term frequency (ttf = df + ttf_delta at tier>=T2). Absent + // term -> *ttf = 0 (OK status). + Status total_term_freq(std::string_view term, uint64_t* ttf) const; + + // 1-byte encoded doc-length norm for docid (raw byte from the norms POD). + // Out-of-range docid -> InvalidArgument; index without norms -> InvalidArgument. + Status encoded_norm(uint32_t docid, uint8_t* out) const; + + bool has_norms() const { return has_norms_; } + +private: + const reader::LogicalIndexReader* idx_ = nullptr; + uint64_t doc_count_ = 0; + uint64_t indexed_doc_count_ = 0; + uint64_t sum_total_term_freq_ = 0; + bool has_norms_ = false; + // Owned copy of the framed norms section bytes; norms_reader_ borrows from it. + std::vector norms_bytes_; + format::NormsPodReader norms_reader_; +}; + +} // namespace doris::snii::stats diff --git a/be/src/storage/index/snii/version.h b/be/src/storage/index/snii/version.h new file mode 100644 index 00000000000000..1f2ebe79cd9852 --- /dev/null +++ b/be/src/storage/index/snii/version.h @@ -0,0 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once +#define SNII_VERSION_MAJOR 0 +#define SNII_VERSION_MINOR 1 +#define SNII_VERSION_STRING "0.1.0" diff --git a/be/src/storage/index/snii/writer/bigram_drop_filter.h b/be/src/storage/index/snii/writer/bigram_drop_filter.h new file mode 100644 index 00000000000000..9600b93fbf4e4c --- /dev/null +++ b/be/src/storage/index/snii/writer/bigram_drop_filter.h @@ -0,0 +1,194 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +namespace doris::snii::writer { + +// EVER-DROPPED bloom over evicted phrase-bigram terms (G04 "bigram diet" +// phase 2). One instance lives per SpimiTermBuffer; every bigram term the +// vocab-cap eviction drops mid-build is inserted here, and at flush the +// writer's process_term drops any bigram term the filter says it MAY have +// evicted -- IN ADDITION to the G01 df threshold. The direction of error is +// one-sided by design: +// * a FALSE POSITIVE only OVER-drops a never-evicted bigram, which is safe +// under the G01 reader contract (a bigram dict miss on a segment whose +// meta declares pruning falls back to the generic positions phrase path, +// same results, just slower); +// * a false NEGATIVE is impossible (a bloom never forgets an insertion), so +// an evicted term -- whose surviving postings would be INCOMPLETE (they +// miss the docids dropped with the eviction) -- can never materialize. +// +// SIZING: ~kBitsPerEntry (10) bits per expected eviction, kNumHashes (4) +// probes -> ~1.2% per-partition false-positive rate at design fill. The total +// eviction count is not knowable up front (it tracks the corpus' unique-pair +// tail, not any config), so the filter is SCALABLE: partitions of +// geometrically doubling capacity are chained as insertions accumulate; a +// query probes every partition (compound FPP ~= partition count * 1.2%, and +// every extra hit is still only a safe over-drop). The first partition is +// sized by the caller from the vocab cap (roughly the live-term count the cap +// can hold), so the common case is one partition. +// +// Keys are the RAW SYNTHETIC TERM BYTES (make_phrase_bigram_term's +// marker+varint+left+right layout) hashed with the same FNV-1a 64 the SPIMI +// intern table uses, so insert (at eviction, from the owned vocab string) and +// probe (at flush, from TermPostings::term) agree byte-for-byte. +// +// PREHASHED entry points (G05 pair-keyed bigrams): a caller that never +// composed the synthetic term string can insert/probe with the FNV-1a 64 it +// computed PIECEWISE over the same byte layout (marker / varint(len(left)) / +// left / right -- SpimiTermBuffer's fragment hash). Because the piecewise +// update feeds the identical byte sequence, insert_hashed(piecewise_fnv) sets +// exactly the bits insert(composed_string) would, and the flush-time +// maybe_contains(TermPostings::term) probe still matches -- the filter's key +// contract stays "the synthetic term bytes" regardless of which side composed +// a string. +// +// insert() is CHECK-THEN-INSERT: a key the filter already reports as present +// consumes no new bits and does not advance the partition fill count, so a +// pair that is evicted, re-interned, and evicted again (each eviction +// re-inserting the same bytes) never double-counts toward partition growth. +// +// Single-threaded (one writer build path), like the rest of the SPIMI writer. +class BigramDropFilter { +public: + static constexpr uint32_t kBitsPerEntry = 10; + static constexpr uint32_t kNumHashes = 4; + + // `initial_capacity` is the first partition's design insertion count (the + // caller derives it from the vocab cap); clamped to a small floor so a + // degenerate cap still yields a usable filter. + explicit BigramDropFilter(uint64_t initial_capacity) + : next_capacity_(initial_capacity < kMinPartitionCapacity ? kMinPartitionCapacity + : initial_capacity) {} + + BigramDropFilter(const BigramDropFilter&) = delete; + BigramDropFilter& operator=(const BigramDropFilter&) = delete; + + // Records `term` as ever-dropped. No-op (and no bit writes) when the filter + // already reports the key as maybe-present. + void insert(std::string_view term) { insert_hashed(fnv1a64(term)); } + + // May this term have been evicted? False negatives never occur; a false + // positive only over-drops (safe, see header comment). + bool maybe_contains(std::string_view term) const { + return maybe_contains_hashed(fnv1a64(term)); + } + + // Prehashed variants: `h` MUST be the FNV-1a 64 of the synthetic term bytes + // (same constants/byte order as fnv1a64 below). The SPIMI pair-keyed + // eviction computes it piecewise from the two unigram strings without ever + // composing the term, and the flush-time string probe above lands on the + // identical bits. + void insert_hashed(uint64_t h) { + if (maybe_contains_hashed(h)) { + return; + } + if (partitions_.empty() || partitions_.back().inserted >= partitions_.back().capacity) { + add_partition(); + } + Partition& p = partitions_.back(); + const uint64_t h2 = (h >> 32) | 1ULL; // odd step (Kirsch-Mitzenmacher) + for (uint32_t i = 0; i < kNumHashes; ++i) { + const uint64_t bit = (h + static_cast(i) * h2) % p.n_bits; + p.bits[bit >> 6] |= (1ULL << (bit & 63)); + } + ++p.inserted; + ++insertions_; + } + + bool maybe_contains_hashed(uint64_t h) const { + if (partitions_.empty()) { + return false; + } + const uint64_t h2 = (h >> 32) | 1ULL; + for (const Partition& p : partitions_) { + bool all = true; + for (uint32_t i = 0; i < kNumHashes; ++i) { + const uint64_t bit = (h + static_cast(i) * h2) % p.n_bits; + if ((p.bits[bit >> 6] & (1ULL << (bit & 63))) == 0) { + all = false; + break; + } + } + if (all) { + return true; + } + } + return false; + } + + // Distinct-ish insertions accepted (check-then-insert dedups repeats). + uint64_t insertions() const { return insertions_; } + bool empty() const { return insertions_ == 0; } + size_t partition_count() const { return partitions_.size(); } + + // Resident filter bytes (the bitsets); observability / RSS accounting. + uint64_t allocated_bytes() const { + uint64_t total = 0; + for (const Partition& p : partitions_) { + total += p.bits.size() * sizeof(uint64_t); + } + return total; + } + +private: + static constexpr uint64_t kMinPartitionCapacity = 4096; + + struct Partition { + std::vector bits; + uint64_t n_bits = 0; + uint64_t capacity = 0; // design insertion count for this partition + uint64_t inserted = 0; + }; + + // Same FNV-1a 64 constants/byte order as the SPIMI intern hash, so the + // composed-string key hashed here equals the piecewise intern hash of the + // same pair (identical byte sequence through the identical update). + static uint64_t fnv1a64(std::string_view bytes) noexcept { + uint64_t h = 1469598103934665603ULL; + for (const char c : bytes) { + h ^= static_cast(c); + h *= 1099511628211ULL; + } + return h; + } + + void add_partition() { + Partition p; + p.capacity = next_capacity_; + next_capacity_ *= 2; // geometric growth: few partitions even at huge tails + p.n_bits = p.capacity * kBitsPerEntry; + // Round up to whole 64-bit words; n_bits is the modulus so keep it exact + // to the allocated word count (no phantom always-zero tail bits). + const uint64_t words = (p.n_bits + 63) / 64; + p.n_bits = words * 64; + p.bits.assign(static_cast(words), 0); + partitions_.push_back(std::move(p)); + } + + std::vector partitions_; + uint64_t next_capacity_ = kMinPartitionCapacity; + uint64_t insertions_ = 0; +}; + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/compact_posting_pool.cpp b/be/src/storage/index/snii/writer/compact_posting_pool.cpp new file mode 100644 index 00000000000000..9cef1a38b9f572 --- /dev/null +++ b/be/src/storage/index/snii/writer/compact_posting_pool.cpp @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/compact_posting_pool.h" + +#include +#include +#include + +namespace doris::snii::writer { + +// Gentle (~1.5x) many-level payload-capacity schedule. Starting at 5 bytes with a +// slow ramp keeps the over-allocated FINAL slice small for the millions of low-df +// terms (the dominant arena-overhead source) while still reaching multi-KiB slices +// for high-df chains in a bounded number of hops (so the per-slice 4-byte forward +// pointer stays a small fraction of a large chain's bytes). +const uint32_t CompactPostingPool::kSliceSizes[kLevelCount] = { + 5, 8, 12, 18, 27, 40, 60, 90, 135, 202, 303, 455, 683, 1024, 1536, 2304}; +const uint8_t CompactPostingPool::kNextLevel[kLevelCount] = {1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 15}; + +CompactPostingPool::CompactPostingPool() = default; + +uint32_t CompactPostingPool::kSliceSizes_level0() { + return kSliceSizes[0]; +} + +uint32_t CompactPostingPool::kSliceSize_at(int level) { + return kSliceSizes[level]; +} + +uint8_t CompactPostingPool::kNextLevel_at(int level) { + return kNextLevel[level]; +} + +void CompactPostingPool::reset() { + std::vector>().swap(blocks_); + next_offset_ = 0; + payload_bytes_ = 0; +} + +uint32_t CompactPostingPool::alloc_run(uint32_t bytes) { + const uint32_t in_block = next_offset_ & kBlockMask; + // A fresh block is needed when (a) there is no tail block yet, (b) the run does + // not fit in the current tail block's remaining space, or (c) next_offset_ sits + // exactly on a block boundary whose block has not been allocated (a previous run + // that exactly filled the tail leaves next_offset_ == blocks_.size()*kBlockSize, + // so in_block == 0 must NOT be mistaken for an empty fresh block). + const bool tail_exists = (next_offset_ >> kBlockShift) < blocks_.size(); + const bool need_block = !tail_exists || in_block + bytes > kBlockSize; + // Hard invariant (see arena_bytes()): the uint32 offset must never wrap. The spimi + // accumulator force-spills below 4 GiB, but enforce it here too -- in release as + // well as debug -- so any direct user of the pool fails loudly instead of silently + // aliasing block 0. We are a library: throw and let the caller decide how to + // handle it, rather than aborting the process. The run starts either in the + // current tail or at a new block's base; compute that start in 64 bits before the + // uint32 arithmetic can wrap. + const uint64_t run_start = + need_block ? static_cast(blocks_.size()) * kBlockSize : next_offset_; + if (run_start + bytes > UINT32_MAX) { + throw std::overflow_error( + "snii: CompactPostingPool arena exceeded the 4 GiB uint32 offset limit; " + "the caller must spill before this point"); + } + if (need_block) { + blocks_.emplace_back(kBlockSize, 0); + next_offset_ = static_cast((blocks_.size() - 1) * kBlockSize); + } + const uint32_t off = next_offset_; + next_offset_ += bytes; + return off; +} + +uint32_t CompactPostingPool::alloc_slice(int level, uint32_t* slice_end) { + const uint32_t cap = kSliceSizes[level]; + const uint32_t first = alloc_run(cap + kPtrBytes); + *slice_end = first + cap; + // Zero the forward pointer so a not-yet-extended tail slice reads next_head == 0. + std::memset(at(*slice_end), 0, kPtrBytes); + return first; +} + +uint32_t CompactPostingPool::read_ptr(uint32_t slice_end) const { + uint32_t v; + std::memcpy(&v, at(slice_end), sizeof(v)); + return v; +} + +void CompactPostingPool::write_ptr(uint32_t slice_end, uint32_t next_head) { + std::memcpy(at(slice_end), &next_head, sizeof(next_head)); +} + +uint32_t CompactPostingPool::start_chain(SliceWriter* w, uint8_t* level) { + *level = 0; + const uint32_t head = alloc_slice(0, &w->slice_end); + w->cur = head; + return head; +} + +CompactPostingPool::Cursor::Cursor(const CompactPostingPool* pool, uint32_t head, uint64_t budget) + : pool_(pool), cur_(head), level_(0), budget_(budget) { + // The first slice is level 0; its payload region ends kSliceSizes[0] bytes in. + slice_end_ = head + CompactPostingPool::kSliceSizes[0]; +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/compact_posting_pool.h b/be/src/storage/index/snii/writer/compact_posting_pool.h new file mode 100644 index 00000000000000..df1f2ffeb474ab --- /dev/null +++ b/be/src/storage/index/snii/writer/compact_posting_pool.h @@ -0,0 +1,271 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +namespace doris::snii::writer { + +// SEGMENTED BYTE ARENA with per-term SLICED runs (a ByteBlockPool, after Lucene). +// +// WHY: the SPIMI accumulator's bulk memory is the per-term posting bytes. Backing +// each term with its own std::vector pays two taxes that dominate peak +// RSS at scale: (1) geometric-growth doubling slack (~1.17x of the live payload), +// and (2) a 24-32 B vector/struct header per term (hundreds of thousands of +// terms). This pool removes both: all term bytes live in a few large fixed-size +// blocks (so slack is ~one block, amortized to ~1.05x), and a term needs only two +// 32-bit cursors of live state (chain head for reads + write head for appends). +// +// HOW (slices): a term's bytes are not stored contiguously. They live in a chain +// of SLICES of geometrically growing payload capacity (the kSliceSizes schedule: +// 4, 8, 16, ... bytes of payload). Each slice is laid out as +// [ payload bytes ... ][ 4-byte forward pointer ] +// The forward pointer holds the absolute offset of the next slice's first payload +// byte (0 while the slice is still the tail of the chain). When a slice's payload +// region fills, the writer allocates a larger slice, stores its head into the old +// slice's 4 pointer bytes, and keeps appending. A reader walks the chain by +// reading payload bytes until a slice boundary, then following the pointer. +// +// Both writer and reader recompute each slice's capacity from the chain's slice +// INDEX (0, 1, 2, ...) via the deterministic schedule, so neither needs to store +// per-slice sizes. The writer carries the current slice's end offset in its +// SliceWriter handle; the reader recomputes capacities as it advances. +// +// Offsets are GLOBAL absolute byte indices into the logical concatenation of all +// blocks: offset = block_index * kBlockSize + byte_in_block. kBlockSize is a power +// of two, so offset -> (block, byte) is a shift/mask. +class CompactPostingPool { +public: + // Block size (power of two). 32 KiB blocks keep per-block tail waste tiny (it + // matters at the smaller 1M scale where the whole arena is only tens of MiB) and + // bound the outer vector header cost; at the 5M scale a few thousand + // blocks is still cheap. Empirically the lowest peak across both scales. + static constexpr uint32_t kBlockShift = 15; + static constexpr uint32_t kBlockSize = 1u << kBlockShift; // 32 KiB + static constexpr uint32_t kBlockMask = kBlockSize - 1; + + // Per-slice forward-pointer width (absolute uint32 next-slice offset). + static constexpr uint32_t kPtrBytes = 4; + + // Geometric slice payload-capacity schedule and the level transition. Level i + // slices hold kSliceSizes[i] payload bytes; on overflow the chain advances to + // kNextLevel[i] (capping at the largest level). A GENTLE (~1.5x) many-level + // schedule starting small minimizes the over-allocated final slice (the + // dominant arena overhead) while keeping the per-slice forward-pointer count + // bounded for high-df chains. + static constexpr int kLevelCount = 16; + + CompactPostingPool(); + + CompactPostingPool(const CompactPostingPool&) = delete; + CompactPostingPool& operator=(const CompactPostingPool&) = delete; + + // Payload capacity (bytes) of a fresh level-0 slice. Exposed for tests that need + // to fill exactly one slice without hardcoding the schedule. + static uint32_t kSliceSizes_level0(); + + // Payload capacity of the slice at `level`, and the level a chain advances to when + // that slice overflows. Exposed (like kSliceSizes_level0) so tests can simulate the + // arena's bump allocator exactly -- e.g. to construct an EXACT block-boundary fill -- + // without hardcoding the private schedule. `level` must be in [0, kLevelCount). + static uint32_t kSliceSize_at(int level); + static uint8_t kNextLevel_at(int level); + + // Live append handle for one term's chain. POD, 8 bytes: the absolute write + // cursor and the absolute end of the current slice's payload region. The chain's + // current slice LEVEL is kept by the caller (a uint8, packed alongside its other + // flags) so this handle stays 8 bytes -- shaving the per-term accumulator. `head` + // (the chain's first payload offset) is also stored by the CALLER (the read entry + // point); start_chain returns it. + struct SliceWriter { + uint32_t cur = 0; // next byte to write (absolute) + uint32_t slice_end = 0; // one-past-last payload byte of the current slice + }; + + // Begins a fresh chain, initializing `w` to its first (level-0) slice and + // *level to 0, and returns the chain head (absolute first payload offset). + uint32_t start_chain(SliceWriter* w, uint8_t* level); + + // Appends one payload byte to the chain described by `w` / `*level`, growing the + // chain with a new linked slice (and advancing *level) when the current slice's + // payload region is exhausted. + void append_byte(SliceWriter* w, uint8_t* level, uint8_t value); + + // Total live payload bytes ever written across all chains (excludes slice + // forward-pointer overhead). Drives the spill-threshold estimate only. + uint64_t payload_bytes() const { return payload_bytes_; } + + // Bytes the arena currently occupies (block_count * kBlockSize). The pool + // addresses bytes with a uint32 offset (next_offset_), so the arena MUST stay + // below 4 GiB or alloc_run wraps and silently aliases block 0. The accumulator + // watches this to force a safety spill before the wrap; alloc_run also enforces it + // directly (throws std::overflow_error on a would-be wrap) so a direct user of the + // pool fails loudly rather than silently corrupting. + // Hard invariant: a single CompactPostingPool never exceeds UINT32_MAX bytes. + uint64_t arena_bytes() const { return static_cast(blocks_.size()) << kBlockShift; } + + // Releases ALL blocks back to the OS. Called after the accumulator is fully + // drained (or before a spill's next fill) so no input-side bytes stay resident. + void reset(); + + // ---- Reader ---------------------------------------------------------------- + // Forward cursor over one term's chain, yielding its payload bytes in write + // order by walking the slice forward pointers. + // + // CONTRACT of the `budget` ctor argument (single, unambiguous meaning): + // `budget` is an UPPER BOUND on the number of bytes this cursor may yield. It + // is NOT required to equal the exact payload length: passing the exact length + // is fine, and so is passing any value >= it (the production caller passes the + // chain's write-head offset, which always bounds the payload from above). The + // cursor is SELF-TERMINATING: once it walks off the last written byte it sees + // the tail slice's zero forward pointer and stops, regardless of how much + // budget remains. So an over-large budget can never make next() read past the + // chain (no aliasing of block 0, no off-chain access) -- the budget is purely a + // secondary cap. has_next() is therefore a reliable "more bytes remain" + // predicate for ANY budget >= the true length: it becomes false at the smaller + // of (budget exhausted, chain tail reached). + class Cursor { + public: + Cursor(const CompactPostingPool* pool, uint32_t head, uint64_t budget); + + // True while the cursor can still yield a REAL payload byte: the budget is not + // spent AND the cursor has not reached the chain tail. It peeks the tail forward + // pointer at a slice boundary so it never reports a phantom trailing byte, making + // has_next()/next() a safe loop for any budget >= the true payload length. + bool has_next() const; + // Yields the next payload byte. Returns 0 (and yields no more) once the chain + // tail is reached or the budget is spent -- never reads past the chain. + uint8_t next(); + + private: + const CompactPostingPool* pool_; + uint32_t cur_; // absolute read cursor + uint32_t slice_end_; // one-past-last payload byte of the current slice + uint32_t level_; // current slice level + uint64_t budget_; // remaining byte budget (upper bound on bytes to yield) + }; + + // Builds a cursor over the chain at `head`. `budget` is an UPPER BOUND on bytes to + // read (see Cursor's contract): the exact payload length or anything larger. The + // production caller passes the write-head offset, which always bounds the payload + // from above; the cursor self-terminates at the chain tail regardless. + Cursor cursor(uint32_t head, uint64_t budget) const { return Cursor(this, head, budget); } + +private: + static const uint32_t kSliceSizes[kLevelCount]; + static const uint8_t kNextLevel[kLevelCount]; + + uint8_t* at(uint32_t off) { return &blocks_[off >> kBlockShift][off & kBlockMask]; } + const uint8_t* at(uint32_t off) const { return &blocks_[off >> kBlockShift][off & kBlockMask]; } + + // Reads/writes the 4-byte forward pointer at the END of a slice whose payload + // region ends at `slice_end` (pointer occupies [slice_end, slice_end+4)). + uint32_t read_ptr(uint32_t slice_end) const; + void write_ptr(uint32_t slice_end, uint32_t next_head); + + // Reserves `bytes` contiguous bytes from the arena tail (a fresh block if the + // current tail cannot hold them) and returns the first reserved absolute offset. + // `bytes` must be <= kBlockSize. + uint32_t alloc_run(uint32_t bytes); + + // Allocates a slice at `level` (payload region + 4 pointer bytes), zeroes its + // forward pointer, and returns the first payload offset; sets *slice_end. + uint32_t alloc_slice(int level, uint32_t* slice_end); + + std::vector> blocks_; // fixed kBlockSize blocks + uint32_t next_offset_ = 0; // global bump pointer (absolute) into the tail block + uint64_t payload_bytes_ = 0; +}; + +// ---- Inlined per-byte hot paths -------------------------------------------- +// append_byte (one call per encoded payload byte during SPIMI ingest) and +// Cursor::has_next/next (one call per arena byte during finalize drain/merge) +// are the writer's two hottest per-byte loops. Their callers live in a DIFFERENT +// translation unit (spimi_term_buffer.cpp), and the build has no LTO/IPO/unity +// build, so an out-of-line .cpp definition forces a non-inlinable cross-TU call +// (plus stack spill of cur_/slice_end_/budget_) on every single byte. Defining +// the bodies inline HERE lets the caller's TU inline them, keeping the cursor +// state in registers and eliminating the per-byte call/ret. They are placed +// AFTER the class so every private member they touch (at, read_ptr, alloc_slice, +// write_ptr, kSliceSizes, kNextLevel, payload_bytes_) is already declared and +// visible; Cursor is a nested class, so it may use the enclosing pool's privates +// (read_ptr/at) through pool_. The cold slice-overflow helpers (alloc_slice, +// write_ptr) stay out-of-line in the .cpp -- only the per-byte work is inlined. +// Pure code move from the .cpp: behavior and produced bytes are identical. + +inline void CompactPostingPool::append_byte(SliceWriter* w, uint8_t* level, uint8_t value) { + if (w->cur == w->slice_end) { + // Current slice payload region is full: grow the chain with a larger slice and + // record the link in the old slice's trailing pointer bytes. + const uint8_t next_level = kNextLevel[*level]; + uint32_t new_end = 0; + const uint32_t new_head = alloc_slice(next_level, &new_end); + write_ptr(w->slice_end, new_head); + *level = next_level; + w->cur = new_head; + w->slice_end = new_end; + } + *at(w->cur) = value; + ++w->cur; + ++payload_bytes_; +} + +inline bool CompactPostingPool::Cursor::has_next() const { + if (budget_ == 0) { + return false; + } + // At a slice boundary, the chain continues only if the forward pointer is non-zero; + // a zero pointer is the tail marker (offset 0 is never a valid next-slice head). Peek + // it so has_next() never reports a phantom byte that next() would have to fabricate. + if (cur_ == slice_end_) { + return pool_->read_ptr(slice_end_) != 0; + } + return true; +} + +inline uint8_t CompactPostingPool::Cursor::next() { + // Budget guard: the caller's stated upper bound is spent -- yield nothing more. + if (budget_ == 0) { + return 0; + } + if (cur_ == slice_end_) { + // Reached this slice's payload boundary. Follow the forward pointer to the next + // slice -- UNLESS it is zero, which marks the CHAIN TAIL (offset 0 is always the + // pool's very first slice, never a valid *next*-slice head, so a zero pointer is + // unambiguously "no more slices"). Without this tail check, an over-reading caller + // would follow the zero pointer to offset 0 and alias block 0's bytes (or read an + // unallocated block) -- UB. Stopping here makes the cursor self-terminating and + // safe regardless of how large a budget the caller passed. + const uint32_t next_head = pool_->read_ptr(slice_end_); + if (next_head == 0) { + budget_ = 0; // chain exhausted: no further bytes exist + return 0; + } + level_ = CompactPostingPool::kNextLevel[level_]; + cur_ = next_head; + slice_end_ = next_head + CompactPostingPool::kSliceSizes[level_]; + } + const uint8_t v = *pool_->at(cur_); + ++cur_; + --budget_; + return v; +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/global_memory_limiter.cpp b/be/src/storage/index/snii/writer/global_memory_limiter.cpp new file mode 100644 index 00000000000000..ed8038b03d3b49 --- /dev/null +++ b/be/src/storage/index/snii/writer/global_memory_limiter.cpp @@ -0,0 +1,166 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/global_memory_limiter.h" + +#include +#include +#include + +#include "common/logging.h" + +namespace doris::snii::writer { + +GlobalMemoryLimiter* GlobalMemoryLimiter::instance() { + // Intentionally leaked (never destroyed): buffers un-register from their + // destructors, which may run during static teardown at process exit -- a + // destroyed registry there would be use-after-free. A leaked singleton + // makes the "limiter outlives every attached buffer" contract + // unconditional. + static auto* g_instance = new GlobalMemoryLimiter(); + return g_instance; +} + +void GlobalMemoryLimiter::register_buffer(std::atomic* spill_flag, int64_t resident_bytes, + int64_t arena_bytes) { + std::lock_guard guard(mutex_); + Entry& slot = entries_[spill_flag]; // zero-initialized on first insert + total_bytes_locked_ += resident_bytes - slot.resident; + slot.resident = resident_bytes; + slot.arena = arena_bytes; +} + +void GlobalMemoryLimiter::report(std::atomic* spill_flag, int64_t resident_bytes, + int64_t arena_bytes) { + std::lock_guard guard(mutex_); + auto it = entries_.find(spill_flag); + if (it == entries_.end()) { + // Not registered (or already unregistered): ignore rather than + // resurrect an entry nobody will remove. + return; + } + total_bytes_locked_ += resident_bytes - it->second.resident; + it->second.resident = resident_bytes; + it->second.arena = arena_bytes; + request_spills_locked(); +} + +void GlobalMemoryLimiter::unregister_buffer(std::atomic* spill_flag) { + std::lock_guard guard(mutex_); + auto it = entries_.find(spill_flag); + if (it == entries_.end()) { + return; + } + total_bytes_locked_ -= it->second.resident; + entries_.erase(it); +} + +int64_t GlobalMemoryLimiter::total_bytes() const { + std::lock_guard guard(mutex_); + return total_bytes_locked_; +} + +size_t GlobalMemoryLimiter::registered_count() const { + std::lock_guard guard(mutex_); + return entries_.size(); +} + +bool GlobalMemoryLimiter::budget_degraded() const { + std::lock_guard guard(mutex_); + return degraded_; +} + +void GlobalMemoryLimiter::request_spills_locked() { + const int64_t budget = budget_bytes_.load(std::memory_order_relaxed); + if (budget <= 0 || total_bytes_locked_ <= budget) { + return; + } + // BUDGET SANITY (graceful degradation): the budget bounds SPILLABLE + // memory only -- a forced spill reclaims a buffer's posting arena, never + // its persistent vocab / pair-map structures. When the per-writer budget + // share falls below the minimum useful working set, those persistent + // bytes alone exceed the budget and no amount of forced spilling can get + // under it; flagging would only manufacture a storm of floor-sized runs + // for zero net relief (the conc=16 wikipedia field failure). Log ONCE per + // degradation episode and stop flagging until the ratio recovers (writers + // draining/unregistering, or a raised budget). + const auto writer_count = static_cast(entries_.size()); + const int64_t min_useful = min_useful_budget_per_writer_bytes_.load(std::memory_order_relaxed); + if (writer_count > 0 && min_useful > 0 && budget / writer_count < min_useful) { + if (!degraded_) { + degraded_ = true; + LOG(WARNING) << "SNII global index-writer memory budget cannot be met by spilling: " + << "budget=" << budget << " B across " << writer_count + << " registered writers is below the minimum useful " << min_useful + << " B/writer (persistent per-writer structures dominate; the budget " + << "bounds SPILLABLE memory, not persistent memory). Forced spilling is " + << "suspended until writers drain or the budget " + << "(snii_index_writer_global_memory_bytes) is raised."; + } + return; + } + if (degraded_) { + degraded_ = false; // episode over; a relapse will log once again + LOG(INFO) << "SNII global index-writer memory budget recovered (" << budget << " B / " + << writer_count << " writers); forced spilling resumes."; + } + const int64_t overage = total_bytes_locked_ - budget; + // FORCED-SPILL FLOOR + PER-BUFFER COOLDOWN: only buffers holding at least + // the floor of RECLAIMABLE arena are eligible victims -- flagging a + // smaller arena would cut a tiny run and reclaim next to nothing, and a + // buffer that just honored a forced spill (arena ~0) stays exempt until + // its arena regrows past the floor. Never below one byte: an empty arena + // has nothing to write to a run. + const int64_t victim_floor = + std::max(min_victim_arena_bytes_.load(std::memory_order_relaxed), 1); + // Largest RECLAIMABLE consumers first: victims are ranked by their + // spillable ARENA (what the forced spill frees), not the + // persistent-dominated resident total. n is the live writer count of the + // process (at most a few hundred), and this only runs while over budget, + // so the sort under the mutex is bounded, allocation-light work. + std::vector*>> by_arena; + by_arena.reserve(entries_.size()); + for (const auto& [flag, entry] : entries_) { + if (entry.arena >= victim_floor) { + by_arena.emplace_back(entry.arena, flag); + } + } + std::sort(by_arena.begin(), by_arena.end(), + [](const auto& a, const auto& b) { return a.first > b.first; }); + int64_t covered = 0; + for (const auto& [arena, flag] : by_arena) { + if (covered >= overage) { + break; + } + // An ALREADY-pending flag (set by an earlier report, owner not yet at + // its next token) counts toward the covered sum without a fresh store: + // re-flagging it would be a no-op, and skipping the store avoids + // dirtying the owner's cache line every over-budget report. + if (!flag->load(std::memory_order_relaxed)) { + flag->store(true, std::memory_order_relaxed); + } + // Count the ARENA toward coverage: it is all a forced spill of this + // victim can actually reclaim. When the persistent remainder keeps the + // sum over budget even after every eligible arena is flagged, the loop + // simply flags them all -- each still cuts a >= floor-sized run, and + // the cooldown above keeps any one buffer from being re-victimized + // before it has a floor's worth of arena again. + covered += arena; + } +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/global_memory_limiter.h b/be/src/storage/index/snii/writer/global_memory_limiter.h new file mode 100644 index 00000000000000..2f19168fc3a4a8 --- /dev/null +++ b/be/src/storage/index/snii/writer/global_memory_limiter.h @@ -0,0 +1,201 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include + +namespace doris::snii::writer { + +// Process-wide SNII build-RAM limiter (G09) -- the index-build analogue of +// Doris's MemTableMemoryLimiter. Every live SPIMI accumulator registers here +// and forwards its ACCURATE resident-byte total (G08) plus its SPILLABLE +// arena bytes through its existing debounced report path; when the resident +// sum across ALL writers of the process (tablets x segments x concurrent +// loads) exceeds the configured budget, the limiter requests spills from the +// largest-ARENA eligible buffers until the flagged (reclaimable) arena sum +// covers the overage. +// +// WHY: the per-writer gate-2 cap (e.g. 512 MiB) bounds ONE writer, but a load +// keeps (tablets x concurrency) writers alive at once -- wikipedia at +// concurrency 16 held 100+ writers at 300-500 MB each (~41 GiB), none of which +// ever reached its own cap, so per-writer spilling never fired. This registry +// bounds the SUM. +// +// ASYNC-SAFE REQUESTS: the SPIMI structures are single-threaded, so the +// limiter must never spill on the reporting thread. A request is a relaxed +// atomic FLAG on the target buffer (SpimiTermBuffer::global_spill_requested_) +// that the OWNER's next add_token / maybe_spill_after_token observes and +// honors on its own thread (bypassing the G08 per-writer anti-churn floor but +// still requiring the FORCED-SPILL FLOOR of reclaimable arena -- see below -- +// so every forced run is worth its fixed costs). Flags are ADVISORY: the +// owner may have just spilled or drained -- the flag is then a (harmless) +// no-op or one extra floor-sized run. The limiter itself only ever takes its +// registry mutex and flips atomics; it never blocks a reporting thread beyond +// that mutex and never calls back into a buffer. +// +// LIFETIME: buffers un-register in their destructor. register / report / +// unregister all serialize on the registry mutex, and flags are only ever set +// UNDER that mutex, so once unregister_buffer returns no thread can touch the +// (about-to-die) flag again. The limiter must outlive every attached buffer +// (trivial for the process singleton; test-local instances are declared before +// the buffers they serve). +// +// THE BUDGET BOUNDS SPILLABLE MEMORY, NOT PERSISTENT MEMORY: a forced spill +// releases only the buffer's posting ARENA; the persistent vocab / pair-map / +// slot structures (~100-500 MB per wikipedia writer) survive it. If the +// persistent bytes of all writers alone exceed the budget, NO amount of +// spilling can reach the budget -- the budget is a back-pressure valve over +// the reclaimable arenas, not a hard cap on resident RSS. Three defenses keep +// an unreachable budget from degenerating into a forced-spill storm (the +// conc=16 wikipedia field failure: every report re-flagged every buffer, each +// honoring with one 32 KiB arena block -> thousands of tiny runs per buffer -> +// EMFILE re-opening them for the k-way merge -> failed loads): +// * VICTIMS BY ARENA: victims are selected by their reported SPILLABLE arena +// bytes -- the only bytes a forced spill can actually reclaim -- never by +// the persistent-dominated resident total, and only buffers whose arena is +// at least min_victim_arena_bytes (config snii_forced_spill_min_arena_bytes) +// are eligible. Every forced run is therefore at least floor-sized. +// * PER-BUFFER COOLDOWN: right after a buffer honors a forced spill its +// arena is ~0, below the victim floor, so it is EXEMPT from new flags +// until the arena regrows past the floor. No timer state: the eligibility +// rule IS the cooldown. +// * BUDGET SANITY: when budget / registered_count falls below a per-writer +// minimum useful share (kMinUsefulBudgetPerWriterBytes), persistent +// structures alone dominate and the budget is provably unreachable by +// spilling; the limiter LOGS ONCE (per degradation episode) and stops +// flagging until the ratio recovers (writers finishing / budget raised). +class GlobalMemoryLimiter { +public: + // Victim floor default (mirrors config snii_forced_spill_min_arena_bytes): + // a buffer is only ever asked to force-spill once its reclaimable arena + // holds at least this much, so no forced run is smaller than this. + static constexpr int64_t kDefaultMinVictimArenaBytes = 64LL << 20; // 64 MiB + // Minimum useful per-writer budget share: below budget/registered_count == + // this, spilling cannot meet the budget (persistent per-writer structures + // alone exceed the share) -- the limiter degrades to log-once-and-stop. + static constexpr int64_t kMinUsefulBudgetPerWriterBytes = 96LL << 20; // 96 MiB + + // Local instances are constructible for unit tests; production code uses + // the process singleton below. + GlobalMemoryLimiter() = default; + GlobalMemoryLimiter(const GlobalMemoryLimiter&) = delete; + GlobalMemoryLimiter& operator=(const GlobalMemoryLimiter&) = delete; + + // Process singleton (never destroyed before the writers that use it). + static GlobalMemoryLimiter* instance(); + + // Budget in bytes across every registered buffer; <= 0 disables the + // limiter (registration still tracked, but no flag is ever set). Refreshed + // from the mutable BE config at each writer init, so a config change takes + // effect for the whole registry at the next writer creation. + void set_budget_bytes(int64_t budget_bytes) { + budget_bytes_.store(budget_bytes, std::memory_order_relaxed); + } + int64_t budget_bytes() const { return budget_bytes_.load(std::memory_order_relaxed); } + + // Victim-eligibility floor over a buffer's reported SPILLABLE arena bytes + // (see the class comment). Refreshed from the mutable BE config alongside + // the budget at each writer init. Values < 1 behave as 1 (an empty arena + // is never a victim -- there would be nothing to write to the run). + void set_min_victim_arena_bytes(int64_t bytes) { + min_victim_arena_bytes_.store(bytes, std::memory_order_relaxed); + } + int64_t min_victim_arena_bytes() const { + return min_victim_arena_bytes_.load(std::memory_order_relaxed); + } + + // Per-writer minimum useful budget share for the degradation check. + // Production keeps the default (kMinUsefulBudgetPerWriterBytes); tests + // lower it to exercise selection with synthetic byte scales. + void set_min_useful_budget_per_writer_bytes(int64_t bytes) { + min_useful_budget_per_writer_bytes_.store(bytes, std::memory_order_relaxed); + } + + // True while the limiter is in the degraded log-once-and-stop state (the + // per-writer budget share fell below the useful minimum at the last + // over-budget report). Observability / tests. + bool budget_degraded() const; + + // Adds `spill_flag` (the owning buffer's advisory request flag; also the + // entry's identity) with its current resident bytes and its SPILLABLE + // arena bytes (<= resident; what a forced spill can reclaim, the victim + // selection key). Re-registering an already-registered flag just updates + // its bytes. Never sets flags itself: a single registration cannot create + // NEW overage worth reacting to before the buffer's first report. + void register_buffer(std::atomic* spill_flag, int64_t resident_bytes, + int64_t arena_bytes); + + // Updates the entry's resident and spillable-arena bytes (ABSOLUTE totals, + // not deltas -- self-healing across any missed report). When the + // registered RESIDENT sum exceeds the budget, sets the request flags of + // the largest-ARENA eligible entries (arena >= the victim floor; see the + // class comment) -- counting entries whose flag is ALREADY pending toward + // the covered sum, so an in-flight request is not amplified -- until the + // flagged ARENA bytes cover the overage or eligible entries run out. A + // report for a flag that is not registered is ignored. + void report(std::atomic* spill_flag, int64_t resident_bytes, int64_t arena_bytes); + + // Removes the entry (subtracting its bytes from the total). After this + // returns, the limiter never touches `spill_flag` again -- safe to destroy + // the owning buffer. + void unregister_buffer(std::atomic* spill_flag); + + // Registered sum / entry count, for tests and observability. + int64_t total_bytes() const; + size_t registered_count() const; + +private: + // One registered buffer's last reported byte totals. `resident` feeds the + // over-budget decision (the sum the budget bounds); `arena` -- the + // SPILLABLE subset a forced spill can actually reclaim -- feeds victim + // selection and eligibility. + struct Entry { + int64_t resident = 0; + int64_t arena = 0; + }; + + // Called with mutex_ held whenever the total may exceed the budget: after + // the budget-sanity check (degrade to log-once-and-stop when the + // per-writer share is below the useful minimum), sorts the ELIGIBLE + // entries (arena >= victim floor) by ARENA descending and flags from the + // top until the flagged arena sum covers (total - budget) or eligible + // entries run out. O(n log n) over the live writer count (at most a few + // hundred) -- bounded work under the mutex, no I/O, no callbacks. + void request_spills_locked(); + + mutable std::mutex mutex_; + std::atomic budget_bytes_ {0}; + std::atomic min_victim_arena_bytes_ {kDefaultMinVictimArenaBytes}; + std::atomic min_useful_budget_per_writer_bytes_ {kMinUsefulBudgetPerWriterBytes}; + // All below guarded by mutex_. total_bytes_locked_ is the maintained sum + // of entries_ resident values (avoids an O(n) walk per report). + int64_t total_bytes_locked_ = 0; + // Degradation episode latch: set (with ONE warning log) when the budget + // sanity check fails, cleared when a later over-budget report finds the + // ratio recovered -- so a relapse logs again, but a sustained episode + // logs exactly once. + bool degraded_ = false; + phmap::flat_hash_map*, Entry> entries_; +}; + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/logical_index_writer.cpp b/be/src/storage/index/snii/writer/logical_index_writer.cpp new file mode 100644 index 00000000000000..4e978e259ebd51 --- /dev/null +++ b/be/src/storage/index/snii/writer/logical_index_writer.cpp @@ -0,0 +1,989 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/logical_index_writer.h" + +#include +#include +#include +#include +#include +#include + +#include "common/logging.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/encoding/zstd_codec.h" +#include "storage/index/snii/format/bsbf.h" +#include "storage/index/snii/format/dict_block.h" +#include "storage/index/snii/format/dict_block_directory.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/norms_pod.h" +#include "storage/index/snii/format/null_bitmap.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/format/prx_pod.h" + +namespace doris::snii::writer { + +using format::BlockRef; +using format::DictBlockBuilder; +using format::DictBlockDirectoryBuilder; +using format::DictEntry; +using format::DictEntryEnc; +using format::DictEntryKind; +using format::FrqPreludeColumns; +using format::PerIndexMetaBuilder; +using format::SampledTermIndexBuilder; +using format::SectionRefs; +using format::WindowMeta; + +namespace { + +// Target false-positive probability for the block-split bloom XFilter. Sizes +// the filter via Parquet OptimalNumOfBytes; L0 keeps the probe in memory and L1 +// keeps the per-query cost at one 32-byte block. +constexpr double kBsbfFpp = 0.01; +// Force-raw level for .frq dd/freq regions. Their plaintext is PFOR-bit-packed +// doc-deltas/freqs -- already high-entropy, so zstd shrinks ~30 MB of input by +// <0.1 MiB while burning ~0.4s CPU (and an extra crc pass over the compressed +// bytes) at 5M. We force raw here and keep zstd only on .prx (which compresses +// ~77%). Output stays self-describing: the region meta records zstd=false. +constexpr int kRawFrqRegion = 0; +// G16-i: auto raw-vs-zstd (choose smaller) for dd regions of docs-only bigram +// windows -- see the EncodeRegionsInto call in BuildWindowedPosting. +constexpr int kAutoZstdRegion = -1; +// Windows per super-block in the two-level prelude directory (design section +// 5). +constexpr uint32_t kPreludeGroupSize = 64; +// zstd level for whole-DICT-block compression comes from +// SniiIndexInput::dict_block_zstd_level (default 3: ~40% on the 64KiB +// front-coded blocks at ~120 MiB/s encode / ~600 MiB/s decode; higher levels +// trade import CPU for size, decode speed unchanged). G16-h made it (and the +// .prx auto level) caller-tunable. + +using format::FrqRegionMeta; + +// Encodes one window's dd region (and freq region when has_freq) into separate +// buffers, returning their codec metadata. The dd region is the docs-only data; +// the freq region is the skippable suffix. Used for both the grouped windowed +// layout (regions concatenated into posting-level blocks) and the single-window +// slim/inline layout ([dd_region][freq_region]). +Status EncodeRegions(std::span docids, std::span freqs, + uint64_t win_base, bool has_freq, std::vector* dd_out, + FrqRegionMeta* dd_meta, std::vector* freq_out, + FrqRegionMeta* freq_meta) { + ByteSink dd_sink; + RETURN_IF_ERROR(format::build_dd_region(docids, win_base, kRawFrqRegion, &dd_sink, dd_meta)); + *dd_out = dd_sink.take(); + if (!has_freq) { + *freq_out = std::vector(); + *freq_meta = FrqRegionMeta {}; + return Status::OK(); + } + ByteSink freq_sink; + RETURN_IF_ERROR(format::build_freq_region(freqs, kRawFrqRegion, &freq_sink, freq_meta)); + *freq_out = freq_sink.take(); + return Status::OK(); +} + +// Reusable per-window scratch for the windowed builder. Each ByteSink RETAINS +// its capacity across windows (clear(), not re-construct), so encoding a +// high-df term split into thousands of windows allocates the scratch ONCE +// instead of churning thousands of small buffers (which fragment the heap arena +// and raise peak RSS). +struct WindowScratch { + ByteSink dd_sink; + ByteSink freq_sink; + ByteSink prx_sink; +}; + +// Encodes one window's dd (and freq) region into the scratch sinks and appends +// the bytes directly to the grouped blocks via LayoutWindowRegions. Reuses the +// sinks. +Status EncodeRegionsInto(WindowScratch* sc, std::span docids, + std::span freqs, uint64_t win_base, bool has_freq, + FrqRegionMeta* dd_meta, FrqRegionMeta* freq_meta, + int dd_zstd_level = kRawFrqRegion) { + sc->dd_sink.clear(); + RETURN_IF_ERROR( + format::build_dd_region(docids, win_base, dd_zstd_level, &sc->dd_sink, dd_meta)); + if (has_freq) { + sc->freq_sink.clear(); + RETURN_IF_ERROR(format::build_freq_region(freqs, kRawFrqRegion, &sc->freq_sink, freq_meta)); + } else { + *freq_meta = FrqRegionMeta {}; + } + return Status::OK(); +} + +// Builds a single .prx window directly from a FLAT positions slice + its +// parallel freqs slice (doc d owns the next freqs[d] entries). Byte-identical +// to building from per-doc vectors, but with NO vector-of-vectors +// materialization: the writer indexes straight into the term's flat positions +// buffer. +Status MakePrxWindow(std::span positions_flat, std::span freqs, + int prx_zstd_level, std::vector* out) { + ByteSink sink; + // Negative level == prx auto mode at |level| (G16-h); -3 == the historic auto. + RETURN_IF_ERROR(format::build_prx_window_flat(positions_flat, freqs, -prx_zstd_level, &sink)); + *out = sink.take(); + return Status::OK(); +} + +uint32_t MaxOf(std::span v) { + uint32_t m = 0; + for (uint32_t x : v) { + if (x > m) m = x; + } + return m; +} + +// Fused single-pass term-level freq statistics: total_freq (running sum) and +// max_freq (running max) in ONE scan, reused by validate_term (has_prx +// position-count budget), stats_.sum_total_term_freq, and the DictEntry +// ttf_delta/max_freq. Byte-identical to the former separate SumOf/MaxOf scans: +// same left-to-right accumulation order and the same max init of 0, so a freq of +// 0 never lowers the max. Bumps the test-only op-count seam exactly once per +// term-level scan (one call per term from process_term). +FreqStats fuse_freq_stats(const std::vector& freqs) { + testing::note_term_freq_scan(); + FreqStats fs; + for (uint32_t f : freqs) { + fs.total_freq += f; + fs.max_freq = std::max(f, fs.max_freq); + } + return fs; +} + +// Computes a window's WAND max_norm: the encoded norm yielding the LARGEST BM25 +// length contribution (smallest length penalty), i.e. the SMALLEST encoded norm +// among the window's docs (smaller dl => higher score). When norms are +// unavailable (no scoring), returns 0 -- decode_norm(0)=1.0 is the smallest +// possible dl, giving a correct (loosest) upper bound. +uint8_t WindowMaxNorm(const std::vector& norms, std::span docs) { + if (norms.empty() || docs.empty()) return 0; + uint8_t best = 0xFF; // decode_norm uses the byte directly; min byte => max score + for (uint32_t docid : docs) { + if (docid >= norms.size()) continue; // defensive: out-of-range doc has no norm + if (norms[docid] < best) best = norms[docid]; + } + return best == 0xFF ? 0 : best; +} + +// Window doc count by df: high-df windowed terms combine kFrqBaseUnit units +// into larger (kAdaptiveWindowDocs) windows; both are whole multiples of the +// base unit so .prx alignment and win_base/last_docid semantics are preserved. +uint32_t AdaptiveWindowDocs(uint32_t df) { + return df >= format::kAdaptiveWindowDfThreshold ? format::kAdaptiveWindowDocs + : format::kFrqBaseUnit; +} + +// Builds the two-level .frq prelude for a windowed term and returns its bytes. +Status BuildPrelude(const std::vector& windows, bool has_freq, bool has_prx, + std::vector* out) { + FrqPreludeColumns cols; + cols.has_freq = has_freq; + cols.has_prx = has_prx; + cols.group_size = kPreludeGroupSize; + cols.windows = windows; + ByteSink sink; + RETURN_IF_ERROR(format::build_frq_prelude(cols, &sink)); + *out = sink.take(); + return Status::OK(); +} + +void AppendBytes(std::vector* dst, const std::vector& src) { + dst->insert(dst->end(), src.begin(), src.end()); +} + +// One windowed term's grouped .frq layout (design 1.6): all dd regions form the +// dd-block, all freq regions form the freq-block. The final frq span is +// [prelude][dd-block][freq-block]. The .prx windows are STREAMED straight to +// the posting sink (the container output) during pass 1 (not buffered here) -- +// so the widest term's ~tens-of-MiB prx bytes never co-exist with the dd/freq +// blocks at peak RSS; only prx_total_len (the entry's prx byte span) is +// tracked. Per-window metadata (region offsets/lens/modes/crcs, prx_off within +// the entry) is recorded for the prelude. +struct WindowedPosting { + std::vector dd_block; // dd_region_0 ++ dd_region_1 ++ ... + std::vector freq_block; // freq_region_0 ++ ... (empty if !has_freq) + uint64_t prx_total_len = 0; // total .prx bytes streamed for this entry + std::vector windows; +}; + +// Fills a window's region locator fields in m from its dd/freq region metas and +// the running dd-block / freq-block offsets, then appends the region bytes to +// the blocks. has_freq controls whether the freq region is laid out. +void LayoutWindowRegions(const FrqRegionMeta& dd_meta, const std::vector& dd_bytes, + const FrqRegionMeta& freq_meta, const std::vector& freq_bytes, + bool has_freq, WindowedPosting* out, WindowMeta* m) { + m->dd_zstd = dd_meta.zstd; + m->dd_off = static_cast(out->dd_block.size()); + m->dd_disk_len = dd_meta.disk_len; + m->dd_uncomp_len = dd_meta.uncomp_len; + m->crc_dd = dd_meta.crc; + AppendBytes(&out->dd_block, dd_bytes); + if (!has_freq) return; + m->freq_zstd = freq_meta.zstd; + m->freq_off = static_cast(out->freq_block.size()); + m->freq_disk_len = freq_meta.disk_len; + m->freq_uncomp_len = freq_meta.uncomp_len; + m->crc_freq = freq_meta.crc; + AppendBytes(&out->freq_block, freq_bytes); +} + +// Splits a windowed term's postings into base-unit-aligned windows (size chosen +// by df via AdaptiveWindowDocs). Each window's dd/freq regions are encoded +// separately and grouped: all dd regions into the dd-block, all freq regions +// into the freq-block. Records per-window region metadata + WAND max_norm. +// +// TWO-PASS, MEMORY-AWARE: the widest term (df in the millions) is the dominant +// merge-phase peak-RSS source -- its flat positions_flat alone is tens of MiB +// and would otherwise co-exist with the encoded output blocks at the peak +// moment. +// pass 1 (prx): builds every window's .prx bytes, then FREES positions_flat +// (the single largest source array) before any dd/freq block +// grows. +// pass 2 (dd/freq): encodes the dd/freq regions from docids/freqs only. +// `tp` is taken by mutable reference; positions_flat is freed after pass 1 and +// docids/freqs are freed by the caller after this returns. Output bytes are +// byte-identical to the single-pass build (regions/prelude/prx are +// independent). +Status BuildWindowedPosting(TermPostings& tp, bool has_freq, bool has_prx, + const std::vector& norms, io::FileWriter* posting_out, + WindowedPosting* out, uint32_t window_docs_override = 0, + int prx_zstd_level = 3) { + // window_docs_override > 0 (G16-e docs-only bigrams) replaces the df-based + // adaptive sizing; 0 keeps the byte-identical adaptive layout. + const uint32_t unit = window_docs_override > 0 + ? window_docs_override + : AdaptiveWindowDocs(static_cast(tp.docids.size())); + const size_t n = tp.docids.size(); + const std::span all_docs(tp.docids); + const std::span all_freqs(tp.freqs); + + // Size the per-term transient blocks up front so a very-high-df term (split + // into thousands of windows, dd/freq blocks of MiB) does not grow them by + // geometric doubling -- which would briefly hold the old+new buffer + // co-resident at the build peak. windows count is exact; dd/freq use a + // conservative byte/doc upper estimate (PFOR-packed deltas are typically <= 2 + // B/doc). Slack is freed when the term ends. + out->windows.reserve((n + unit - 1) / unit); + out->dd_block.reserve(n * 2); + if (has_freq) out->freq_block.reserve(n); + + WindowScratch sc; // reused across all windows (no per-window allocation churn) + + // ---- pass 1: prx (STREAMED to the output) + window skeleton ---- + // Each window's .prx bytes are appended straight to the posting sink + // (container output) as they are built, so the entry's full prx payload (tens + // of MiB for the widest term) is never buffered in RAM alongside the dd/freq + // blocks that pass 2 grows. m.prx_off is the byte offset WITHIN this entry's + // prx span (running prx_total_len), matching the reader's prx_off_delta + + // meta.prx_off contract. + { + // Positions come either from the flat buffer or, for very-high-df terms, + // from a sequential pump (so the term's full positions are never + // materialized). Both yield the SAME positions in the SAME order, so the + // prx bytes are identical. + const bool streamed = static_cast(tp.pos_pump); + const std::span all_pos(tp.positions_flat); + std::vector win_pos_buf; // reused per window when streaming + uint64_t win_base = 0; + size_t pos_off = 0; + for (size_t start = 0; start < n; start += unit) { + const size_t len = std::min(unit, n - start); + const auto docs = all_docs.subspan(start, len); + const auto freqs = all_freqs.subspan(start, len); + WindowMeta m; + m.last_docid = docs.back(); + m.win_base = win_base; + m.doc_count = static_cast(docs.size()); + m.max_freq = MaxOf(freqs); + m.max_norm = WindowMaxNorm(norms, docs); + size_t win_pos = 0; + for (uint32_t f : freqs) win_pos += f; + if (has_prx) { + std::span pos_span; + if (streamed) { + win_pos_buf.resize(win_pos); + if (win_pos != 0) tp.pos_pump(win_pos_buf.data(), win_pos); + pos_span = std::span(win_pos_buf); + } else { + pos_span = all_pos.subspan(pos_off, win_pos); + } + sc.prx_sink.clear(); + RETURN_IF_ERROR(format::build_prx_window_flat(pos_span, freqs, -prx_zstd_level, + &sc.prx_sink)); + m.prx_off = out->prx_total_len; + m.prx_len = static_cast(sc.prx_sink.size()); + RETURN_IF_ERROR(posting_out->append(sc.prx_sink.view())); + out->prx_total_len += m.prx_len; + } + pos_off += win_pos; + out->windows.push_back(m); + win_base = m.last_docid; + } + } + // Positions are fully consumed; free the largest source array before pass 2 + // grows the dd/freq blocks, so the source positions never co-exist with them. + std::vector().swap(tp.positions_flat); + + // ---- pass 2: dd (and freq) regions from docids/freqs only ---- + uint64_t win_base = 0; + size_t wi = 0; + for (size_t start = 0; start < n; start += unit, ++wi) { + const size_t len = std::min(unit, n - start); + const auto docs = all_docs.subspan(start, len); + const auto freqs = all_freqs.subspan(start, len); + FrqRegionMeta dd_meta, freq_meta; + // G16-i: docs-only bigram windows (window_docs_override > 0) try zstd on + // the dd region (choose-smaller; FrqRegionMeta.zstd self-describes). + // Ultra-dense pair postings carry highly repetitive PFOR delta patterns + // that raw encoding leaves ~3 bits/doc on the table; unigram dd stays + // force-raw (measured: zstd shrinks ~30MB of PFOR input by <0.1MB). + RETURN_IF_ERROR( + EncodeRegionsInto(&sc, docs, freqs, win_base, has_freq, &dd_meta, &freq_meta, + window_docs_override > 0 ? kAutoZstdRegion : kRawFrqRegion)); + LayoutWindowRegions(dd_meta, sc.dd_sink.buffer(), freq_meta, sc.freq_sink.buffer(), + has_freq, out, &out->windows[wi]); + win_base = out->windows[wi].last_docid; + } + return Status::OK(); +} + +} // namespace + +namespace testing { +namespace { +// Function-local-static op-count seam backing term_freq_scans(). One atomic, +// relaxed: the writer build path is single-threaded, so only the COUNT matters, +// not ordering (the atomic keeps it race-clean if a test ever parallelizes). +std::atomic& term_freq_scan_counter() { + static std::atomic counter {0}; + return counter; +} +} // namespace + +void note_term_freq_scan() { + term_freq_scan_counter().fetch_add(1, std::memory_order_relaxed); +} +uint64_t term_freq_scans() { + return term_freq_scan_counter().load(std::memory_order_relaxed); +} +void reset_term_freq_scans() { + term_freq_scan_counter().store(0, std::memory_order_relaxed); +} + +namespace { +// G01 bigram-diet counters: bumped once per non-sentinel phrase-bigram TERM (not +// per token) at the process_term materialize/prune decision. Always-on relaxed +// atomics like term_freq_scan_counter -- one branch + one add per bigram term is +// noise next to the term's encode work. +std::atomic& bigram_materialized_counter() { + static std::atomic counter {0}; + return counter; +} +std::atomic& bigram_pruned_counter() { + static std::atomic counter {0}; + return counter; +} +std::atomic& bigram_max_pruned_counter() { + static std::atomic counter {0}; + return counter; +} +std::atomic& bigram_bloom_dropped_counter() { + static std::atomic counter {0}; + return counter; +} +std::atomic& bigram_freq_elided_counter() { + static std::atomic counter {0}; + return counter; +} +} // namespace + +void note_bigram_term_materialized() { + bigram_materialized_counter().fetch_add(1, std::memory_order_relaxed); +} +void note_bigram_term_pruned() { + bigram_pruned_counter().fetch_add(1, std::memory_order_relaxed); +} +void note_bigram_term_max_pruned() { + bigram_max_pruned_counter().fetch_add(1, std::memory_order_relaxed); +} +void note_bigram_bloom_dropped() { + bigram_bloom_dropped_counter().fetch_add(1, std::memory_order_relaxed); +} +void note_bigram_freq_elided() { + bigram_freq_elided_counter().fetch_add(1, std::memory_order_relaxed); +} +uint64_t bigram_terms_materialized() { + return bigram_materialized_counter().load(std::memory_order_relaxed); +} +uint64_t bigram_terms_pruned() { + return bigram_pruned_counter().load(std::memory_order_relaxed); +} +uint64_t bigram_terms_max_pruned() { + return bigram_max_pruned_counter().load(std::memory_order_relaxed); +} +uint64_t bigram_bloom_dropped() { + return bigram_bloom_dropped_counter().load(std::memory_order_relaxed); +} +uint64_t bigram_freqs_elided() { + return bigram_freq_elided_counter().load(std::memory_order_relaxed); +} +void reset_bigram_prune_counters() { + bigram_materialized_counter().store(0, std::memory_order_relaxed); + bigram_pruned_counter().store(0, std::memory_order_relaxed); + bigram_max_pruned_counter().store(0, std::memory_order_relaxed); + bigram_bloom_dropped_counter().store(0, std::memory_order_relaxed); + bigram_freq_elided_counter().store(0, std::memory_order_relaxed); +} +// Forwards to the real fused helper so pure boundary tests exercise production +// code (not a test-local re-implementation). +FreqStats fuse_freq_stats_for_test(const std::vector& freqs) { + return fuse_freq_stats(freqs); +} +} // namespace testing + +LogicalIndexWriter::LogicalIndexWriter(const SniiIndexInput& in) + : index_id_(in.index_id), + index_suffix_(in.index_suffix), + tier_(format::tier_of(in.config)), + has_prx_(format::has_positions(in.config)), + // G16-c: the caller can drop freq layout entirely (in.write_freq == + // false) on a freq-capable tier -- see SniiIndexInput::write_freq. + has_freq_(format::tier_of(in.config) >= format::IndexTier::kT2 && in.write_freq), + has_norms_(format::has_scoring(in.config)), + // Bigram df-pruning only makes sense for positions-capable configs (the + // only ones that emit hidden phrase bigrams); force 0 otherwise so the + // meta never (mis)declares pruning on a docs-only index. + bigram_prune_min_df_(format::has_positions(in.config) ? in.bigram_prune_min_df : 0), + // G15 upper bound rides the same non-positional force-off: a docs-only + // index has no bigrams, and declaring pruning there would send every + // 2-term phrase-shaped miss down a pointless fallback branch. + bigram_prune_max_df_(format::has_positions(in.config) ? in.bigram_prune_max_df : 0), + // The direct-load writer only arms this for positional indexes. Keep the + // format invariant local to the writer as well so core callers cannot + // publish a no-bigram capability on a docs-only index. + phrase_bigrams_deferred_(format::has_positions(in.config) && in.phrase_bigrams_deferred), + // The G04 bloom drop rides the SAME reader contract as the df prune (a + // dict miss falls back only when the meta declares pruning), so it is + // honored ONLY when the effective threshold is non-zero. NOTE: the + // threshold expression above must match -- use the member-init ordering + // guarantee (bigram_prune_min_df_ is declared before this member). + bigram_ever_dropped_(bigram_prune_min_df_ > 0 ? in.bigram_ever_dropped : nullptr), + doc_count_(in.doc_count), + null_docids_(in.null_docids), + terms_(in.terms), + term_source_(in.term_source), + encoded_norms_(in.encoded_norms), + target_dict_block_bytes_(in.target_dict_block_bytes != 0 + ? in.target_dict_block_bytes + : format::kDefaultTargetDictBlockBytes), + dict_block_zstd_level_(in.dict_block_zstd_level), + prx_zstd_level_(in.prx_zstd_level), + // No independent dict cap: the dict spills via the writer's UNIFIED + // gate-2 cap (in.mem_reporter->over_cap()); UINT64_MAX disables the local + // per-buffer cap. + dict_buf_(UINT64_MAX, "dict", in.mem_reporter) {} + +Status LogicalIndexWriter::validate_term(const TermPostings& tp, uint64_t total_freq, + bool expect_positions) const { + if (tp.freqs.size() != tp.docids.size()) { + return Status::Error( + "logical_index: freqs length must equal docids"); + } + if (expect_positions) { + // total_freq is the fused sum(freqs) computed once by the caller (no + // internal re-sum). Streamed positions (pos_pump set): validate against the + // declared pos_total (positions_flat is intentionally empty). Otherwise + // validate the flat buffer. Skipped (expect_positions == false) exactly + // when this term writes no .prx: G04 position-suppressed bigram terms + // arrive with an empty positions_flat by design. + const uint64_t have = tp.pos_pump ? tp.pos_total : tp.positions_flat.size(); + if (total_freq != have) { + return Status::Error( + "logical_index: positions count must equal sum(freqs)"); + } + } + for (size_t i = 1; i < tp.docids.size(); ++i) { + if (tp.docids[i] <= tp.docids[i - 1]) { + return Status::Error( + "logical_index: docids must be strictly ascending"); + } + } + return Status::OK(); +} + +// Emits a windowed term: splits into base-unit windows, encodes each window's +// dd/freq regions separately, groups them at posting level, builds a two-level +// prelude, and lays out [prx span][prelude][dd-block][freq-block] CONTIGUOUSLY +// in the single posting region (prx span first, then the frq span). Sets +// enc=windowed + has_sb. frq_docs_len = prelude_len + dd_block_len is the +// contiguous docs-only prefix, which stays INSIDE the frq span. +Status LogicalIndexWriter::build_windowed_entry(TermPostings& tp, uint64_t frq_base, + uint64_t prx_base, bool write_prx, bool write_freq, + DictEntry* e) { + // The prx span starts here: pass 1 streams each .prx window straight into + // the posting sink, so prx_off_delta is measured against the live + // posting-sink size. write_prx == false (G01 docs-only bigram) streams no + // prx bytes at all and leaves the entry's prx locator zeroed; the per-term + // prelude records has_prx=false so the layout stays self-describing. + // write_freq == false (G16 docs-only bigram) additionally drops the + // freq-block and the per-window freq locators/crcs from the prelude; the + // prelude flags record has_freq=false, so frq_len == frq_docs_len and a + // want_freq read of such an entry fails closed in region decode. + const uint64_t prx_off = posting_size(); + // G16-e: docs-only (prune-mode) bigrams use much larger windows -- their + // 2-term hit path reads the full docs-only prefix (never window-narrowed), + // and bigger dd regions give the per-region zstd a real context. Legacy + // bigrams (write_prx) keep the adaptive sizing (byte-identical output). + const uint32_t window_docs = + (!write_prx && format::is_phrase_bigram_term(tp.term)) ? format::kBigramWindowDocs : 0; + WindowedPosting wp; + RETURN_IF_ERROR(BuildWindowedPosting(tp, write_freq, write_prx, encoded_norms_, posting_out_, + &wp, window_docs, prx_zstd_level_)); + // wp.prx_total_len bytes were just streamed straight to the posting sink (0 + // when !has_prx). docids/freqs are now fully encoded into wp; release the + // source arrays before the (potentially large) wp blocks are appended to + // disk. + std::vector().swap(tp.docids); + std::vector().swap(tp.freqs); + std::vector prelude; + RETURN_IF_ERROR(BuildPrelude(wp.windows, write_freq, write_prx, &prelude)); + + e->kind = DictEntryKind::kPodRef; + e->enc = DictEntryEnc::kWindowed; + e->has_sb = true; // prelude is always a two-level skip directory. + e->prelude_len = static_cast(prelude.size()); + e->frq_docs_len = + e->prelude_len + static_cast(wp.dd_block.size()); // [prelude][dd-block] + + // The frq span starts immediately AFTER the prx span, in the SAME sink. The + // writer-side property frq_off == prx_off + wp.prx_total_len holds because + // nothing is appended to the posting sink between the prx pass and here -- + // but the delta is measured from the live size, not assumed. + const uint64_t frq_off = posting_size(); + RETURN_IF_ERROR(posting_out_->append(Slice(prelude))); + RETURN_IF_ERROR(posting_out_->append(Slice(wp.dd_block))); + RETURN_IF_ERROR(posting_out_->append(Slice(wp.freq_block))); + e->frq_off_delta = frq_off - frq_base; + e->frq_len = posting_size() - frq_off; + if (write_prx) { + e->prx_off_delta = prx_off - prx_base; + e->prx_len = wp.prx_total_len; // == frq_off - prx_off + } + return Status::OK(); +} + +// Emits a slim term as a single .frq window (win_base=0) laid out [dd][freq]: +// inline when the encoded bytes are tiny, otherwise a slim pod_ref (no +// prelude). The dd region is the docs-only prefix; the freq region (when +// has_freq) is the skippable suffix. Region codecs are recorded in the +// DictEntry. For a pod_ref, the term's [prx][frq] spans are appended to the +// single posting region with the prx span FIRST (consistent with the windowed +// path); the reader resolves each delta independently so the relative order is +// not load-bearing. +Status LogicalIndexWriter::build_slim_entry(TermPostings& tp, uint64_t frq_base, uint64_t prx_base, + bool write_prx, DictEntry* e) { + std::vector dd_bytes, freq_bytes; + FrqRegionMeta dd_meta, freq_meta; + RETURN_IF_ERROR(EncodeRegions(tp.docids, tp.freqs, /*win_base=*/0, has_freq_, &dd_bytes, + &dd_meta, &freq_bytes, &freq_meta)); + std::vector frq_win = dd_bytes; // [dd_region][freq_region] + AppendBytes(&frq_win, freq_bytes); + std::vector prx_win; + if (write_prx) { + RETURN_IF_ERROR(MakePrxWindow(tp.positions_flat, tp.freqs, prx_zstd_level_, &prx_win)); + } + + e->enc = DictEntryEnc::kSlim; + e->dd_meta = dd_meta; + e->freq_meta = freq_meta; + + if (frq_win.size() <= format::kDefaultInlineThreshold) { + e->kind = DictEntryKind::kInline; + e->inline_dd_disk_len = dd_meta.disk_len; + e->frq_bytes = std::move(frq_win); + // !write_prx (G01 docs-only bigram) leaves prx_bytes empty; the T2 entry + // encoding round-trips prx_len == 0 fine. + if (write_prx) e->prx_bytes = std::move(prx_win); + return Status::OK(); + } + + // POD_REF: write [prx][frq] into the single posting sink, prx span first. + e->kind = DictEntryKind::kPodRef; + e->frq_docs_len = dd_meta.disk_len; // docs-only prefix = the single dd region + if (write_prx) { + const uint64_t prx_off = posting_size(); + RETURN_IF_ERROR(posting_out_->append(Slice(prx_win))); + e->prx_off_delta = prx_off - prx_base; + e->prx_len = posting_size() - prx_off; + } + const uint64_t frq_off = posting_size(); // immediately after the prx span + RETURN_IF_ERROR(posting_out_->append(Slice(frq_win))); + e->frq_off_delta = frq_off - frq_base; + e->frq_len = posting_size() - frq_off; + return Status::OK(); +} + +// Builds the DictEntry for one term. Inline entries embed their .frq/.prx +// bytes; pod_ref entries append [prx][frq] bytes to the single posting region +// and record off_delta relative to frq_base/prx_base (the posting-region size +// captured when the block opened; both bases hold that same value). +Status LogicalIndexWriter::build_entry(TermPostings& tp, uint64_t frq_base, uint64_t prx_base, + const FreqStats& fs, bool write_prx, bool write_freq, + DictEntry* e) { + e->term = tp.term; + e->df = static_cast(tp.docids.size()); + e->ttf_delta = fs.total_freq; // reused fused total (was SumOf(tp.freqs)) + e->max_freq = fs.max_freq; // reused fused max (was MaxOf(tp.freqs)) + + if (e->df >= format::kSlimDfThreshold) { + if (has_freq_ && !write_freq) testing::note_bigram_freq_elided(); + RETURN_IF_ERROR(build_windowed_entry(tp, frq_base, prx_base, write_prx, write_freq, e)); + } else { + // On a freq-WRITING index (has_freq_), slim/inline entries always keep + // the freq region: their DictEntry region metadata is tier-conditioned + // (freq meta fields present iff tier>=T2), so a per-TERM drop is not + // flag-describable there -- only the windowed prelude (flags bit0) is. + // A per-INDEX drop (G16-c write_freq=false -> has_freq_=false) removes + // freq for ALL shapes value-driven: slim/inline then carry a + // zero-length freq region (frq span == docs-only prefix). + RETURN_IF_ERROR(build_slim_entry(tp, frq_base, prx_base, write_prx, e)); + } + // G16 section accounting from the finished entry's own fields (no plumbing): + // windowed: frq span = [prelude][dd][freq]; slim pod: [dd][freq], prelude 0; + // inline: bytes live in frq_bytes/prx_bytes (also counted in dict bytes). + const size_t cls = format::is_phrase_bigram_term(tp.term) ? 1 : 0; + section_stats_.entries[cls]++; + if (e->kind == DictEntryKind::kInline) { + section_stats_.dd_bytes[cls] += e->inline_dd_disk_len; + section_stats_.freq_bytes[cls] += e->frq_bytes.size() - e->inline_dd_disk_len; + section_stats_.prx_bytes[cls] += e->prx_bytes.size(); + } else { + section_stats_.prelude_bytes[cls] += e->prelude_len; + section_stats_.dd_bytes[cls] += e->frq_docs_len - e->prelude_len; + section_stats_.freq_bytes[cls] += e->frq_len - e->frq_docs_len; + section_stats_.prx_bytes[cls] += e->prx_len; + } + return Status::OK(); +} + +// Serializes the current open block, zstd-compresses it (the dict region is the +// single largest section -- term keys + entry meta + inline postings -- and the +// 64KiB blocks compress ~40%), streams the compressed bytes into the dict +// scratch file, and records a directory entry. The block-level crc32c +// (rec.checksum) covers the UNCOMPRESSED bytes, so DictBlockReader::open +// verifies integrity after the reader decompresses. A compressed block also +// shrinks the bytes a term lookup fetches from S3 -- aligning with the +// read-byte thesis. If zstd does not shrink a (tiny) block, it is stored raw so +// a lookup never pays a pointless decompress. +Status LogicalIndexWriter::flush_block(DictBlockBuilder* block, std::string first_term) { + ByteSink bsink; + block->finish(&bsink); + const Slice plain = bsink.view(); + BlockRecord rec; + rec.rel_offset = dict_buf_.size(); + rec.n_entries = block->n_entries(); + rec.checksum = crc32c(plain); // crc over UNCOMPRESSED block bytes + rec.first_term = std::move(first_term); + + section_stats_.dict_entry_bytes[0] += block->entry_bytes_total() - block->entry_bytes_bigram(); + section_stats_.dict_entry_bytes[1] += block->entry_bytes_bigram(); + + std::vector comp; + Status zs = zstd_compress(plain, dict_block_zstd_level_, &comp); + if (zs.ok() && comp.size() < plain.size()) { + rec.flags = format::block_ref_flags::kZstd; + rec.uncomp_len = static_cast(plain.size()); + rec.length = static_cast(comp.size()); + RETURN_IF_ERROR(dict_buf_.append_move(std::move(comp))); + } else { + rec.flags = 0; + rec.uncomp_len = 0; + rec.length = static_cast(plain.size()); + RETURN_IF_ERROR(dict_buf_.append_move(bsink.take())); + } + blocks_.push_back(std::move(rec)); + return Status::OK(); +} + +// Running state for the in-flight DICT block while terms stream past. +struct LogicalIndexWriter::BlockState { + std::unique_ptr block; + std::string block_first_term; + uint64_t frq_base = 0; + uint64_t prx_base = 0; +}; + +Status LogicalIndexWriter::process_term(TermPostings& tp, BlockState* st) { + // G01 bigram diet: the materialize/prune decision for hidden phrase-bigram + // terms happens HERE -- the single choke point both drain paths (streaming + // for_each_term_sorted and the materialized vector) funnel through, and the + // first point where the term's FINAL df (tp.docids.size(), post-merge) is + // known. A pruned term is dropped before ANY materialization side effect: + // no dict entry, no posting bytes, no bloom membership, no term_count/stats + // contribution. The sentinel term is never prunable (it flags "bigram + // feature present" for legacy reader semantics). + const bool is_bigram = format::is_phrase_bigram_term(tp.term); + if (is_bigram && !format::is_phrase_bigram_sentinel_term(tp.term)) { + if (bigram_prune_min_df_ > 0 && tp.docids.size() < bigram_prune_min_df_) { + testing::note_bigram_term_pruned(); + return Status::OK(); + } + // G15: symmetric UPPER gate. A near-ubiquitous pair (final df above + // ratio * doc_count, resolved by the caller) pays dict + posting bytes + // for almost no selectivity, so it is dropped to the SAME dict-miss + // fallback. Unlike a min-df miss, the fallback for a pruned-HIGH pair + // verifies positions over a LARGE unigram candidate set -- the ratio + // must stay high enough that only stopword-like pairs cross it. Applied + // ONLY here (never in the mid-feed drain): df grows monotonically, but + // so does the doc-count-scaled threshold, so a partial df exceeding a + // partial threshold proves nothing about the final comparison. + if (bigram_prune_max_df_ > 0 && tp.docids.size() > bigram_prune_max_df_) { + testing::note_bigram_term_max_pruned(); + return Status::OK(); + } + // G04: drop a df-surviving bigram the ever-dropped bloom MAY contain -- + // the vocab-cap eviction dropped (at least) one of its docids mid-build, + // so the postings here are INCOMPLETE; materializing them would silently + // lose phrase hits, while dropping routes the pair to the G01 dict-miss + // fallback (bigram_ever_dropped_ is non-null only when the meta declares + // pruning). A bloom false positive lands here too: pure over-drop, safe. + if (bigram_ever_dropped_ != nullptr && bigram_ever_dropped_->maybe_contains(tp.term)) { + testing::note_bigram_bloom_dropped(); + return Status::OK(); + } + testing::note_bigram_term_materialized(); + } + // Surviving bigram postings are DOCS+FREQ ONLY in prune mode: the 2-term + // phrase bigram hit path answers from docid membership alone (it never reads + // bigram positions -- phrase_prefix multi-tail uses unigram prx and filters + // bigram tails), so their .prx spans are pure dead bytes. Legacy mode + // (threshold 0) keeps writing positions, byte-identical to pre-G01 output. + const bool write_prx = has_prx_ && !(is_bigram && bigram_prune_min_df_ > 0); + // G16: prune-mode bigram postings also drop their freq region. NO query + // path reads pair freqs (the 2-term hit path answers from df/docids, the + // 3+-term chain reads docids only, and scoring can never target a hidden + // marker term), so the bytes are write-only. Same legacy escape hatch as + // write_prx: threshold 0 keeps the byte-identical pre-G01 layout. Applied + // on the WINDOWED path only -- see build_entry for the slim-format + // constraint. + const bool write_freq = has_freq_ && !(is_bigram && bigram_prune_min_df_ > 0); + + // ONE fused term-level scan of freqs: total + max in a single pass, reused by + // validate (position-count budget), stats, and the DictEntry + // ttf_delta/max_freq. Computed BEFORE validate_term so the validator receives + // the budget total instead of re-summing. The position-count check is gated + // on write_prx (== has_prx_ for every term except prune-mode bigrams, whose + // positions the G04 diet stopped buffering). + const FreqStats fs = fuse_freq_stats(tp.freqs); + RETURN_IF_ERROR(validate_term(tp, fs.total_freq, write_prx)); + // Collect only the 8-byte filter key per term (no whole-vocabulary string + // copy). BSBF key = XXH64 seed 0 (Parquet-canonical). + term_hashes_.push_back(format::bsbf_hash(tp.term)); + ++term_count_; + stats_.sum_total_term_freq += fs.total_freq; // reused fused total (was SumOf) + + if (!st->block) { + // Both bases come from the SAME posting sink, snapshotted at block open. + const uint64_t base = posting_size(); + st->frq_base = base; + st->prx_base = base; + // G16-f: a freq-dropped index also omits the per-entry ttf/max_freq + // stats (BM25-only); the block header flag makes it self-describing. + st->block = std::make_unique(tier_, has_prx_, st->frq_base, st->prx_base, + /*anchor_interval=*/16, + /*term_stats=*/has_freq_); + st->block_first_term = tp.term; + } + + DictEntry e; + RETURN_IF_ERROR(build_entry(tp, st->frq_base, st->prx_base, fs, write_prx, write_freq, &e)); + // `e` is not used after this point, so move it into the block to avoid a + // per-term DictEntry copy (two vector heap allocations for inline entries). + st->block->add_entry(std::move(e)); + + if (st->block->estimated_bytes() >= target_dict_block_bytes_) { + RETURN_IF_ERROR(flush_block(st->block.get(), st->block_first_term)); + st->block.reset(); + } + return Status::OK(); +} + +Status LogicalIndexWriter::build_blocks() { + BlockState st; + if (term_source_ != nullptr) { + // G06: sink the flush-time phrase-bigram df gate into the buffer drain. + // bigram_prune_min_df_ is the SAME effective threshold process_term + // gates with below (already forced to 0 for non-positional configs and + // in legacy no-prune mode, which disables the gate), so a pair-keyed + // bigram term the drain drops -- provably-exact df below the gate -- is + // exactly a term process_term would have pruned AFTER paying its + // composed-string materialization, chain decode and emission. + // process_term keeps gating everything that still reaches it + // (string-keyed terms, spill-materialized pair terms whose df only the + // merge can total, out-of-order-fed pair terms). + term_source_->set_bigram_drain_min_df(bigram_prune_min_df_); + Status streamed = Status::OK(); + // Drain the SPIMI buffer term-by-term; only one TermPostings is alive at a + // time, so the input+output never fully coexist. The returned Status covers + // both spill/merge I/O errors and add_token validation errors (the latter + // flow through merge_runs -> spill_status_), so a separate status() check + // is no longer needed. + RETURN_IF_ERROR(term_source_->for_each_term_sorted([&](TermPostings&& tp) { + if (streamed.ok()) streamed = process_term(tp, &st); + })); + RETURN_IF_ERROR(streamed); + } else { + // Materialized fallback (tests / callers holding a vector): process_term + // frees the term's arrays, so feed a per-term COPY to keep terms_ intact + // for the caller. This path is not the large out-of-core build, so the copy + // is cheap. + for (const auto& tp : terms_) { + TermPostings copy = tp; + RETURN_IF_ERROR(process_term(copy, &st)); + } + } + if (st.block) RETURN_IF_ERROR(flush_block(st.block.get(), st.block_first_term)); + return Status::OK(); +} + +Status LogicalIndexWriter::build(io::FileWriter* posting_out) { + if (posting_out == nullptr) { + return Status::Error( + "logical_index: null posting sink"); + } + if (has_norms_ && encoded_norms_.size() != doc_count_) { + return Status::Error( + "logical_index: norms length must equal doc_count"); + } + // The interleaved posting region streams STRAIGHT into the container output + // (no temp round-trip): posting_size() is the region-relative byte count, + // derived from the output offset advanced since this index's region began. + // The DICT region is staged in dict_buf_ (tiered: RAM under the cap = + // spill-only; spills above it) since it must land contiguously after the + // concurrently-streamed posting region. + posting_out_ = posting_out; + posting_off0_ = posting_out->bytes_written(); + + RETURN_IF_ERROR(build_blocks()); + // Seal the dict buffer so a spilled temp is flushed before + // stream_dict_region_into reads it back. A no-op for a RAM-resident dict. + RETURN_IF_ERROR(dict_buf_.seal()); + + stats_.doc_count = doc_count_; + stats_.indexed_doc_count = doc_count_ - static_cast(null_docids_.size()); + stats_.term_count = term_count_; + stats_.null_count = static_cast(null_docids_.size()); + + if (has_norms_) { + format::NormsPodWriter nw; + for (uint8_t n : encoded_norms_) nw.add(n); + ByteSink nsink; + nw.finish(&nsink); + norms_section_ = nsink.take(); + } + + if (!null_docids_.empty()) { + format::NullBitmapWriter null_writer; + for (uint32_t docid : null_docids_) null_writer.add_null(docid); + ByteSink null_sink; + null_writer.finish(doc_count_, &null_sink); + null_bitmap_section_ = null_sink.take(); + } + + // Build the absent-term filter (block-split bloom, Parquet-canonical) from + // the per-term keys (no retained strings) as a [28B header][bitset] blob; the + // compound writer places it as a PHYSICAL section probed one 32-byte block on + // demand. + bsbf_bytes_.clear(); + if (!term_hashes_.empty()) { + format::BsbfBuilder bf; + RETURN_IF_ERROR(format::BsbfBuilder::create(static_cast(term_hashes_.size()), + kBsbfFpp, &bf)); + for (uint64_t k : term_hashes_) bf.insert(k); + ByteSink bsink; + RETURN_IF_ERROR(bf.serialize(&bsink)); + bsbf_bytes_ = bsink.take(); + } + std::vector().swap(term_hashes_); // release + + const auto& ss = section_stats_; + LOG(INFO) << "SNII section stats idx=" << index_id_ << " suffix=" << index_suffix_ + << " uni{n=" << ss.entries[0] << " dd=" << ss.dd_bytes[0] + << " freq=" << ss.freq_bytes[0] << " prx=" << ss.prx_bytes[0] + << " prelude=" << ss.prelude_bytes[0] << " dict=" << ss.dict_entry_bytes[0] + << "} bg{n=" << ss.entries[1] << " dd=" << ss.dd_bytes[1] + << " freq=" << ss.freq_bytes[1] << " prx=" << ss.prx_bytes[1] + << " prelude=" << ss.prelude_bytes[1] << " dict=" << ss.dict_entry_bytes[1] + << "} dict_region=" << dict_buf_.size() << " doc_count=" << doc_count_; + return Status::OK(); +} + +Status LogicalIndexWriter::finish_meta(const SectionRefs& abs_refs, uint64_t dict_region_offset, + ByteSink* out) const { + if (out == nullptr) + return Status::Error("logical_index: null meta sink"); + + SampledTermIndexBuilder sti; + for (const auto& b : blocks_) sti.add_block_first_term(b.first_term); + ByteSink sti_sink; + sti.finish(&sti_sink); + + DictBlockDirectoryBuilder dir; + for (const auto& b : blocks_) { + BlockRef ref; + ref.offset = dict_region_offset + b.rel_offset; + ref.length = b.length; + ref.n_entries = b.n_entries; + ref.flags = b.flags; + ref.checksum = b.checksum; + ref.uncomp_len = b.uncomp_len; + dir.add(ref); + } + ByteSink dir_sink; + dir.finish(&dir_sink); + + uint32_t flags = bsbf_bytes_.empty() ? 0u : PerIndexMetaBuilder::kHasBsbf; + // Persist positions capability explicitly (the R1 fix): the reader must NOT + // infer it from posting_region.length, which is non-zero for any docs-only + // pod_ref index. + if (has_prx_) { + flags |= PerIndexMetaBuilder::kHasPositions; + } + if (phrase_bigrams_deferred_) { + flags |= PerIndexMetaBuilder::kPhraseBigramsDeferred; + } + PerIndexMetaBuilder builder(index_id_, index_suffix_, flags); + builder.set_stats(stats_); + builder.set_sampled_term_index(sti_sink.view()); + builder.set_dict_block_directory(dir_sink.view()); + // The BSBF is a physical section (abs_refs.bsbf), not embedded in the meta. + builder.set_section_refs(abs_refs); + // G01/G15: declare the APPLIED bigram df-prune thresholds (both 0 emits + // nothing -- legacy segments stay byte-identical) so the reader's 2-term + // phrase path knows a bigram dict miss means "fall back", not "empty". + builder.set_bigram_prune_min_df(bigram_prune_min_df_); + builder.set_bigram_prune_max_df(bigram_prune_max_df_); + return builder.finish(out); +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/logical_index_writer.h b/be/src/storage/index/snii/writer/logical_index_writer.h new file mode 100644 index 00000000000000..3ba31c2c2b2b49 --- /dev/null +++ b/be/src/storage/index/snii/writer/logical_index_writer.h @@ -0,0 +1,435 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/dict_block.h" +#include "storage/index/snii/format/dict_block_directory.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/per_index_meta.h" +#include "storage/index/snii/format/sampled_term_index.h" +#include "storage/index/snii/format/stats_block.h" +#include "storage/index/snii/io/file_writer.h" +#include "storage/index/snii/writer/memory_reporter.h" +#include "storage/index/snii/writer/spillable_byte_buffer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +// LogicalIndexWriter -- builds the per-logical-index section bytes (interleaved +// posting region + DICT block region) and the meta sub-sections (SampledTermIndex, +// DICT block directory, StatsBlock, XFilter) for ONE logical index. It owns the +// in-memory section bytes and the metadata needed by the container orchestrator +// (SniiCompoundWriter) to resolve absolute offsets and emit the per-index meta +// block. +// +// This module deliberately produces ONLY relative bytes/structures: it has no +// knowledge of the absolute file position where the sections will land. The +// orchestrator stitches the absolute offsets in afterward (append-only, no +// seek-back). See snii_compound_writer.h for the precise offset contract. +// +// POSTING REGION (single interleaved sink): the former separate .frq POD and .prx +// POD are merged into ONE posting region. For each pod_ref term, in term order, the +// writer appends its prx span FIRST then its frq span, contiguously: +// posting region = concat over pod_ref terms of [prx span][frq span]. +// The prx span is empty when !has_prx (docs-only / keyword tier). INLINE terms +// append NOTHING to the posting region. +// +// Per-term encoding policy (v1): +// df >= kSlimDfThreshold (512): WINDOWED pod_ref. The term's [prx windows] are +// appended to the posting region first, then its [prelude][dd-block][freq-block] +// frq span. The DictEntry records frq/prx off_delta+len relative to +// frq_base/prx_base (see below). +// df < kSlimDfThreshold: SLIM. The postings are encoded as a single .frq +// window (and .prx window). If the encoded .frq bytes are small +// (<= kDefaultInlineThreshold), they are stored INLINE inside the DictEntry +// (kind=inline); otherwise the term's [prx][frq] spans are appended to the +// posting region as a slim pod_ref (kind=pod_ref, enc=slim, no prelude). +// +// frq_base / prx_base convention (DOCUMENTED CONTRACT): +// For each DICT block, frq_base == prx_base == the running byte offset into THIS +// index's posting region at the moment the block opens (the posting-region size +// when the block's first POD-backed entry is appended). A windowed/slim pod_ref +// entry then sets frq_off_delta = (offset of its frq span within the posting +// region) - frq_base, so the reader computes the absolute file offset as +// section_refs.posting_region.offset + frq_base + frq_off_delta. +// prx_base / prx_off_delta follow the identical rule against the SAME region. +// Because [prx][frq] are written contiguously per term, a writer-side property +// holds when has_prx: frq_off_delta == prx_off_delta + prx_len. The reader does +// NOT rely on it -- each delta is resolved independently. +// Inline entries carry no off_delta (bytes live in the entry). +namespace doris::snii::writer { + +// Inputs describing one logical index to be written. +struct SniiIndexInput { + uint64_t index_id = 0; + std::string index_suffix; + format::IndexConfig config = format::IndexConfig::kDocsPositions; + uint32_t doc_count = 0; + std::vector null_docids; + // Per-doc 1-byte encoded norm (length doc_count); only consumed when the + // config has scoring. May be empty otherwise. + std::vector encoded_norms; + // G16-h: zstd levels for the dict-block whole-block compression and the + // .prx window auto mode (both default 3 == the historical constants). + // Higher levels trade import CPU for size; decode speed is unaffected. + int dict_block_zstd_level = 3; + int prx_zstd_level = 3; + // G16-c: whether freq-capable (tier>=T2) postings lay out freq regions at + // all. Freq bytes serve ONLY BM25 scoring (want_freq=true lives solely in + // scoring_query), so the CALLER resolves the policy -- the Doris adapter + // passes has_scoring(config) || config::snii_positions_index_write_freq, + // i.e. plain kDocsPositions indexes drop freq unless the escape hatch is + // set. Defaults to true so the core library and existing callers keep the + // full T2 layout unless they opt out. The drop is value-driven on disk + // (windowed prelude flags bit0; slim/inline zero-length freq regions), so + // readers need no index-level flag. Ignored for docs-only configs. + bool write_freq = true; + // Lexicographically sorted terms with ascending-docid postings. Used when + // `term_source` is null (callers that already hold a materialized vector, + // e.g. unit tests). The writer reads but does not retain these. + std::vector terms; + // Optional streaming term source. When non-null, the writer DRAINS it via + // SpimiTermBuffer::for_each_term_sorted so that only one term's postings is + // materialized at a time (avoiding the full TermPostings vector and its + // second-copy peak). `terms` is ignored when this is set. The buffer is + // consumed (emptied) by build(); the caller must keep it alive until build() + // returns and must not reuse it afterwards. + SpimiTermBuffer* term_source = nullptr; + // Target DICT block size in bytes; a block is cut once its estimate reaches + // this. 0 uses kDefaultTargetDictBlockBytes. Smaller values yield more blocks + // (and a finer-grained sampled-term index). + uint32_t target_dict_block_bytes = 0; + // EFFECTIVE phrase-bigram df-prune threshold (G01). 0 (default) == legacy: + // every term materializes as before. > 0 (positions-capable configs only; + // ignored otherwise): a hidden phrase-bigram term (marker prefix, sentinel + // excluded) whose FINAL df < this is skipped entirely at materialization (no + // dict entry, no postings, no bloom membership, no stats), and every + // SURVIVING bigram term writes its posting docs+freq-only (no .prx) even in + // kDocsPositions config. The applied value is recorded in the per-index meta + // (kBigramPruneInfo) so the reader's 2-term phrase path knows a bigram dict + // miss must fall back to positions verification. The caller (Doris adapter) + // resolves config::snii_bigram_prune_min_df + the doc-count formula into + // this field; the core writer just applies it. + uint32_t bigram_prune_min_df = 0; + // EFFECTIVE phrase-bigram df-prune UPPER bound (G15), an ABSOLUTE doc + // count. 0 (default) == no upper gate. > 0 (positions-capable configs only; + // ignored otherwise): a hidden phrase-bigram term (sentinel excluded) whose + // FINAL df EXCEEDS this is skipped at materialization exactly like a + // min-df-pruned term -- near-ubiquitous pairs pay dict + posting bytes for + // almost no selectivity. Recorded in kBigramPruneInfo alongside the min + // threshold, so the reader falls back on a dict miss when EITHER gate is + // declared. The caller (Doris adapter) resolves + // config::snii_bigram_prune_max_df_ratio * doc_count (floored at + // 2 * bigram_prune_min_df when the min gate is active) into this field; the + // core writer just applies it. Unlike the min gate it is NEVER sunk into + // the mid-feed drain: a partial df above a threshold derived from the + // still-growing doc count proves nothing about the final df/threshold pair. + uint64_t bigram_prune_max_df = 0; + // Fresh direct-load segment deliberately omits every hidden phrase-bigram + // posting and the sentinel. Persisted as resident per-index metadata so the + // reader avoids probing an impossible pair before positions verification. + bool phrase_bigrams_deferred = false; + // G04: EVER-DROPPED bloom over bigram terms the SPIMI vocab-cap eviction + // dropped mid-build (SpimiTermBuffer::bigram_dropped_filter(); the caller + // wires it). When non-null AND pruning is active (bigram_prune_min_df > 0), + // process_term drops any non-sentinel bigram term the filter may contain IN + // ADDITION to the df threshold: an evicted-then-reappearing pair's + // re-accumulated postings are INCOMPLETE (they miss the docids dropped at + // eviction), so materializing them would silently lose phrase hits; dropping + // them instead routes the pair to the G01 dict-miss fallback (correct, just + // slower). Bloom false positives only over-drop -- safe under the same + // contract. Ignored (with pruning forced off) when bigram_prune_min_df == 0: + // a bloom drop on a segment whose meta declares NO pruning would break the + // legacy miss==empty reader semantics. Borrowed; null when no eviction fired. + const BigramDropFilter* bigram_ever_dropped = nullptr; + // Optional writer-level build-RAM reporter (one per SniiCompoundWriter = one + // segment inverted index). When non-null, the dict buffer reports its REAL + // resident-byte deltas (positive on grow, negative on spill). The SPIMI side + // (arena + slot index) reports through the SAME reporter, injected directly at + // the term_source's construction by the caller. null in bench / unit tests -> no + // reporting. NEVER report live_bytes_ (a gated estimate); report + // arena_bytes()+slot_of_+dict ram_bytes_. + MemoryReporter* mem_reporter = nullptr; +}; + +// Fused single-pass term-level frequency statistics for ONE term: total_freq is +// sum(freqs) and max_freq is max(freqs), computed in a SINGLE scan (see +// fuse_freq_stats) and reused for the has_prx position-count check +// (validate_term), stats_.sum_total_term_freq, and the DictEntry +// ttf_delta/max_freq. Byte-identical to the former separate sum/max scans (same +// accumulation order; max initialized to 0 so a freq of 0 never pollutes it). +struct FreqStats { + uint64_t total_freq = 0; + uint32_t max_freq = 0; +}; + +// Builds and holds the section bytes + meta sub-sections for one logical index. +class LogicalIndexWriter { +public: + explicit LogicalIndexWriter(const SniiIndexInput& in); + + // Builds DICT blocks, the interleaved posting region, sampled-term index, dict + // directory, stats and bsbf. The posting region is written STRAIGHT into + // `posting_out` as terms are produced (no temp round-trip for the bulk); the + // orchestrator captures its absolute offset/length from posting_out->bytes_written() + // around this call. Must be called once before the accessors below. Returns + // InvalidArgument on a null sink or inconsistent input (e.g. norms/doc_count + // mismatch when scoring is enabled, or non-ascending docids). + Status build(io::FileWriter* posting_out); + + // DICT region byte length (relative; orchestrator decides its absolute offset). The + // DICT region (zstd-compressed blocks) is built into a tiered buffer during build() + // -- it must land contiguously AFTER the posting region (streamed concurrently), so + // it cannot stream directly. The buffer stays in RAM while small (spill-only build) + // and spills to a temp once it crosses the RAM cap (bounded peak RSS for a huge + // dict). Its bytes are emitted via stream_dict_region_into below. The posting region + // went straight to the output during build(), so it has no length accessor here -- + // the orchestrator measures it directly. norms stays in RAM (1 byte/doc). + uint64_t dict_region_size() const { return dict_buf_.size(); } + const std::vector& norms_bytes() const { return norms_section_; } + const std::vector& null_bitmap_bytes() const { return null_bitmap_section_; } + // Block-split bloom XFilter blob ([28B header][bitset]); empty when no terms. + const std::vector& bsbf_bytes() const { return bsbf_bytes_; } + bool has_bsbf() const { return !bsbf_bytes_.empty(); } + bool has_null_bitmap() const { return !null_bitmap_section_.empty(); } + + // Streams the DICT region (RAM or spilled temp) into the append-only container + // after its posting region. + Status stream_dict_region_into(io::FileWriter* out) const { return dict_buf_.stream_into(out); } + + bool has_prx() const { return has_prx_; } + bool has_norms() const { return has_norms_; } + format::IndexTier tier() const { return tier_; } + uint64_t index_id() const { return index_id_; } + const std::string& index_suffix() const { return index_suffix_; } + + // Builds the per-index meta block bytes given the resolved ABSOLUTE section + // refs (filled by the orchestrator), appending them to out. The DICT block + // directory entries are rebased to absolute offsets using dict_region_offset. + Status finish_meta(const format::SectionRefs& abs_refs, uint64_t dict_region_offset, + ByteSink* out) const; + +private: + // One DICT block's directory record. The block's serialized bytes are appended to + // the in-RAM dict buffer as soon as the block is cut; only this compact summary + // (offset within the dict region + length + entry count + checksum) is kept to + // build the DICT block directory at finish_meta time. The absolute file offset is + // computed as dict_region_offset + rel_offset. + struct BlockRecord { + uint64_t rel_offset = 0; // byte offset of this block within the dict region + uint64_t length = 0; // ON-DISK block length (compressed when flags&kZstd) + uint32_t n_entries = 0; + uint32_t checksum = 0; // crc32c of the UNCOMPRESSED block bytes + uint8_t flags = 0; // block_ref_flags::* (kZstd when block is compressed) + uint64_t uncomp_len = 0; // uncompressed block length (when flags&kZstd) + std::string first_term; + }; + + // Validates one term's shape (parallel lengths, strictly ascending docids). + // `total_freq` is the fused sum(freqs) the caller computes once via + // fuse_freq_stats; the position-count check compares against it instead of + // re-summing freqs internally. `expect_positions` is the PER-TERM positions + // switch (the caller's write_prx): a G04 position-suppressed bigram term + // legitimately arrives with an empty positions_flat while freqs sum > 0, and + // its .prx is never written -- so the count check is skipped exactly when the + // term writes no positions. For every other term (and all terms in legacy + // mode) expect_positions == has_prx_, preserving the strict check. + Status validate_term(const TermPostings& tp, uint64_t total_freq, bool expect_positions) const; + // Iterates terms (from the streaming source or the materialized vector), + // splitting DICT blocks by target size and filling PODs + blocks_. + Status build_blocks(); + // Per-term driver shared by both the streaming and materialized paths: + // validates the term, opens a block if needed, builds its DictEntry, and cuts + // the block once it reaches the target size. Mutates the running block state. + struct BlockState; + // `tp` is taken by mutable reference: the encode FREES the term's large flat + // arrays (docids/freqs/positions_flat) as soon as they are consumed, so the + // widest term's source does not co-exist with its encoded output at peak RSS. + Status process_term(TermPostings& tp, BlockState* st); + // Region-relative byte count of the posting bytes written so far (the offset basis + // for frq_base/prx_base + frq_off_delta/prx_off_delta). During build() the only + // writes to posting_out_ are this index's posting region, so the count is the + // output offset advanced since the region began. + uint64_t posting_size() const { return posting_out_->bytes_written() - posting_off0_; } + // Builds one DictEntry (inline or pod_ref), growing the posting region as + // needed. `fs` is the fused term-level freq stats (reused for ttf_delta / + // max_freq, so no second sum/max scan). `write_prx` is the PER-TERM positions + // switch: has_prx_ for normal terms, false for phrase-bigram terms in + // prune mode (G01 docs-only bigrams) -- the entry then carries no .prx span + // (prx_len == 0) while the dd/freq regions stay identical. `write_freq` is + // the PER-TERM freq switch (G16): has_freq_ for normal terms, false for + // prune-mode bigrams, whose freq bytes no query path ever reads. Honored on + // the WINDOWED path only (the prelude flags self-describe freq presence); + // slim/inline entries keep freq because their DictEntry region metadata is + // tier-conditioned, not per-entry. + Status build_entry(TermPostings& tp, uint64_t frq_base, uint64_t prx_base, const FreqStats& fs, + bool write_prx, bool write_freq, format::DictEntry* e); + // Builds a windowed (df >= kSlimDfThreshold) entry: multi-window + two-level + // prelude. The term's [prx span][frq span] is appended to the posting region + // (prx span empty when !write_prx, freq-block empty when !write_freq; the + // per-term prelude is self-describing). + Status build_windowed_entry(TermPostings& tp, uint64_t frq_base, uint64_t prx_base, + bool write_prx, bool write_freq, format::DictEntry* e); + // Builds a slim (df < kSlimDfThreshold) entry: single window, inline or + // pod_ref, no prelude. + Status build_slim_entry(TermPostings& tp, uint64_t frq_base, uint64_t prx_base, bool write_prx, + format::DictEntry* e); + // Serializes the current open block, streams its bytes into the dict scratch + // file, and records a compact directory entry (no block bytes retained). + Status flush_block(format::DictBlockBuilder* block, std::string first_term); + + uint64_t index_id_; + std::string index_suffix_; + format::IndexTier tier_; + bool has_prx_; + bool has_freq_; // tier >= T2: a freq region is encoded per window + bool has_norms_; + // G01 effective bigram df-prune threshold (0 == off/legacy). Forced to 0 for + // non-positional configs (no bigrams are ever emitted there). Recorded into + // the per-index meta by finish_meta when non-zero. + uint32_t bigram_prune_min_df_; + // G15 effective bigram df-prune UPPER bound, absolute (0 == no upper gate). + // Forced to 0 for non-positional configs like the min threshold. Recorded + // into the per-index meta by finish_meta alongside it. + uint64_t bigram_prune_max_df_; + // The input's fresh-segment capability flag. Full builds and every existing + // segment keep this false, preserving their hidden-bigram fast path. + bool phrase_bigrams_deferred_; + // G04 ever-dropped bloom (borrowed from SniiIndexInput). Non-null ONLY when + // pruning is active (see the SniiIndexInput field contract): probed once per + // df-surviving bigram term in process_term. + const BigramDropFilter* bigram_ever_dropped_; + uint32_t doc_count_; + std::vector null_docids_; + const std::vector& terms_; // materialized fallback (may be empty) + SpimiTermBuffer* term_source_; // streaming source (null => use terms_) + uint64_t term_count_ = 0; // distinct terms actually consumed + const std::vector& encoded_norms_; + + uint32_t target_dict_block_bytes_; + // G16-h: zstd levels (dict whole-block / prx auto mode), from SniiIndexInput. + int dict_block_zstd_level_ = 3; + int prx_zstd_level_ = 3; + // The DICT region (zstd-compressed blocks) is staged here as blocks flush. It must + // land contiguously AFTER the posting region (which streams concurrently to the + // output), so it cannot stream directly; the orchestrator streams it into the + // container right after the posting region. It has NO independent local cap -- it + // spills to a temp via the writer's UNIFIED gate-2 cap (the MemoryReporter from + // SniiIndexInput, null off-Doris), the same single cap the SPIMI arena uses, so one + // threshold bounds the writer's total build RAM. The dict self-reports its ram_bytes_ + // deltas; the SPIMI term_source self-reports its arena+slot deltas (its reporter is + // injected at the source's own construction by the caller). + SpillableByteBuffer dict_buf_; + // The interleaved [prx][frq] posting region streams STRAIGHT into the container + // output during build() -- no temp. posting_out_ is the container writer (borrowed + // for the duration of build); posting_off0_ is its absolute offset when this index's + // region began, so posting_size() = bytes_written() - posting_off0_. + io::FileWriter* posting_out_ = nullptr; + uint64_t posting_off0_ = 0; + std::vector norms_section_; + std::vector null_bitmap_section_; + + std::vector blocks_; + // One 8-byte XXH64 (seed 0) filter key per term, collected during the build pass + // so the whole-vocabulary string copy is never retained. + std::vector term_hashes_; + format::StatsBlock stats_; + std::vector bsbf_bytes_; // serialized block-split bloom XFilter section + +public: + // G16 per-index section-byte accounting (measurement-only; nothing on disk + // depends on it). Index [0] = unigram, [1] = hidden phrase-bigram. Posting + // bytes come from the built DictEntry fields (dd/freq/prx/prelude split); + // dict_entry_bytes is the UNCOMPRESSED serialized entry-body split from + // DictBlockBuilder -- inline entries' posting bytes therefore appear in + // BOTH dd/freq/prx and dict_entry_bytes (they physically live in the dict). + // Logged once per index at the end of build(). + struct SectionStats { + uint64_t entries[2] {0, 0}; + uint64_t dd_bytes[2] {0, 0}; + uint64_t freq_bytes[2] {0, 0}; + uint64_t prx_bytes[2] {0, 0}; + uint64_t prelude_bytes[2] {0, 0}; + uint64_t dict_entry_bytes[2] {0, 0}; + }; + const SectionStats& section_stats() const { return section_stats_; } + +private: + SectionStats section_stats_; +}; + +// TEST-ONLY observability seam (mirrors the reader-side decode-counter and the +// SPIMI vocab-materialization patterns). term_freq_scans() returns a +// process-global count of term-level fused freqs scans -- fuse_freq_stats is +// called EXACTLY ONCE per term, so a build of N terms yields N (it was 3N for a +// docs-only build, or 4N with positions, across the former separate +// validate-sum / stats-sum / ttf-sum / max scans). note_term_freq_scan() bumps +// the counter (called only from fuse_freq_stats); reset_term_freq_scans() zeroes +// it between tests; fuse_freq_stats_for_test() exposes the real fused helper so +// pure boundary tests exercise production code. Process-global; reset between +// tests. Not part of the production API. +namespace testing { +void note_term_freq_scan(); +uint64_t term_freq_scans(); +void reset_term_freq_scans(); +FreqStats fuse_freq_stats_for_test(const std::vector& freqs); + +// G01 bigram-diet observability seam (same always-on relaxed-atomic pattern as +// term_freq_scans). Counts the writer's per-bigram materialization decisions -- +// bumped ONCE per non-sentinel phrase-bigram term reaching process_term: +// bigram_terms_materialized : the term got a dict entry (df >= threshold, or +// pruning disabled). +// bigram_terms_pruned : the term was skipped entirely (pruning active +// and final df < threshold): no dict entry, no +// postings, no bloom membership, no stats. +// bigram_terms_max_pruned : the term was skipped by the G15 UPPER gate +// (final df > bigram_prune_max_df): same total +// skip, counted separately from the min gate. +// The sentinel term counts toward NONE of these (it is never prunable). +// Process-global; reset between tests. Not part of the production API. +void note_bigram_term_materialized(); +void note_bigram_term_pruned(); +void note_bigram_term_max_pruned(); +uint64_t bigram_terms_materialized(); +uint64_t bigram_terms_pruned(); +uint64_t bigram_terms_max_pruned(); +// G16 seam: bumped ONCE per WINDOWED entry built with its freq region elided +// as a PER-TERM decision (prune-mode bigram on a freq-writing index). When the +// whole index drops freq (G16-c write_freq=false -> has_freq_=false) nothing +// counts here -- there is no per-term elision to observe. Slim/inline bigrams +// keep freq on freq-writing indexes and never count. Reset with the prune +// counters. +void note_bigram_freq_elided(); +uint64_t bigram_freqs_elided(); +void reset_bigram_prune_counters(); + +// G04 flush-side seam: bumped ONCE per bigram term that SURVIVED the df +// threshold but was dropped because the ever-dropped bloom may contain it +// (i.e. the drop the bloom -- not the threshold -- caused). df-dropped terms +// count toward bigram_terms_pruned as before, never here. Reset together with +// the prune counters by reset_bigram_prune_counters(). +void note_bigram_bloom_dropped(); +uint64_t bigram_bloom_dropped(); +} // namespace testing + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/memory_reporter.h b/be/src/storage/index/snii/writer/memory_reporter.h new file mode 100644 index 00000000000000..27d9436595566d --- /dev/null +++ b/be/src/storage/index/snii/writer/memory_reporter.h @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +namespace doris::snii::writer { + +// Per-WRITER accurate byte counter for build-time RAM (one per SniiCompoundWriter = +// one per segment's inverted index). Modules report their own resident-byte deltas; +// current_bytes() is that writer's accurate live usage. OBSERVE-ONLY -- SNII never +// makes a flush decision from it (gate 1 belongs to Doris; gate 2 is the internal +// threshold). consume_release mirrors the delta into Doris's LOAD MemTracker so the +// inverted-index RAM is counted by MemTableMemoryLimiter's pressure decision; it is +// null off-Doris (bench / unit tests), where only the local atomic is updated. +class MemoryReporter { +public: + using ConsumeReleaseFn = std::function; // null off-Doris + // cap_bytes is the UNIFIED gate-2 buffer cap for the WHOLE writer (e.g. Doris's + // 512 MiB inverted-index buffer config); 0 = unlimited. Every build buffer of this + // writer (SPIMI arena + dict) self-spills when over_cap() is true -- one threshold on + // the unified total, not a separate per-buffer threshold. + explicit MemoryReporter(ConsumeReleaseFn consume_release = nullptr, uint64_t cap_bytes = 0) + : consume_release_(std::move(consume_release)), cap_bytes_(cap_bytes) {} + + MemoryReporter(const MemoryReporter&) = delete; + MemoryReporter& operator=(const MemoryReporter&) = delete; + + // delta > 0 grows, delta < 0 shrinks/frees. Exactly one report per change site. + void report(int64_t delta) { + current_.fetch_add(delta, std::memory_order_relaxed); + if (consume_release_) consume_release_(delta); // mirror into Doris load tracker + } + + int64_t current_bytes() const { return current_.load(std::memory_order_relaxed); } + + // True once the writer's UNIFIED total build RAM (arena + slot index + dict + ...) + // reaches the cap. The single gate-2 trigger shared by every buffer of the writer. + bool over_cap() const { + return cap_bytes_ != 0 && current_bytes() >= static_cast(cap_bytes_); + } + uint64_t cap_bytes() const { return cap_bytes_; } + +private: + std::atomic current_ {0}; + ConsumeReleaseFn consume_release_; + uint64_t cap_bytes_ = 0; +}; + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/snii_compound_writer.cpp b/be/src/storage/index/snii/writer/snii_compound_writer.cpp new file mode 100644 index 00000000000000..f361931ed170ec --- /dev/null +++ b/be/src/storage/index/snii/writer/snii_compound_writer.cpp @@ -0,0 +1,167 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/snii_compound_writer.h" + +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/format/bootstrap_header.h" +#include "storage/index/snii/format/per_index_meta.h" // SectionRefs +#include "storage/index/snii/format/tail_meta_region.h" +#include "storage/index/snii/format/tail_pointer.h" + +namespace doris::snii::writer { + +using format::BootstrapHeader; +using format::SectionRefs; +using format::TailMetaRegionBuilder; +using format::TailPointer; + +SniiCompoundWriter::SniiCompoundWriter(io::FileWriter* out) : out_(out) {} + +Status SniiCompoundWriter::append(const std::vector& bytes) { + if (bytes.empty()) return Status::OK(); + return out_->append(Slice(bytes)); +} + +// The bootstrap header occupies offset 0 and must precede the first posting region, +// which streams straight into the output during build(). Written lazily exactly once +// (on the first add, or in finish() for an empty container). +Status SniiCompoundWriter::ensure_bootstrap() { + if (bootstrap_written_) return Status::OK(); + bootstrap_written_ = true; + return write_bootstrap(); +} + +Status SniiCompoundWriter::add_logical_index(const SniiIndexInput& in) { + if (out_ == nullptr) + return Status::Error("compound: null file writer"); + if (finished_) + return Status::Error("compound: add after finish"); + RETURN_IF_ERROR(ensure_bootstrap()); + auto liw = std::make_unique(in); + Placement p; + // The posting region streams DIRECTLY into the container during build() -- no temp + // round-trip for the bulk -- followed immediately by this index's compact DICT + // trailer (produced interleaved into a temp, but laid out right after its posting + // region, preserving the per-index [posting][dict] layout). Offsets are read off + // the output writer (the single source of truth -- no separate cursor). + p.post_off = out_->bytes_written(); + RETURN_IF_ERROR(liw->build(out_)); + p.post_len = out_->bytes_written() - p.post_off; + p.dict_off = out_->bytes_written(); + RETURN_IF_ERROR(liw->stream_dict_region_into(out_)); + p.dict_len = out_->bytes_written() - p.dict_off; + indexes_.push_back(std::move(liw)); + placements_.push_back(p); + return Status::OK(); +} + +Status SniiCompoundWriter::write_bootstrap() { + BootstrapHeader bh; + bh.tail_pointer_size = static_cast(format::tail_pointer_size()); + ByteSink sink; + RETURN_IF_ERROR(format::encode_bootstrap_header(bh, &sink)); + return append(sink.buffer()); +} + +// Writes each index's norms POD then bsbf section (in add order), after all the +// per-index [posting][dict] regions. +Status SniiCompoundWriter::write_norms() { + for (size_t i = 0; i < indexes_.size(); ++i) { + const LogicalIndexWriter& w = *indexes_[i]; + if (!w.has_norms() || w.norms_bytes().empty()) continue; + Placement& p = placements_[i]; + p.norms_off = out_->bytes_written(); + RETURN_IF_ERROR(append(w.norms_bytes())); + p.norms_len = out_->bytes_written() - p.norms_off; + } + for (size_t i = 0; i < indexes_.size(); ++i) { + const LogicalIndexWriter& w = *indexes_[i]; + if (!w.has_null_bitmap()) continue; + Placement& p = placements_[i]; + p.null_off = out_->bytes_written(); + RETURN_IF_ERROR(append(w.null_bitmap_bytes())); + p.null_len = out_->bytes_written() - p.null_off; + } + for (size_t i = 0; i < indexes_.size(); ++i) { + const LogicalIndexWriter& w = *indexes_[i]; + if (!w.has_bsbf()) continue; + Placement& p = placements_[i]; + p.bsbf_off = out_->bytes_written(); + RETURN_IF_ERROR(append(w.bsbf_bytes())); + p.bsbf_len = out_->bytes_written() - p.bsbf_off; + } + return Status::OK(); +} + +Status SniiCompoundWriter::write_tail() { + TailMetaRegionBuilder region; + for (size_t i = 0; i < indexes_.size(); ++i) { + const LogicalIndexWriter& w = *indexes_[i]; + const Placement& p = placements_[i]; + + SectionRefs refs; + refs.dict_region = {p.dict_off, p.dict_len}; + refs.posting_region = {p.post_off, p.post_len}; + refs.norms = {p.norms_off, p.norms_len}; + refs.null_bitmap = {p.null_off, p.null_len}; + refs.bsbf = {p.bsbf_off, p.bsbf_len}; + + ByteSink meta; + RETURN_IF_ERROR(w.finish_meta(refs, p.dict_off, &meta)); + region.add_index(w.index_id(), w.index_suffix(), meta.view()); + } + + ByteSink region_sink; + region.finish(®ion_sink); + const uint64_t region_off = out_->bytes_written(); + RETURN_IF_ERROR(append(region_sink.buffer())); + const uint64_t region_len = out_->bytes_written() - region_off; + + TailPointer tp; + tp.meta_region_offset = region_off; + tp.meta_region_length = region_len; + tp.hot_off = 0; + tp.meta_region_checksum = crc32c(region_sink.view()); + // Reserved: the bootstrap header carries (and decode_bootstrap_header verifies) its + // OWN internal crc32c, so a tail-pointer copy is redundant. Left 0 until a cross- + // region check needs it; the tail pointer's own tail_checksum still covers this + // field's bytes. + tp.bootstrap_header_checksum = 0; + ByteSink tail_sink; + RETURN_IF_ERROR(format::encode_tail_pointer(tp, &tail_sink)); + return append(tail_sink.buffer()); +} + +Status SniiCompoundWriter::finish() { + if (out_ == nullptr) + return Status::Error("compound: null file writer"); + if (finished_) + return Status::Error("compound: finish called twice"); + finished_ = true; + + RETURN_IF_ERROR(ensure_bootstrap()); // empty container still gets a header + RETURN_IF_ERROR(write_norms()); + RETURN_IF_ERROR(write_tail()); + return out_->finalize(); +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/snii_compound_writer.h b/be/src/storage/index/snii/writer/snii_compound_writer.h new file mode 100644 index 00000000000000..d7ce68645227bc --- /dev/null +++ b/be/src/storage/index/snii/writer/snii_compound_writer.h @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/io/file_writer.h" +#include "storage/index/snii/writer/logical_index_writer.h" + +// SniiCompoundWriter -- orchestrates a single-segment SNII container for one or +// more logical indexes, written front-to-back through an append-only +// io::FileWriter (no seek-back). It resolves all back-references by writing the +// tail meta region and the fixed tail pointer LAST. +// +// CONTAINER LAYOUT PRODUCED (this is the on-disk contract the reader matches): +// [bootstrap_header] (kBootstrapHeaderSize bytes) +// for each logical index, in add order: +// [posting region] interleaved [prx][frq] per pod_ref term, term order +// (prx span empty when !has_prx) +// [DICT blocks region] concatenated DICT blocks, split by +// target_dict_block_bytes +// for each logical index, in add order: +// [norms POD] NormsPodWriter::finish (scoring only; else absent) +// [null bitmap POD] NullBitmapWriter::finish (when nulls exist) +// [tail_meta_region] one per_index_meta block per index + directory +// [tail_pointer] encode_tail_pointer at EOF +// +// (The posting region is streamed BEFORE the DICT region per index: postings are +// the large append-only term-ordered stream; the DICT region is the compact +// compressed trailer.) +// +// OFFSET CONVENTIONS (ABSOLUTE file offsets unless stated otherwise): +// - SectionRefs in each per_index_meta record ABSOLUTE file offset+length of +// that index's posting_region, dict_region, norms. Absent regions are (0,0) +// (e.g. norms for a docs-positions index; null_bitmap is always (0,0) in v1). +// A present-but-empty posting_region (all-INLINE index) is (off, 0). +// - DictBlockDirectory entries record each DICT block's ABSOLUTE file offset + +// length. +// - A windowed/slim pod_ref entry's absolute .frq offset = +// section_refs.posting_region.offset + frq_base + frq_off_delta +// where frq_base is the posting-region-relative running offset captured at the +// block's open (see logical_index_writer.h). prx follows the identical rule +// against the SAME region (prx_base == frq_base). +// - tail_pointer.meta_region_offset/length point at the tail_meta_region; +// hot_off = 0 (no hot region in v1). +namespace doris::snii::writer { + +class SniiCompoundWriter { +public: + explicit SniiCompoundWriter(io::FileWriter* out); + + // Buffers one logical index: builds its section bytes and meta sub-sections. + // The actual file writing happens in finish() (single front-to-back pass). + Status add_logical_index(const SniiIndexInput& in); + + // Writes bootstrap header + all index sections + norms + tail meta region + + // tail pointer, then finalizes the underlying writer. May be called once. + Status finish(); + +private: + // Absolute placement of one index's sections, resolved during finish(). + struct Placement { + uint64_t dict_off = 0; + uint64_t dict_len = 0; + uint64_t post_off = 0; // interleaved [prx][frq] posting region (was frq + prx) + uint64_t post_len = 0; + uint64_t norms_off = 0; + uint64_t norms_len = 0; + uint64_t null_off = 0; + uint64_t null_len = 0; + uint64_t bsbf_off = 0; + uint64_t bsbf_len = 0; + }; + + Status ensure_bootstrap(); + Status write_bootstrap(); + Status write_norms(); + Status write_tail(); + Status append(const std::vector& bytes); + + io::FileWriter* out_; + std::vector> indexes_; + // Per-index placement; post_off/post_len are filled as each index's posting region + // streams in during add_logical_index, the rest during finish(). The absolute write + // offset is out_->bytes_written() (the single source of truth -- no separate cursor). + std::vector placements_; + bool bootstrap_written_ = false; + bool finished_ = false; +}; + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/spill_run_codec.cpp b/be/src/storage/index/snii/writer/spill_run_codec.cpp new file mode 100644 index 00000000000000..f5d7d37172e53a --- /dev/null +++ b/be/src/storage/index/snii/writer/spill_run_codec.cpp @@ -0,0 +1,729 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/spill_run_codec.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "storage/index/snii/encoding/varint.h" +#include "storage/index/snii/format/format_constants.h" + +namespace doris::snii::writer { + +namespace { + +// Flush staging once it grows past this. A LARGE write buffer (4 MiB) collapses +// the per-flush write() syscall count by ~64x: at 64 KiB the 5M build issued +// ~8800 write()s to ext4 (~9s of syscall overhead) for ~553 MiB of runs, versus +// a raw dd of the same bytes taking ~1.2s. Runs are PRIVATE temp files, so the +// on-disk index is unaffected; the only cost is a slightly larger transient +// RunWriter staging buffer (4 MiB, bounded, freed at close()). +constexpr size_t kWriteFlushBytes = 1u << 22; // 4 MiB +// RunReader reads this much per disk fill; the window slides so a single record +// never needs the whole run in RAM (only the current term's encoded span). KEEP +// this small (64 KiB): a large read chunk x many open runs would inflate the +// merge-phase peak RSS at low spill thresholds (each reader holds a window). +constexpr size_t kReadChunkBytes = 1u << 16; // 64 KiB + +void AppendVarint(std::vector* buf, uint64_t v) { + uint8_t tmp[10]; + const size_t n = encode_varint64(v, tmp); + buf->insert(buf->end(), tmp, tmp + n); +} + +// Appends a block of `count` uint32 values as RAW little-endian fixed-width bytes +// (memcpy from contiguous source). Runs are private temp files; the on-disk index +// is unaffected. Raw blocks make encode/decode ~10x cheaper than per-value varint +// for the freqs/positions streams (which compress poorly as varints anyway), at +// the cost of a modestly larger temp run. Empty source is a no-op. +void AppendRawU32(std::vector* buf, const uint32_t* src, size_t count) { + if (count == 0) return; + const auto* bytes = reinterpret_cast(src); + buf->insert(buf->end(), bytes, bytes + count * sizeof(uint32_t)); +} + +// Writes the full byte range [data, data+len) to fd, looping over short writes. +Status WriteAll(int fd, const uint8_t* data, size_t len) { + size_t off = 0; + while (off < len) { + const ssize_t n = ::write(fd, data + off, len - off); + if (n < 0) { + if (errno == EINTR) continue; + return Status::Error(std::string("run write failed: ") + + std::strerror(errno)); + } + off += static_cast(n); + } + return Status::OK(); +} + +} // namespace + +// --------------------------------------------------------------------------- +// RunWriter +// --------------------------------------------------------------------------- + +RunWriter::~RunWriter() { + if (fd_ >= 0) ::close(fd_); +} + +Status RunWriter::open(const std::string& path) { + fd_ = ::open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0600); + if (fd_ < 0) { + return Status::Error("run open(" + path + + "): " + std::strerror(errno)); + } + buf_.clear(); + return Status::OK(); +} + +Status RunWriter::flush() { + if (buf_.empty()) return Status::OK(); + RETURN_IF_ERROR(WriteAll(fd_, buf_.data(), buf_.size())); + buf_.clear(); + return Status::OK(); +} + +Status RunWriter::write_term(uint32_t term_id, const TermPostings& tp) { + AppendVarint(&buf_, term_id); + AppendVarint(&buf_, tp.docids.size()); + // Docids are a RAW fixed-width u32 block (bulk memcpy), NOT per-value VInt. + // Per-value varint over ~60M docids cost ~1.5s of encode CPU on the spill feed + // side; raw is a single memcpy and the decode side becomes a memcpy too. Runs + // are PRIVATE temp files written then read back from page cache, so the modestly + // larger run (no delta packing) costs ~0 extra real I/O. Absolute docids are + // stored (the merge concatenates per-term across runs and re-deltas at encode). + AppendRawU32(&buf_, tp.docids.data(), tp.docids.size()); + // Freqs + positions are RAW fixed-width u32 blocks (bulk memcpy). The decoder + // reads them back the same way; n_pos == positions_flat.size() is recoverable + // from sum(freqs), but is written explicitly so a reader can size the block. + AppendRawU32(&buf_, tp.freqs.data(), tp.freqs.size()); + const uint64_t n_pos = tp.positions_flat.size(); + AppendVarint(&buf_, n_pos); + AppendRawU32(&buf_, tp.positions_flat.data(), tp.positions_flat.size()); + if (buf_.size() >= kWriteFlushBytes) RETURN_IF_ERROR(flush()); + return Status::OK(); +} + +Status RunWriter::close() { + if (fd_ < 0) return Status::OK(); + RETURN_IF_ERROR(flush()); + const int fd = fd_; + fd_ = -1; + if (::close(fd) != 0) { + return Status::Error(std::string("run close: ") + + std::strerror(errno)); + } + return Status::OK(); +} + +// --------------------------------------------------------------------------- +// RunReader +// --------------------------------------------------------------------------- + +RunReader::~RunReader() { + if (fd_ >= 0) ::close(fd_); +} + +Status RunReader::open(const std::string& path, bool has_positions) { + fd_ = ::open(path.c_str(), O_RDONLY); + if (fd_ < 0) { + return Status::Error("run reopen(" + path + + "): " + std::strerror(errno)); + } + // Record the run's byte size so every length decoded from the stream can be + // bounded against it before allocating (no record holds more u32s than the whole + // file). Honors the header's "lengths validated against the file size" contract, + // turning a corrupt/truncated length into Status::Corruption rather than an + // uncaught std::bad_alloc from a giant resize(). + struct stat st {}; + if (::fstat(fd_, &st) != 0) { + return Status::Error(std::string("run fstat: ") + + std::strerror(errno)); + } + file_size_ = static_cast(st.st_size); + has_positions_ = has_positions; + exhausted_ = false; + eof_ = false; + pos_ = 0; + pos_count_ = 0; + pos_remaining_ = 0; + window_.clear(); + return advance(); +} + +// Slides consumed bytes out of the window, then appends one disk chunk. +Status RunReader::fill() { + if (pos_ > 0) { + window_.erase(window_.begin(), window_.begin() + pos_); + pos_ = 0; + } + if (eof_) return Status::OK(); + const size_t base = window_.size(); + window_.resize(base + kReadChunkBytes); + ssize_t n; + do { + n = ::read(fd_, window_.data() + base, kReadChunkBytes); + } while (n < 0 && errno == EINTR); + if (n < 0) + return Status::Error(std::string("run read: ") + + std::strerror(errno)); + window_.resize(base + static_cast(n)); + if (n == 0) eof_ = true; + return Status::OK(); +} + +// Buffered bytes available to the decoder right now (from pos_ to window end). +// fill() may slide the window (erasing consumed bytes), so callers must compare +// THIS quantity -- not window_.size() -- to decide whether more data arrived. +size_t RunReader::available() const { + return window_.size() - pos_; +} + +Status RunReader::ensure(size_t n) { + while (available() < n) { + const size_t had = available(); + RETURN_IF_ERROR(fill()); + if (available() == had && eof_) { + return Status::Error( + "run truncated: needed more bytes than available"); + } + } + return Status::OK(); +} + +// Streamed varint: decode from the current window; if it straddles the buffered +// boundary, top up from disk and retry. A varint is at most 10 bytes, so this +// loops at most a couple of times. Bounds-safe: decode_varint64 never reads past +// `end`, and a partial varint at true eof is reported as corruption. +Status RunReader::read_varint(uint64_t* v) { + while (true) { + const uint8_t* p = window_.data() + pos_; + const uint8_t* end = window_.data() + window_.size(); + const uint8_t* next = nullptr; + Status s = decode_varint64(p, end, v, &next); + if (s.ok()) { + pos_ += static_cast(next - p); + return Status::OK(); + } + if (eof_) + return Status::Error( + "run truncated: incomplete varint"); + const size_t had = available(); + RETURN_IF_ERROR(fill()); + if (available() == had && eof_) { + return Status::Error( + "run truncated: incomplete varint at eof"); + } + } +} + +// Streams `count` raw little-endian u32s from the window into `dst` (caller-owned +// storage of at least count*4 bytes), topping up the window from disk as needed. +// Copies whatever is buffered each pass (the window may hold only part of a large +// block), so a high-df term's freqs/positions stream through in 64 KiB chunks +// without ever needing the whole block resident at once. +Status RunReader::pull_raw_u32(uint8_t* dst, size_t count) { + if (count == 0) return Status::OK(); + size_t need = count * sizeof(uint32_t); + size_t written = 0; + while (need > 0) { + if (available() == 0) { + const size_t had = available(); + RETURN_IF_ERROR(fill()); + if (available() == had && eof_) { + return Status::Error( + "run truncated: needed more raw bytes than available"); + } + } + const size_t take = std::min(need, available()); + std::memcpy(dst + written, window_.data() + pos_, take); + pos_ += take; + written += take; + need -= take; + } + return Status::OK(); +} + +// Bulk-decodes `count` raw u32s into `out` (resized to count). +Status RunReader::read_raw_u32(size_t count, std::vector* out) { + // Bound `count` against the run's byte size BEFORE resize(): a record can never + // hold more u32s than the whole file. Rejects a corrupt/truncated length varint + // (which is otherwise an unbounded resize -> uncaught std::bad_alloc). + if (count > file_size_ / sizeof(uint32_t)) { + return Status::Error( + "run: raw u32 count exceeds file size"); + } + out->resize(count); + if (count == 0) return Status::OK(); + return pull_raw_u32(reinterpret_cast(out->data()), count); +} + +// Materializes the current term's deferred position block into positions_flat. +// A no-op once the positions are already drained (idempotent within a term). +Status RunReader::materialize_positions() { + if (pos_remaining_ == 0) { + current_.positions_flat.clear(); + return Status::OK(); + } + const size_t n = static_cast(pos_remaining_); + if (has_positions_) { + RETURN_IF_ERROR(read_raw_u32(n, ¤t_.positions_flat)); + } else { + // No-positions runs should carry n_pos == 0; tolerate (skip) a stray block. + std::vector skip; + RETURN_IF_ERROR(read_raw_u32(n, &skip)); + current_.positions_flat.clear(); + } + pos_remaining_ = 0; + return Status::OK(); +} + +// Streams the next `n` positions of the current term straight from the window. +Status RunReader::stream_positions(uint32_t* dst, size_t n) { + if (n == 0) return Status::OK(); + if (n > pos_remaining_) { + return Status::Error( + "run: stream_positions past block end"); + } + RETURN_IF_ERROR(pull_raw_u32(reinterpret_cast(dst), n)); + pos_remaining_ -= n; + return Status::OK(); +} + +// Discards any positions of the current term left unread, so the window cursor +// lands at the next record boundary before advance() reads the next term. +Status RunReader::skip_remaining_positions() { + if (pos_remaining_ == 0) return Status::OK(); + const size_t n = static_cast(pos_remaining_); + std::vector skip; + RETURN_IF_ERROR(read_raw_u32(n, &skip)); + pos_remaining_ = 0; + return Status::OK(); +} + +Status RunReader::advance() { + // Drain any positions the owner left unread for the previous term so the window + // cursor lands at the next record boundary. + RETURN_IF_ERROR(skip_remaining_positions()); + // End-of-run detection: at a record boundary, if no bytes remain we are done. + if (available() == 0) { + RETURN_IF_ERROR(fill()); + if (available() == 0 && eof_) { + exhausted_ = true; + return Status::OK(); + } + } + uint64_t term_id = 0; + RETURN_IF_ERROR(read_varint(&term_id)); + if (term_id > UINT32_MAX) + return Status::Error( + "run term_id exceeds uint32"); + current_id_ = static_cast(term_id); + current_.term.clear(); // runs store only the id; owner resolves the string + + uint64_t n_docs = 0; + RETURN_IF_ERROR(read_varint(&n_docs)); + // Docids: RAW absolute u32 block (bulk read), matching the writer's AppendRawU32. + RETURN_IF_ERROR(read_raw_u32(static_cast(n_docs), ¤t_.docids)); + // Freqs: RAW u32 block (bulk read), matching the writer's AppendRawU32. + RETURN_IF_ERROR(read_raw_u32(static_cast(n_docs), ¤t_.freqs)); + uint64_t n_pos = 0; + RETURN_IF_ERROR(read_varint(&n_pos)); + // Positions are LAZY: record the block count and leave the window cursor parked + // at the block start. The owner picks materialize_positions() (default) or + // stream_positions() (wide-term merge pump). The widest term's tens-of-MiB + // position block is thus never resident unless the owner asks for it whole. + current_.positions_flat.clear(); + pos_count_ = n_pos; + pos_remaining_ = n_pos; + return Status::OK(); +} + +// --------------------------------------------------------------------------- +// K-way merge +// --------------------------------------------------------------------------- + +namespace { + +// Min-heap entry: orders by the run's current term-id's PRECOMPUTED integer +// string-rank (rank[term_id] == its lexicographic rank over the dense vocabulary), +// tie-broken by run index so equal terms are gathered run-order (keeping +// concatenated docids ascending). The rank is a lexicographic bijection on a dense +// vocab, so ordering by the dense 4 B rank array reproduces the exact dictionary +// order a vocab-string compare would -- with an integer compare and zero random +// vocab string access in the inner loop. +struct HeapItem { + uint32_t term_id; + size_t run; +}; +struct HeapGreater { + const std::vector* rank; + bool operator()(const HeapItem& a, const HeapItem& b) const { + const uint32_t ra = (*rank)[a.term_id]; + const uint32_t rb = (*rank)[b.term_id]; + if (ra != rb) { + return ra > rb; + } // smaller rank first (lexicographic min-heap) + return a.run > b.run; // same term across runs: run-order tie-break + } +}; + +// Appends src's postings onto dst (run order). Later runs only cover docids +// >= dst's last, so docids stay ascending. COALESCE the boundary doc: if a spill +// fell BETWEEN two tokens of the same doc, that doc ends one run and begins the +// next with the SAME docid -- merge them (sum freqs, splice positions) so the +// merged term has exactly one entry per docid (matching the in-memory build). +// +// Positions are FLAT: doc order, partitioned by freqs. Because both dst and src +// already store doc-ordered flat positions, the common (no-boundary-overlap) case +// is a single bulk append. The boundary-overlap case must INSERT src's first +// doc's positions right after dst's last doc's positions so flat order stays +// consistent with the merged (coalesced) freqs. +void Concat(TermPostings* dst, const TermPostings& src, bool has_positions) { + if (src.docids.empty()) return; + // PER-TERM positions presence: a G04 position-suppressed bigram term is + // written with an EMPTY position block (n_pos == 0) even in a has_positions + // run. Its freqs are still real (> 0), so indexing positions_flat by freqs + // would read out of range -- gate every positions splice on the term + // actually carrying position bytes. Suppression is a property of the TERM + // (its marker prefix), so it is consistent across every run that holds it. + const bool src_has_pos = has_positions && !src.positions_flat.empty(); + size_t start = 0; + size_t src_pos_start = 0; // flat offset of src positions to append after splice + if (!dst->docids.empty() && dst->docids.back() == src.docids.front()) { + const uint32_t head_fc = src.freqs.front(); + if (src_has_pos && head_fc != 0) { + // Splice src's first-doc positions in right after dst's last-doc positions. + // dst's last doc owns dst->freqs.back() entries at the tail of positions_flat + // BEFORE we bump that freq, so insert at end() (last doc is the tail run). + auto& flat = dst->positions_flat; + flat.insert(flat.end(), src.positions_flat.begin(), + src.positions_flat.begin() + head_fc); + } + dst->freqs.back() += head_fc; + src_pos_start = head_fc; + start = 1; // boundary doc folded in; append the rest + } + dst->docids.insert(dst->docids.end(), src.docids.begin() + start, src.docids.end()); + dst->freqs.insert(dst->freqs.end(), src.freqs.begin() + start, src.freqs.end()); + if (src_has_pos) { + dst->positions_flat.insert(dst->positions_flat.end(), + src.positions_flat.begin() + src_pos_start, + src.positions_flat.end()); + } +} + +// Coalesces ONLY docids/freqs (no positions). Used by the WIDE-term path, whose +// positions are streamed via a pos_pump instead of materialized. The boundary-doc +// freq merge (dst->freqs.back() += head_fc) is identical to Concat's, so the +// merged df / freqs / ttf are bit-for-bit the same; positions are emitted in pure +// run-order concatenation by the pump (the same byte stream Concat would build). +void ConcatDocsFreqs(TermPostings* dst, const TermPostings& src) { + if (src.docids.empty()) return; + size_t start = 0; + if (!dst->docids.empty() && dst->docids.back() == src.docids.front()) { + dst->freqs.back() += src.freqs.front(); + start = 1; // boundary doc folded in; append the rest + } + dst->docids.insert(dst->docids.end(), src.docids.begin() + start, src.docids.end()); + dst->freqs.insert(dst->freqs.end(), src.freqs.begin() + start, src.freqs.end()); +} + +// A merged term is emitted with a STREAMED position pump (instead of a +// materialized positions_flat) when it is wide enough that its full flat +// positions would dominate the merge-phase peak RSS. The writer routes any term +// with df >= kSlimDfThreshold through the windowed path (build_windowed_entry), +// which is the only path that consumes pos_pump; a slim term reads positions_flat +// directly, so it must always be materialized. Gating on the same df threshold +// the writer uses keeps the two in lockstep and is conservative: only the few +// genuinely-wide terms (led by the single widest, the merge-phase peak driver) +// take the streamed path. total_pos is also required so a degenerate wide term +// with no positions still has something to stream. +bool ShouldStreamPositions(uint64_t total_docs, uint64_t total_pos, bool has_positions) { + return has_positions && total_pos != 0 && total_docs >= format::kSlimDfThreshold; +} + +} // namespace + +Status MergeRuns(const std::vector& run_paths, const std::vector& vocab, + const std::vector& string_rank, bool has_positions, + const std::function& fn, bool allow_stream_positions) { + // The heap/gather key is string_rank[term_id]; require it sized to the vocab so + // every in-vocab-range term-id (enforced by the current_id() < vocab.size() guards + // below) also indexes string_rank in bounds. A mismatch is a caller wiring bug. + if (string_rank.size() != vocab.size()) { + return Status::Error( + "MergeRuns: string_rank/vocab size mismatch"); + } + std::vector> readers; + readers.reserve(run_paths.size()); + std::priority_queue, HeapGreater> heap( + HeapGreater {&string_rank}); + for (size_t i = 0; i < run_paths.size(); ++i) { + auto r = std::make_unique(); + RETURN_IF_ERROR(r->open(run_paths[i], has_positions)); + if (!r->exhausted()) { + if (r->current_id() >= vocab.size()) { + return Status::Error( + "run term_id out of vocab range"); + } + heap.push({r->current_id(), i}); + } + readers.push_back(std::move(r)); + } + + std::vector matching; // run indices contributing the current term + while (!heap.empty()) { + const uint32_t id = heap.top().term_id; + TermPostings merged; + merged.term = vocab[id]; // resolve the id -> dictionary string once per term + // Gather every run whose head is THIS term-id (integer equality -- the heap's + // run tie-break keeps them in run order, so concatenated docids stay ascending). + // A dense vocab maps each id to a distinct string, so the same term across runs + // shares one id; comparing ids avoids any vocab string access in this loop. The + // matching runs' current slices are already loaded in their readers (they were + // read to seed the heap), so summing their sizes here costs nothing extra in RAM. + matching.clear(); + uint64_t total_docs = 0, total_pos = 0; + while (!heap.empty() && heap.top().term_id == id) { + const size_t ri = heap.top().run; + heap.pop(); + const RunReader* r = readers[ri].get(); + total_docs += r->current().docids.size(); + total_pos += r->current_pos_count(); // positions are LAZY: use the count + matching.push_back(ri); + } + // Reserve EXACTLY the summed sizes (an upper bound -- boundary-doc coalescing + // only shrinks the final size). This eliminates std::vector's geometric + // over-allocation, which left ~32 MiB of dead capacity on the widest term (df + // in the millions split across spills) -- a dominant merge-phase peak-RSS + // overhang at 5M. The reserved-but-unwritten pages are not faulted in, so the + // empty reservation itself does not raise RSS; only the actual data does. + merged.docids.reserve(static_cast(total_docs)); + merged.freqs.reserve(static_cast(total_docs)); + + bool stream = allow_stream_positions && + ShouldStreamPositions(total_docs, total_pos, has_positions); + if (!stream && has_positions) { + merged.positions_flat.reserve(static_cast(total_pos)); + } + // Coalesce docids/freqs from every matching run (always materialized -- a few + // u32 vectors). For the non-wide case, also coalesce positions here. For the + // wide case, leave positions for the streamed pump and keep the readers PARKED + // at their position blocks until fn() drains the pump. + for (size_t ri : matching) { + RunReader* r = readers[ri].get(); + if (stream) { + ConcatDocsFreqs(&merged, r->current()); + } else { + if (has_positions) RETURN_IF_ERROR(r->materialize_positions()); + Concat(&merged, r->current(), has_positions); + } + } + + // The stream gate keyed on PRE-coalesce total_docs, but the writer's slim vs + // windowed dispatch keys on the POST-coalesce df (merged.docids.size()). + // Boundary-doc coalescing across spill seams can drop df below kSlimDfThreshold + // while total_docs stayed above it; that term routes to build_slim_entry, which + // reads positions_flat directly and ignores pos_pump. Materialize positions now + // from the still-parked readers (mirrors drain_sorted()'s slim fallback). + if (stream && merged.docids.size() < format::kSlimDfThreshold) { + merged.positions_flat.reserve(static_cast(total_pos)); + for (size_t ri : matching) { + RunReader* r = readers[ri].get(); + RETURN_IF_ERROR(r->materialize_positions()); + const std::vector& pf = r->current().positions_flat; + merged.positions_flat.insert(merged.positions_flat.end(), pf.begin(), pf.end()); + } + stream = false; + } + + if (stream) { + // WIDE term: STREAM positions via a pump that walks the matching readers in + // run order (pure flat concatenation == the coalesced positions_flat, + // byte-for-byte). positions_flat stays empty -- the widest term's tens-of-MiB + // position buffer is never resident; only one ~64 KiB window per pull is. The + // readers are still parked at this term's blocks, so the pump pulls from them + // synchronously while fn() runs (fn consumes synchronously -- the windowed + // writer does). After fn(), advance the readers past the (now-drained) blocks. + merged.pos_total = total_pos; + size_t cursor = 0; // index into `matching` for the run currently being drained + Status pump_status = Status::OK(); + std::vector>* rd = &readers; + const std::vector* match = &matching; + // Self-contained liveness guard. The pump captures references into THIS stack + // frame (&cursor, &pump_status) and the parked run readers (rd/match), valid + // ONLY while fn() runs synchronously -- after fn() the readers advance past the + // drained blocks. `pump_alive` is heap-owned and captured BY VALUE, so a + // stored/deferred pos_pump fails loudly (throws) instead of dereferencing + // dangling state. See the contract on TermPostings::pos_pump. + auto pump_alive = std::make_shared(true); + merged.pos_pump = [rd, match, &cursor, &pump_status, pump_alive](uint32_t* dst, + size_t n) { + if (!*pump_alive) { + throw std::logic_error( + "TermPostings::pos_pump invoked after its producing merge scope ended; " + "the streamed TermPostings must be consumed synchronously inside fn() " + "and never stored for later use"); + } + size_t off = 0; + while (off < n) { + // Advance to the next run that still has positions to yield. + while (cursor < match->size() && + (*rd)[(*match)[cursor]]->positions_remaining() == 0) { + ++cursor; + } + if (cursor >= match->size()) break; // defensive: pump over-pulled + RunReader* r = (*rd)[(*match)[cursor]].get(); + const size_t take = + std::min(n - off, static_cast(r->positions_remaining())); + Status s = r->stream_positions(dst + off, take); + if (!s.ok()) { + // Mid-stream I/O / corruption: zero-fill the UNFILLED tail before + // returning. fn() has the pump and will consume dst BEFORE pump_status + // is surfaced after fn(); never hand it uninitialized bytes (the + // failed stream_positions wrote nothing into dst[off..]). The error is + // still latched and surfaced after fn(), so the build aborts -- the + // zero fill only guarantees deterministic, defined bytes meanwhile. + std::memset(dst + off, 0, (n - off) * sizeof(uint32_t)); + if (pump_status.ok()) pump_status = std::move(s); + return; + } + off += take; + } + // Short-fill on over-pull (cursor ran past the matching runs without an + // error status): the readers held fewer positions than n. Zero-fill the + // unfilled tail so the writer never reads uninitialized storage. With + // valid runs n == pos_total == sum(positions_remaining), so off == n and + // this memset spans zero bytes -- the produced .idx is unchanged. + if (off < n) std::memset(dst + off, 0, (n - off) * sizeof(uint32_t)); + }; + fn(std::move(merged)); + *pump_alive = false; // any later pos_pump call now throws instead of UAF + RETURN_IF_ERROR(pump_status); // surface a streamed-read I/O error + } else { + fn(std::move(merged)); + } + + // Advance every matching reader to its next term and re-seed the heap. For the + // wide path this also skips any positions the pump did not pull (none, when fn + // drained the whole stream); for the non-wide path positions were already + // materialized so nothing remains. + for (size_t ri : matching) { + RunReader* r = readers[ri].get(); + RETURN_IF_ERROR(r->advance()); // frees this run's slice, loads next term + if (!r->exhausted()) { + if (r->current_id() >= vocab.size()) { + return Status::Error( + "run term_id out of vocab range"); + } + heap.push({r->current_id(), ri}); + } + } + } + return Status::OK(); +} + +Status CompactRuns(const std::vector& run_paths, + const std::vector& string_rank, bool has_positions, + const std::string& out_path) { + // Same heap machinery as MergeRuns, but the output is a RUN (records keyed + // by term-id, ordered by string rank -- the exact invariant every run file + // carries), not a resolved term stream: no vocab strings are needed, and + // positions are always materialized because the run codec serializes + // positions_flat directly. + std::vector> readers; + readers.reserve(run_paths.size()); + std::priority_queue, HeapGreater> heap( + HeapGreater {&string_rank}); + for (size_t i = 0; i < run_paths.size(); ++i) { + auto r = std::make_unique(); + RETURN_IF_ERROR(r->open(run_paths[i], has_positions)); + if (!r->exhausted()) { + if (r->current_id() >= string_rank.size()) { + return Status::Error( + "run term_id out of rank range"); + } + heap.push({r->current_id(), i}); + } + readers.push_back(std::move(r)); + } + + RunWriter w; + RETURN_IF_ERROR(w.open(out_path)); + std::vector matching; // run indices contributing the current term + while (!heap.empty()) { + const uint32_t id = heap.top().term_id; + TermPostings merged; + matching.clear(); + uint64_t total_docs = 0; + uint64_t total_pos = 0; + while (!heap.empty() && heap.top().term_id == id) { + const size_t ri = heap.top().run; + heap.pop(); + const RunReader* r = readers[ri].get(); + total_docs += r->current().docids.size(); + total_pos += r->current_pos_count(); + matching.push_back(ri); + } + merged.docids.reserve(static_cast(total_docs)); + merged.freqs.reserve(static_cast(total_docs)); + if (has_positions) { + merged.positions_flat.reserve(static_cast(total_pos)); + } + // Concat (WITH boundary-doc coalescing) is deliberately the SAME + // append the final merge applies: coalescing the seam between two + // adjacent input runs here yields exactly what the final merge would + // have produced from the uncompacted pair, so compaction is invisible + // in the emitted term stream. A G04 position-suppressed term carries + // an empty position block in every run (a per-TERM property), so its + // compacted record is re-written with n_pos == 0. + for (size_t ri : matching) { + RunReader* r = readers[ri].get(); + if (has_positions) { + RETURN_IF_ERROR(r->materialize_positions()); + } + Concat(&merged, r->current(), has_positions); + } + RETURN_IF_ERROR(w.write_term(id, merged)); + for (size_t ri : matching) { + RunReader* r = readers[ri].get(); + RETURN_IF_ERROR(r->advance()); + if (!r->exhausted()) { + if (r->current_id() >= string_rank.size()) { + return Status::Error( + "run term_id out of rank range"); + } + heap.push({r->current_id(), ri}); + } + } + } + return w.close(); +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/spill_run_codec.h b/be/src/storage/index/snii/writer/spill_run_codec.h new file mode 100644 index 00000000000000..c6c0b8c16924b3 --- /dev/null +++ b/be/src/storage/index/snii/writer/spill_run_codec.h @@ -0,0 +1,224 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +namespace doris::snii::writer { + +// On-disk SPIMI "run" codec for the spill / k-way-merge out-of-core build path. +// +// A RUN is a self-describing file holding a sequence of terms keyed by TERM-ID, +// each followed by its postings, in this exact wire layout. The file is produced +// and consumed by THIS module only (a private temp file -- the on-disk INDEX is +// unaffected), so the format is chosen for cheap I/O: docids, freqs and positions +// are ALL RAW fixed-width little-endian u32 BLOCKS (bulk memcpy on both ends, +// ~10x cheaper than per-value varint -- which cost ~1.5s of encode CPU over the +// 5M build's ~60M docids and compressed those streams poorly anyway). Decode +// still validates every length against the file size. +// +// run := record* (term-ids ordered by vocab string, +// strictly ascending within a run) +// record := +// VInt term_id (index into the shared vocabulary; the +// string is NOT stored -- smaller runs, +// no per-record string IO) +// VInt n_docs +// u32 docid * n_docs (RAW LE block, memcpy; ABSOLUTE ascending +// docids -- the merge concatenates across +// runs and re-deltas at index encode time) +// u32 freq * n_docs (RAW LE block, memcpy; each >= 1) +// VInt n_pos (== sum(freqs) when has_positions, else 0) +// u32 position * n_pos (RAW LE block, document-order, partitioned +// by freqs) +// +// Decode is fully STREAMED: a RunReader reads a small fixed buffer at a time and +// materializes only the CURRENT term's postings, never the whole run. The k-way +// merge keeps one heap slot per run (each holding only its current term-id + +// that term's postings), so peak memory is bounded by the widest single term +// summed across the runs that contain it -- not by total postings. The merge +// orders runs by a PRECOMPUTED integer string-rank (term-id -> its lexicographic +// rank over the shared dense vocabulary): an integer compare that reproduces the +// exact lexicographic order without touching a vocab string in the inner loops. + +// Writes a sorted sequence of terms (by id) to one run file. Term-ids must be +// handed to write_term in vocab-string ascending order (the spill caller sorts +// before spilling). RAII: the file is flushed and closed on close(); the partial +// file is left for the owning SpimiTermBuffer to delete on its temp-path list. +class RunWriter { +public: + RunWriter() = default; + ~RunWriter(); + + RunWriter(const RunWriter&) = delete; + RunWriter& operator=(const RunWriter&) = delete; + + // Opens `path` for writing (truncating). Returns IoError on failure. + Status open(const std::string& path); + + // Appends one term's postings under `term_id`. `tp.positions_flat` either + // holds sum(freqs) entries in doc order, or is EMPTY -- always when + // !has_positions, and per-term for G04 position-suppressed bigram terms + // (the record's explicit n_pos makes either self-describing). Caller + // guarantees ascending docids and parallel docids/freqs lengths. + Status write_term(uint32_t term_id, const TermPostings& tp); + + // Flushes the buffer and closes the file. Safe to call once; idempotent. + Status close(); + +private: + Status flush(); + + int fd_ = -1; + std::vector buf_; // staging buffer; flushed in fixed-size chunks +}; + +// Streamed reader over one run file. After open() the first term is loaded; +// current()/current_id() expose it; advance() loads the next (or marks +// exhausted). Only the current term's postings live in memory at a time. The +// current record's `term` string is left EMPTY -- runs store only the id; the +// owner resolves the string via the shared vocabulary. +// +// LAZY POSITIONS (peak-RSS optimization for the widest merged term): advance() +// loads term_id / docids / freqs and the position-block COUNT, but does NOT read +// the position bytes -- it leaves the decode window cursor parked at the start of +// the position block. The owner then chooses, per term: +// * materialize_positions(): bulk-reads the block into current().positions_flat +// (the default; behaves exactly as the old eager reader). +// * stream_positions(dst, n): pulls the next n positions straight from the +// window in 64 KiB chunks, never materializing the whole block -- used by the +// k-way merge's wide-term position pump so the widest term's tens-of-MiB +// positions buffer is never resident. +// advance() drains any positions left unread from the previous term before the +// next record, so a partly-streamed (or skipped) term still lands at the right +// record boundary. The yielded byte sequence is identical either way. +class RunReader { +public: + RunReader() = default; + ~RunReader(); + + RunReader(const RunReader&) = delete; + RunReader& operator=(const RunReader&) = delete; + + // Opens `path`, loading the first record (if any). has_positions must match + // the writer's setting so n_pos is interpreted consistently. + Status open(const std::string& path, bool has_positions); + + bool exhausted() const { return exhausted_; } + const TermPostings& current() const { return current_; } + uint32_t current_id() const { return current_id_; } + + // Number of positions in the current term's (lazily-loaded) position block. + uint64_t current_pos_count() const { return pos_count_; } + // True once the current term's positions have been materialized OR fully + // streamed (i.e. nothing remains to read before advance()). + bool positions_drained() const { return pos_remaining_ == 0; } + + // Materializes the current term's position block into current().positions_flat + // (bulk read). Idempotent within a term: a no-op once positions are drained. + Status materialize_positions(); + // Streams the next `n` positions of the current term into dst[0..n) directly + // from the decode window (64 KiB chunks topped up on demand). Caller must not + // request more than positions_remaining(); each call advances the cursor. + Status stream_positions(uint32_t* dst, size_t n); + uint64_t positions_remaining() const { return pos_remaining_; } + + // Loads the next record into current(); sets exhausted() at end of file. Any + // positions of the current term left unread are skipped first. + Status advance(); + +private: + size_t available() const; // buffered bytes from pos_ to window end + Status fill(); // tops up the decode window from disk + Status ensure(size_t n); // guarantees >= n buffered bytes (or eof) + Status read_varint(uint64_t* v); // bounds-checked streamed varint + // Bulk-reads `count` RAW little-endian u32s from the window into `out` (resized + // to count). Bounds-checked against the run's true length (Corruption on EOF). + Status read_raw_u32(size_t count, std::vector* out); + // Streams `count` raw u32s from the window into dst (caller-owned, sized by the + // caller); shared by read_raw_u32 (into a vector) and stream_positions. + Status pull_raw_u32(uint8_t* dst, size_t count); + // Drains (and discards) any remaining positions of the current term so the + // window cursor lands at the next record boundary. + Status skip_remaining_positions(); + + int fd_ = -1; + bool has_positions_ = false; + bool exhausted_ = false; + uint64_t file_size_ = 0; // total run byte size (fstat at open); bounds lengths + std::vector window_; // sliding decode window + size_t pos_ = 0; // consumed offset within window_ + bool eof_ = false; // no more bytes on disk + uint32_t current_id_ = 0; // current record's term-id + uint64_t pos_count_ = 0; // current term's total position count (from n_pos) + uint64_t pos_remaining_ = 0; // positions still unread in the current block + TermPostings current_; +}; + +// K-way merges the given run files into a single term stream ordered by a +// PRECOMPUTED integer string-rank (string_rank[term_id] == the term-id's +// lexicographic rank over the dense vocabulary), invoking `fn` once per distinct +// term-id with its postings concatenated across all runs that contain it (in run +// order -> docids stay ascending) and its `term` resolved from `vocab` once. +// Because a dense vocab maps each id to a distinct string, the rank is a +// lexicographic bijection: ordering by the dense 4 B rank array (an integer +// compare) reproduces the EXACT order a vocab-string compare would -- but never +// reads a vocab string in the inner heap/gather loops. Only one merged term is +// materialized at a time. Returns IoError/Corruption on bad run data, or +// InternalError when string_rank.size() != vocab.size(). has_positions must match +// how the runs were written. `vocab` (term-id -> string) and `string_rank` +// (term-id -> rank) are both borrowed and MUST be sized to the vocabulary. +// +// allow_stream_positions (peak-RSS optimization): when true (the streaming-writer +// path), a WIDE merged term's positions are NOT materialized into positions_flat; +// instead the TermPostings carries a pos_pump that streams positions in document +// order straight from the run readers (which stay parked at this term's blocks +// for the duration of the fn() call). `fn` MUST therefore consume each term +// SYNCHRONOUSLY and must NOT retain the TermPostings past the call (the pump +// references live readers freed when the merge advances). Callers that retain the +// term (e.g. finalize_sorted) MUST pass false, so positions are always fully +// materialized. The produced bytes are identical either way. +Status MergeRuns(const std::vector& run_paths, const std::vector& vocab, + const std::vector& string_rank, bool has_positions, + const std::function& fn, bool allow_stream_positions = true); + +// G09 run-file cap support: k-way merges `run_paths` into ONE new run file at +// `out_path`, keyed and ordered exactly like MergeRuns (heap on +// string_rank[term_id]; per-term postings concatenated across runs in run +// order, boundary docs coalesced -- the same Concat the final merge applies, +// so compact-then-merge emits the identical term stream as merging the +// originals). Positions are always fully materialized per term (the run codec +// serializes positions_flat; no streaming pump), including per-term EMPTY +// position blocks (G04 suppressed bigrams), which are re-written as n_pos == 0. +// Every record's term-id must index string_rank (else Corruption). On error +// `out_path` may hold a partial file the caller must delete; the input runs +// are never modified. Opens run_paths.size() read fds + 1 write fd for the +// call's duration -- the caller (SpimiTermBuffer::compact_runs) bounds that +// fan-in with its run-count cap. +Status CompactRuns(const std::vector& run_paths, + const std::vector& string_rank, bool has_positions, + const std::string& out_path); + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/spillable_byte_buffer.h b/be/src/storage/index/snii/writer/spillable_byte_buffer.h new file mode 100644 index 00000000000000..ddbeb655b5c9c4 --- /dev/null +++ b/be/src/storage/index/snii/writer/spillable_byte_buffer.h @@ -0,0 +1,208 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "io/fs/file_reader.h" +#include "io/fs/file_reader_writer_fwd.h" +#include "io/fs/file_writer.h" +#include "io/fs/local_file_system.h" +#include "io/fs/path.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/io/file_writer.h" +#include "storage/index/snii/writer/memory_reporter.h" +#include "storage/index/snii/writer/temp_dir.h" +#include "util/slice.h" + +namespace doris::snii::writer { + +// A tiered append buffer for one build-time section. While resident it holds the +// bytes as a CHAIN OF CHUNKS (one per append) rather than a single growing vector: +// each append owns a right-sized allocation, so there is NO geometric-doubling +// realloc transient and NO power-of-two capacity slack -- the resident cost is +// exactly the bytes appended, for any section size. Once the running size crosses +// `cap_bytes` the buffer SPILLS to a temp file (resolve_temp_dir()) and routes later +// appends there, so a huge section stays RSS-bounded at ~cap_bytes while a small one +// is RAM-only (zero disk, spill-only build). append order/bytes are identical +// wherever they land; stream_into() reproduces the section in order. RAII-removes the +// temp. (cap_bytes == UINT64_MAX disables spilling -> always RAM.) +class SpillableByteBuffer { +public: + // `reporter` is an OPTIONAL writer-level build-RAM reporter (null off-Doris / + // unit tests). When non-null, every change to ram_bytes_ (the RESIDENT tier) is + // mirrored to it as a signed delta: a positive delta per RAM append, and a single + // negative delta == prior ram_bytes_ when the buffer spills (the resident chunks + // are dropped and the bytes move to disk, so they must NOT be counted as RSS). + // Spilled bytes live on disk and are never reported. + SpillableByteBuffer(uint64_t cap_bytes, std::string tag, MemoryReporter* reporter = nullptr) + : cap_bytes_(cap_bytes), tag_(std::move(tag)), reporter_(reporter) {} + ~SpillableByteBuffer() { + // Balance the reporter: on the common un-spilled path the resident ram_bytes_ was + // reported as positive on append but never released, so release it now (a missed + // negative would leak into Doris's MemTracker). After a spill, spill_to_disk() + // already reported the negative and ram_bytes_ no longer counts as resident. + if (reporter_ && !spilled_ && ram_bytes_ > 0) { + reporter_->report(-static_cast(ram_bytes_)); + } + // Release the Doris temp writer before unlinking: a sealed writer is already CLOSED + // (no-op here); an un-sealed one (error path) is aborted, which closes the fd and + // unlinks the scratch file. Then best-effort remove the path (RAII delete on success). + temp_writer_.reset(); + if (!temp_path_.empty()) std::remove(temp_path_.c_str()); + } + SpillableByteBuffer(const SpillableByteBuffer&) = delete; + SpillableByteBuffer& operator=(const SpillableByteBuffer&) = delete; + + // Total bytes appended so far (the offset basis for callers recording sub-offsets). + uint64_t size() const { return spilled_ ? spilled_bytes_ : ram_bytes_; } + + // Copying append (the Slice bytes are copied into a fresh chunk). + Status append(Slice bytes) { + if (spilled_) { + const ::doris::Slice s(bytes.data(), bytes.size()); + RETURN_IF_ERROR(to_snii(temp_writer_->appendv(&s, 1))); + spilled_bytes_ += bytes.size(); + return Status::OK(); + } + if (!bytes.empty()) { + chunks_.emplace_back(bytes.data(), bytes.data() + bytes.size()); + ram_bytes_ += bytes.size(); + if (reporter_) reporter_->report(static_cast(bytes.size())); + } + if (over_cap()) return spill_to_disk(); + return Status::OK(); + } + + // Move append: the section ADOPTS the caller's vector (no copy, no slack). The + // common dict path -- each flushed block is handed off by move. + Status append_move(std::vector&& v) { + if (spilled_) { + const ::doris::Slice s(v.data(), v.size()); + RETURN_IF_ERROR(to_snii(temp_writer_->appendv(&s, 1))); + spilled_bytes_ += v.size(); + return Status::OK(); + } + if (!v.empty()) { + ram_bytes_ += v.size(); + if (reporter_) reporter_->report(static_cast(v.size())); + chunks_.push_back(std::move(v)); + } + if (over_cap()) return spill_to_disk(); + return Status::OK(); + } + + // Must be called once after the last append, before stream_into(): flushes the temp + // (if spilled) so it can be read back. A no-op for a RAM-resident buffer. + Status seal() { + if (spilled_ && !sealed_) { + RETURN_IF_ERROR(to_snii(temp_writer_->close())); + sealed_ = true; + } + return Status::OK(); + } + + // Streams the whole section (RAM chunks or sealed temp) into `out`, in append order. + Status stream_into(io::FileWriter* out) const { + if (!spilled_) { + for (const auto& c : chunks_) { + if (!c.empty()) RETURN_IF_ERROR(out->append(Slice(c))); + } + return Status::OK(); + } + ::doris::io::FileReaderSPtr reader; + RETURN_IF_ERROR( + to_snii(::doris::io::global_local_filesystem()->open_file(temp_path_, &reader))); + constexpr uint64_t kChunk = 1u << 20; // fixed copy window (no whole-section reload) + std::vector buf; + for (uint64_t off = 0; off < spilled_bytes_; off += kChunk) { + const uint64_t n = std::min(kChunk, spilled_bytes_ - off); + buf.resize(static_cast(n)); + size_t bytes_read = 0; + RETURN_IF_ERROR(to_snii(reader->read_at( + off, ::doris::Slice(buf.data(), static_cast(n)), &bytes_read))); + if (bytes_read != n) { + return Status::Error( + "short read from spill scratch file"); + } + RETURN_IF_ERROR(out->append(Slice(buf.data(), static_cast(n)))); + } + return Status::OK(); + } + + bool spilled() const { return spilled_; } + +private: + // Gate-2 spill condition (UNIFIED): spill when the writer's TOTAL build RAM crosses + // the one shared cap (reporter_->over_cap()), with the local cap_bytes_ kept only as + // a defensive per-buffer hard ceiling (e.g. when no reporter is attached). + bool over_cap() const { + return (reporter_ != nullptr && reporter_->over_cap()) || ram_bytes_ >= cap_bytes_; + } + // Bridge a Doris IO Status into SNII's Status. R01 (status migration) is not done yet, + // so this buffer still returns Status; this mirrors snii_doris_adapter's + // to_snii_status (ok -> OK, otherwise IoError carrying the Doris message). + static Status to_snii(const Status& s) { + if (s.ok()) return Status::OK(); + return Status::Error(s.to_string_no_stack()); + } + Status spill_to_disk() { + temp_path_ = resolve_temp_dir() + "/snii_" + tag_ + "_" + std::to_string(::getpid()) + "_" + + std::to_string(reinterpret_cast(this)) + ".tmp"; + RETURN_IF_ERROR(to_snii( + ::doris::io::global_local_filesystem()->create_file(temp_path_, &temp_writer_))); + for (const auto& c : chunks_) { + if (!c.empty()) { + const ::doris::Slice s(c.data(), c.size()); + RETURN_IF_ERROR(to_snii(temp_writer_->appendv(&s, 1))); + } + } + spilled_bytes_ = ram_bytes_; + // The resident tier is freed: report the full negative delta == prior ram_bytes_ + // so the writer-level RAM counter (and Doris's LOAD tracker) no longer counts + // these bytes as RSS -- they now live on disk. This single negative balances the + // sum of all prior positive append deltas (net-zero RAM after spill). + if (reporter_) reporter_->report(-static_cast(ram_bytes_)); + std::vector>().swap(chunks_); // reclaim the RAM immediately + spilled_ = true; + return Status::OK(); + } + + uint64_t cap_bytes_; + std::string tag_; + MemoryReporter* reporter_ = nullptr; // optional build-RAM reporter (null off-Doris) + std::vector> chunks_; // resident tier: one chunk per append + uint64_t ram_bytes_ = 0; + bool spilled_ = false; + bool sealed_ = false; + ::doris::io::FileWriterPtr temp_writer_; // Doris local writer for the spill scratch file + std::string temp_path_; + uint64_t spilled_bytes_ = 0; +}; + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/spimi_term_buffer.cpp b/be/src/storage/index/snii/writer/spimi_term_buffer.cpp new file mode 100644 index 00000000000000..27957c325de90b --- /dev/null +++ b/be/src/storage/index/snii/writer/spimi_term_buffer.cpp @@ -0,0 +1,1638 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/spimi_term_buffer.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "storage/index/snii/encoding/varint.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/writer/global_memory_limiter.h" +#include "storage/index/snii/writer/spill_run_codec.h" +#include "storage/index/snii/writer/temp_dir.h" + +#if defined(__GLIBC__) +#include +#endif + +namespace doris::snii::writer { + +namespace { + +// Returns freed heap arenas to the OS (glibc only). The spill encode churns many +// small allocations whose freed chunks glibc retains in its arenas; trimming +// before the peak-RSS-defining merge phase recovers that retention. No-op (and +// harmless) on non-glibc libcs. +void TrimMalloc() { +#if defined(__GLIBC__) + ::malloc_trim(0); +#endif +} + +// Process-unique temp path for a spill run under `dir` (pid + monotonic counter so +// parallel builds / multiple buffers never collide). +std::string MakeRunPath(const std::string& dir) { + static std::atomic counter {0}; + const uint64_t n = counter.fetch_add(1); + return dir + "/snii_spill_" + std::to_string(::getpid()) + "_" + std::to_string(n) + ".run"; +} + +// TEST-ONLY seam backing testing::vocab_string_materialization_count(). Bumped once +// per DISTINCT interned term (owned_vocab_.emplace_back), never per token. Relaxed: +// the build path is single-threaded, so only the COUNT matters, not ordering. +std::atomic g_vocab_materializations {0}; + +// G04 bigram vocab-cap seams (same always-on relaxed pattern): evictions from +// the intern table, and bounded incremental sweep steps executed. +std::atomic g_bigram_evictions {0}; +std::atomic g_vocab_cap_sweeps {0}; + +// G05 pair-keyed bigram seams: pair-map probe outcomes on the id-keyed +// add_bigram_token hot path (hit = pair already interned; miss = first-time +// intern, counting re-interns after an eviction). +std::atomic g_bigram_pair_map_hits {0}; +std::atomic g_bigram_pair_map_misses {0}; + +// G06 drain-side df-gate seam: pair terms dropped by the drain df gate without +// materialization (final-drain no-bloom drops + mid-feed bloomed df-gate +// drops; NOT the plain df==1 G04 evictions). Incremented under BE_TEST only -- +// the final drain visits every live pair term (hundreds of millions on a +// wikipedia segment, concurrent writers), the same contention argument that +// gated the pair-map seams above. +std::atomic g_bigram_drain_df_drops {0}; + +// G09 seam: spills that consumed a pending process-wide forced-spill request +// (the limiter flagged this buffer as one of the largest registered consumers +// while the global total exceeded the budget). Incremented under BE_TEST only +// (per-token path, concurrent writers -- same rationale as the pair-map seams). +std::atomic g_global_forced_spills {0}; + +// G09 run-file cap seam: merge-compactions of a buffer's run list (always-on: +// at most one per cap-many spills, contention-free). +std::atomic g_run_compactions {0}; + +// G11 bench seam: when set (BE_TEST paths only), the add-path prefetch hints +// are skipped so the locality bench can A/B them in one process. Production +// builds never read it (the hint compiles in unconditionally there). +std::atomic g_bench_disable_g11_prefetch {false}; + +// G11 add-path prefetch gate: always-on in production; toggleable under +// BE_TEST for the in-process A/B bench. The branch is perfectly predicted, so +// the bench's OFF arm measures the pre-G11 code path faithfully. +inline bool g11_prefetch_enabled() { +#ifdef BE_TEST + return !g_bench_disable_g11_prefetch.load(std::memory_order_relaxed); +#else + return true; +#endif +} + +// Vocabulary ids examined per incremental sweep step. Small enough that a step +// is noise on the per-token add path, large enough that the sweep's amortized +// eviction rate (typically many eligible ids per step -- the over-cap +// vocabulary is dominated by the df==1 tail) outpaces the one-term-per-add +// intern growth that armed it. +constexpr uint32_t kVocabSweepStride = 64; + +// G08: heap payload of one owned-vocab string -- 0 while it fits the SSO buffer +// (those bytes live inside the 32 B header owned_vocab_.capacity() charges), else +// the allocated buffer (capacity + NUL). The SSO capacity is probed from the +// running stdlib so the classification is exact, not hardcoded. +uint64_t StringHeapBytes(const std::string& s) { + static const size_t kSsoCapacity = std::string().capacity(); + return s.capacity() > kSsoCapacity ? static_cast(s.capacity()) + 1 : 0; +} + +} // namespace + +namespace testing { +void set_bench_disable_g11_prefetch(bool disabled) { + g_bench_disable_g11_prefetch.store(disabled, std::memory_order_relaxed); +} +uint64_t vocab_string_materialization_count() { + return g_vocab_materializations.load(std::memory_order_relaxed); +} +void reset_vocab_string_materialization_count() { + g_vocab_materializations.store(0, std::memory_order_relaxed); +} +uint64_t bigram_evictions() { + return g_bigram_evictions.load(std::memory_order_relaxed); +} +uint64_t vocab_cap_sweeps() { + return g_vocab_cap_sweeps.load(std::memory_order_relaxed); +} +void reset_bigram_vocab_cap_counters() { + g_bigram_evictions.store(0, std::memory_order_relaxed); + g_vocab_cap_sweeps.store(0, std::memory_order_relaxed); +} +uint64_t bigram_pair_map_hits() { + return g_bigram_pair_map_hits.load(std::memory_order_relaxed); +} +uint64_t bigram_pair_map_misses() { + return g_bigram_pair_map_misses.load(std::memory_order_relaxed); +} +void reset_bigram_pair_map_counters() { + g_bigram_pair_map_hits.store(0, std::memory_order_relaxed); + g_bigram_pair_map_misses.store(0, std::memory_order_relaxed); +} +uint64_t bigram_drain_df_drops() { + return g_bigram_drain_df_drops.load(std::memory_order_relaxed); +} +void reset_bigram_drain_df_drops() { + g_bigram_drain_df_drops.store(0, std::memory_order_relaxed); +} +uint64_t global_forced_spills() { + return g_global_forced_spills.load(std::memory_order_relaxed); +} +void reset_global_forced_spills() { + g_global_forced_spills.store(0, std::memory_order_relaxed); +} +uint64_t run_compactions() { + return g_run_compactions.load(std::memory_order_relaxed); +} +void reset_run_compactions() { + g_run_compactions.store(0, std::memory_order_relaxed); +} +} // namespace testing + +SpimiTermBuffer::SpimiTermBuffer(const std::vector* vocab, bool has_positions, + size_t spill_threshold_bytes, MemoryReporter* reporter) + : vocab_(vocab), + // Bind the interning set's heterogeneous functors to &owned_vocab_ even in + // borrowed mode: the back-pointer is harmless here because + // add_token(string_view) rejects before touching intern_ (the functors are + // never dereferenced), and binding unconditionally keeps both constructors + // symmetric. Initialized in the member-init list (NOT the body): the functors + // are NESTED types, whose default-constructibility is not yet established at + // the point the flat set's default ctor (whose noexcept spec inspects the + // functors) would be needed for a body assignment. The + // (bucket_count, hash, equal) constructor sidesteps that entirely. + // owned_vocab_ is constructed before intern_ (declaration order) and the + // buffer is non-movable, so &owned_vocab_ is stable for the buffer's life. + intern_(0, OwnedVocabHash {&owned_vocab_}, OwnedVocabEq {&owned_vocab_}), + has_positions_(has_positions), + spill_threshold_bytes_(spill_threshold_bytes), + mem_reporter_(reporter) { + // Borrowed-vocab mode: only the 4 B/id slot-index array is sized to the + // vocabulary; the Term pool (slots_) grows with the LIVE touched count, so an + // all-but-empty vocabulary costs ~4 B/id instead of ~80 B/id. + slot_of_.assign(vocab_->size(), 0); + // The vocab-sized slot index is resident immediately and survives spills; report + // its initial positive delta now. + report_arena_delta(); +} + +SpimiTermBuffer::SpimiTermBuffer(bool has_positions, size_t spill_threshold_bytes, + MemoryReporter* reporter) + : vocab_(&owned_vocab_), + // Owned-vocab mode: bind the interning set's heterogeneous functors to + // &owned_vocab_ so a stored term-id dereferences to its string for content + // hashing and equality. Initialized in the member-init list (NOT the body): + // the functors are NESTED types whose default-constructibility is not yet + // established where the flat set's default ctor (whose noexcept spec inspects + // the functors) would be needed for a body assignment, so the + // (bucket_count, hash, equal) constructor is used instead. owned_vocab_ is + // constructed before intern_ (declaration order) and the buffer is + // non-movable, so &owned_vocab_ is stable for the buffer's life. + intern_(0, OwnedVocabHash {&owned_vocab_}, OwnedVocabEq {&owned_vocab_}), + has_positions_(has_positions), + spill_threshold_bytes_(spill_threshold_bytes), + mem_reporter_(reporter) { + // Owned-vocab mode: the vocabulary grows as strings are interned in + // add_token(string_view, ...). +} + +SpimiTermBuffer::~SpimiTermBuffer() { + // G09: leave the process-wide registry FIRST. unregister_buffer removes the + // entry (and its bytes) under the registry mutex -- the same mutex every + // flag store is made under -- so once it returns, no other thread can touch + // global_spill_requested_ while this buffer dies. + if (global_limiter_ != nullptr) { + global_limiter_->unregister_buffer(&global_spill_requested_); + global_limiter_ = nullptr; + } + // Balance the writer-level / Doris tracker on the error path: if the buffer is + // destroyed while resident bytes were reported but not yet freed-and-reported + // (e.g. a build aborts before draining), return them here so nothing leaks. + if (mem_reporter_ != nullptr && reported_resident_ != 0) { + mem_reporter_->report(-reported_resident_); + reported_resident_ = 0; + } + cleanup_runs(); +} + +void SpimiTermBuffer::attach_global_limiter(GlobalMemoryLimiter* limiter) { + // At-most-once: a re-attach would leave a stale registry entry behind (the + // dtor un-registers only the current limiter). + if (limiter == nullptr || global_limiter_ != nullptr) { + return; + } + global_limiter_ = limiter; + // Race-safe vs report: registration and every report run on the OWNER's + // thread, strictly ordered; the registry serializes them against other + // buffers' calls internally. Register with the CURRENT resident total AND + // the current spillable arena bytes (the victim-selection key) so the + // registry is exact from the first moment (a borrowed-vocab buffer + // already holds its vocab-sized slot index here). + global_limiter_->register_buffer(&global_spill_requested_, + static_cast(resident_bytes()), + static_cast(pool_.arena_bytes())); +} + +void SpimiTermBuffer::configure_bigram_diet(uint64_t vocab_cap_bytes, + uint32_t bigram_drain_min_df) { + // The diet suppresses positions and (with a cap) evicts through the OWNED + // intern table; a borrowed vocab has neither, so reject loudly instead of + // silently doing nothing (mirrors the add_bigram_token mode contract). + if (vocab_ != &owned_vocab_) { + if (spill_status_.ok()) { + spill_status_ = Status::Error( + "spimi: configure_bigram_diet requires owned-vocab mode"); + } + return; + } + bigram_diet_ = true; + bigram_vocab_cap_bytes_ = vocab_cap_bytes; + // G06: governs MID-FEED spill drains only until the flush re-plumbs the + // exact effective threshold via set_bigram_drain_min_df (see header). + bigram_drain_min_df_ = bigram_drain_min_df; +} + +void SpimiTermBuffer::report_arena_delta() { + if (mem_reporter_ == nullptr && global_limiter_ == nullptr) { + return; + } + // Diff the REAL resident bytes (resident_bytes()) against the last reported + // total; emit the signed delta exactly once. + const auto now = static_cast(resident_bytes()); + // Per-token zero-delta debounce: skip the locked fetch_add when resident is + // unchanged (the common case -- arena_bytes() grows only ~every 32 KiB block and + // the other charged structures grow by geometric capacity steps / per new term + // only, so most tokens see delta==0). A + // delta==0 report() is a no-op (current_.fetch_add(0) plus a mirrored + // consume_release(0)) and leaves reported_resident_ == now, so current_bytes(), + // every over_cap() result, and the gate-2 spill timing stay bit-for-bit identical. + // This debounces report() ONLY: accumulate() still evaluates over_cap() + // UNCONDITIONALLY every token, because the writer-level UNIFIED total (shared with + // the dict buffer) can cross the cap while this buffer's local delta is 0 -- gating + // over_cap() on this delta would miss that spill. + if (now == reported_resident_) { + return; + } + if (mem_reporter_ != nullptr) { + mem_reporter_->report(now - reported_resident_); + } + // G09: forward the same debounced total -- as an ABSOLUTE, self-healing + // value -- to the process-wide registry, together with the current + // SPILLABLE arena bytes (the victim-selection key: only the arena is + // reclaimable by a forced spill; the persistent vocab/pair structures are + // not). This is the limiter's decision point: report() flags the + // largest-arena eligible buffers (possibly this one) while the global sum + // exceeds the budget. It only ever takes the registry mutex and flips + // advisory atomics; no lock is held here while spilling (any spill this + // buffer performs happens AFTER this returns, back in + // maybe_spill_after_token, on this thread). + if (global_limiter_ != nullptr) { + global_limiter_->report(&global_spill_requested_, now, + static_cast(pool_.arena_bytes())); + } + reported_resident_ = now; +} + +size_t SpimiTermBuffer::unique_terms() const { + return live_term_count_; +} + +uint64_t SpimiTermBuffer::resident_bytes() const { + // REAL resident accumulator bytes (G08). Pre-G05 this was arena + slot index + // only; the G05 pair-key machinery and the owned vocab / slot-pool / rank + // arrays were INVISIBLE to the gate-2 trigger and the MemoryReporter, so on + // wikipedia each writer peaked at [uncharged structures] + the full 512 MiB + // cap (~1.25 GiB x 16 writers = the observed ~20 GiB). Everything live is + // charged now, by CAPACITY (the reserved tail is resident RSS and survives + // spills -- spill_to_run frees only the arena). All O(1) field reads: this + // runs once per token via report_arena_delta. + uint64_t b = pool_.arena_bytes(); // posting chains: unigram + bigram, docs + prx payload + b += static_cast(slot_of_.capacity()) * sizeof(uint32_t); // vocab-sized slot index + b += static_cast(slots_.capacity()) * sizeof(Term); // live Term pool + b += static_cast(free_slots_.capacity()) * sizeof(uint32_t); + b += static_cast(touched_ids_.capacity()) * sizeof(uint32_t); + // Owned-vocab machinery (all zero in borrowed mode): string headers by vector + // capacity, heap payloads via the incrementally-maintained counter (credited + // by intern_owned_term / materialize_pair_term, debited by evict_bigram_term), + // and the intern set's entries at a fixed per-entry estimate (kept at the + // pre-G10 node-set value so the gate-2 spill points are unchanged; see the + // constant's comment). + b += static_cast(owned_vocab_.capacity()) * sizeof(std::string); + b += owned_vocab_heap_bytes_; + b += static_cast(intern_.size()) * kInternEntryEstimateBytes; + // G05 pair-key machinery: flat-map slots by CAPACITY (16 B pair + 1 control + // byte per slot; erase never shrinks it) and the vocab-sized reverse + // pair-key slots. On wikipedia these -- unbounded by the G04 cap, which can + // only evict the df==1 tail -- were the largest uncharged line. + b += static_cast(bigram_pair_map_.capacity()) * + (sizeof(decltype(bigram_pair_map_)::value_type) + 1); + b += static_cast(pair_of_.capacity()) * sizeof(uint64_t); + // G04 diet bookkeeping + the cached lexicographic rank. + b += static_cast(free_ids_.capacity()) * sizeof(uint32_t); + b += static_cast(id_in_run_.capacity()) * sizeof(uint8_t); + b += static_cast(string_rank_.capacity()) * sizeof(uint32_t); + return b; +} + +// Returns the live Term for `term_id`, claiming a pool slot on first touch (1 == +// new). Reuses a freed slot from free_slots_ when available; otherwise appends a +// fresh Term to slots_. slot_of_[term_id] holds (slot index + 1); 0 means empty. +SpimiTermBuffer::Term& SpimiTermBuffer::term_slot(uint32_t term_id, bool* new_term) { + uint32_t enc = slot_of_[term_id]; + if (enc != 0) { + *new_term = false; + return slots_[enc - 1]; + } + *new_term = true; + uint32_t slot; + if (!free_slots_.empty()) { + slot = free_slots_.back(); + free_slots_.pop_back(); + } else { + slot = static_cast(slots_.size()); + slots_.emplace_back(); + } + slot_of_[term_id] = slot + 1; + return slots_[slot]; +} + +// Appends one byte to a term's chain, starting the chain lazily on first use. +void SpimiTermBuffer::put_byte(Term* t, uint8_t b) { + if (t->head == kNoChain) { + t->head = pool_.start_chain(&t->w, &t->level); + } + pool_.append_byte(&t->w, &t->level, b); +} + +void SpimiTermBuffer::put_varint(Term* t, uint64_t v) { + uint8_t tmp[10]; + const size_t n = encode_varint64(v, tmp); + for (size_t i = 0; i < n; ++i) { + put_byte(t, tmp[i]); + } +} + +void SpimiTermBuffer::accumulate(uint32_t term_id, uint32_t docid, uint32_t pos) { + bool new_term = false; + Term& t = term_slot(term_id, &new_term); + if (new_term) { + touched_ids_.push_back(term_id); + ++live_term_count_; + // G04 position suppression: a hidden phrase-bigram term (marker prefix; + // sentinel included -- its single token is position 0 either way) stores + // no position payload once the diet is on. Decided ONCE per term at slot + // claim (a 20-byte prefix check), never per token. A G05 pair-keyed + // bigram term carries an EMPTY vocab string until flush, so it is + // recognized by its pair-map membership instead (checked first: two + // loads, and it skips the prefix memcmp on the dominant bigram stream). + if (bigram_diet_) { + t.pos_suppressed = + is_pair_term(term_id) || format::is_phrase_bigram_term(vocab()[term_id]); + } + } + // A token starts a new doc unless it continues the most-recent doc for this term. + const bool new_doc = !t.started || t.cur_docid != docid; + // Tagged entry: varint((pos << 1) | new_doc). Positions are tagged 0 when + // disabled buffer-wide OR suppressed for this term (G04 bigram diet: the + // suppressed encoding is bit-identical to the positions-disabled one). The + // new_doc bit lets the decoder recover per-doc freqs by counting. + // Widen to 64-bit so a full 32-bit position survives the << 1 without truncation. + const bool token_has_pos = has_positions_ && !t.pos_suppressed; + const uint64_t tagged = token_has_pos + ? ((static_cast(pos) << 1) | (new_doc ? 1U : 0U)) + : (new_doc ? 1U : 0U); + put_varint(&t, tagged); + if (new_doc) { + // Out-of-order docids are tolerated (zigzag delta is signed) and reordered at + // finalize; flag them so to_postings sorts. The delta base is the previous + // distinct doc (cur_docid), which is 0 for the very first doc (started==false). + const int64_t base = t.started ? static_cast(t.cur_docid) : 0; + if (t.started && docid < t.cur_docid) { + t.sorted = false; + } + const int64_t delta = static_cast(docid) - base; + put_varint(&t, zigzag_encode(delta)); + t.cur_docid = docid; + t.started = true; + // Exact new-doc-GROUP count: >= the coalesced df, == it while t.sorted + // holds. Feeds the G04 df==1 evictability check and the G06 drain-side + // bigram df gate (branch-free; replaces the old saturate-at-2 branch). + ++t.ndocs; + } + ++t.ntok; + ++total_tokens_; + + maybe_spill_after_token(); +} + +// Per-token gate-2 tail (extracted from accumulate, which every add path funnels +// through -- so every add path observes this gate). Reports this token's REAL +// resident growth FIRST so the writer's unified total (reporter_->current_bytes()) +// reflects it before the gate check (single-source diff; cheap: a subtraction + +// relaxed atomic add), then evaluates the spill triggers: +// * Gate-2 (UNIFIED): with a reporter attached, trigger on the writer's TOTAL +// build RAM (arena + vocab/pair structures + dict) crossing the one +// configured cap -- the same total and cap every buffer of this writer +// shares, not a per-buffer threshold. Off Doris (no reporter) fall back to +// the local spill_threshold_bytes_ against resident_bytes(). +// * G08 anti-churn floor: a gate-2 spill reclaims ONLY the posting arena +// (pool_.reset()); the vocab / pair-map / slot structures resident_bytes() +// now also charges SURVIVE it. Once those persistent bytes alone exceed the +// cap (wikipedia: the df>=2 pair map + owned vocab do), an unconditioned +// trigger would spill EVERY subsequent token -- one-block runs, k-way-merge +// and spill-fixed-cost blowup. Honor the cap only when at least a quarter of +// it is reclaimable arena: peak stays bounded at persistent + cap/4 and no +// run is smaller than cap/4, while the one-block minimum keeps small caps +// (tests, tiny configs) spilling on the first block exactly as before. +// * Hard arena safety stop, active even in unlimited mode and BYPASSING the +// floor: when the arena nears the 4 GiB uint32-offset limit, spill now -- +// without it a single >4 GiB in-memory segment wraps alloc_run and silently +// corrupts data. A forced spill + final k-way merge stays byte-identical +// regardless of when it fires. +// spill_to_run() resets the arena and reports its negative internally, so the +// unified total drops (and the trigger self-rearms) after each spill. +void SpimiTermBuffer::maybe_spill_after_token() { + constexpr uint64_t kArenaSpillCap = 0xE0000000ULL; // 3.5 GiB, < UINT32_MAX margin + report_arena_delta(); + const bool over_cap = mem_reporter_ != nullptr ? mem_reporter_->over_cap() + : (spill_threshold_bytes_ != 0 && + resident_bytes() >= spill_threshold_bytes_); + const uint64_t gate_cap = + mem_reporter_ != nullptr ? mem_reporter_->cap_bytes() : spill_threshold_bytes_; + const bool arena_worth_spilling = + pool_.arena_bytes() >= std::max(CompactPostingPool::kBlockSize, gate_cap / 4); + const bool arena_near_limit = pool_.arena_bytes() >= kArenaSpillCap; + // G09: the process-wide limiter flagged this buffer (one of the + // largest-ARENA eligible consumers while the global total exceeded the + // budget). Honored HERE, on the owner's own thread -- never on the + // reporting thread that set the flag. The G08 anti-churn floor (cap/4) is + // deliberately BYPASSED (each victim's arena is below cap/4 by + // construction: it never reached its per-writer gate -- that is exactly + // why the global sum grew), but the FORCED-SPILL FLOOR + // (snii_forced_spill_min_arena_bytes, >= one arena block so a run is + // writable) still applies: a forced spill reclaims ONLY the arena, so + // honoring below the floor would cut a tiny run for near-zero relief. + // Below the floor the request is a NO-OP that stays PENDING -- it is NOT + // retried as a spill each token -- and is honored once the arena regrows + // past the floor (the limiter's victim selection applies the same floor, + // so a below-floor flag only arises from a floor/config race or a test + // seam). A request that finds the owner already drained is never observed + // again -- an advisory no-op (the dtor un-registers) -- and a stale + // re-request after a spill costs at most one extra floor-sized run + // (double-spill is harmless, byte-identical output). + const bool global_requested = global_spill_requested_.load(std::memory_order_relaxed); + const bool global_spill_now = + global_requested && + pool_.arena_bytes() >= std::max(CompactPostingPool::kBlockSize, + forced_spill_min_arena_bytes_); + if (((over_cap && arena_worth_spilling) || global_spill_now || arena_near_limit) && + spill_status_.ok()) { + if (global_requested) { + // Consume the request BEFORE spilling: this spill releases exactly + // the arena a forced spill would, so it satisfies the request no + // matter which trigger won the OR above. + global_spill_requested_.store(false, std::memory_order_relaxed); +#ifdef BE_TEST + // Seam under BE_TEST only: per-token path shared by every + // concurrent writer (same rationale as the pair-map seams). + g_global_forced_spills.fetch_add(1, std::memory_order_relaxed); +#endif + } + // Mid-feed spill: evict df==1 bigrams instead of writing them (a run + // record would pin their vocab strings forever -- see drain_to_writer). + spill_status_ = spill_to_run(/*evict_low_df_bigrams=*/true); + } +} + +void SpimiTermBuffer::add_token(uint32_t term_id, uint32_t docid, uint32_t pos) { + // Hot path: a pooled slot lookup + a couple of pushes. No hashing, no string + // construction per token. Reject (and latch) an out-of-range id. + if (term_id >= slot_of_.size()) { + if (spill_status_.ok()) { + spill_status_ = Status::Error( + "spimi: term_id out of vocab range"); + } + return; + } + accumulate(term_id, docid, pos); +} + +void SpimiTermBuffer::add_token(std::string_view term, uint32_t docid, uint32_t pos) { + (void)add_token_returning_id(term, docid, pos); +} + +uint32_t SpimiTermBuffer::add_token_returning_id(std::string_view term, uint32_t docid, + uint32_t pos) { + // Compatibility path: intern the term into the owned vocabulary on first + // occurrence, then accumulate by its id. ONLY valid in OWNED-vocab mode. In + // BORROWED-vocab mode vocab_ points at the caller's vector, NOT &owned_vocab_: + // interning here would grow owned_vocab_ / intern_ / slot_of_ out of step with + // the active (borrowed) vocab, so the new id indexes the WRONG string and writes + // a slot_of_ entry the borrowed-vocab build never reconciles -- silent + // corruption. Reject (and latch) instead of forwarding by a bogus id. + if (vocab_ != &owned_vocab_) { + if (spill_status_.ok()) { + spill_status_ = Status::Error( + "spimi: add_token(string_view) requires owned-vocab mode"); + } + return kInvalidTermId; + } + // F03 single-store invariant, fixed at compile time: the interning set keys on the + // 4-byte term-id, NEVER on a std::string, so each vocab string lives in exactly one + // place (owned_vocab_). A regression to a string key would reintroduce the + // double-store and fail this build. + static_assert(std::is_same_v, + "intern_ must key on term-id (single-store); a string key reintroduces F03"); + + // Heterogeneous probe with the string_view directly: NO per-token temporary + // std::string (F21). The set element is the term-id; its content is resolved + // through owned_vocab_ by the transparent functors. + auto it = intern_.find(term); + uint32_t term_id; + if (it == intern_.end()) { + // First occurrence: materialize the string exactly once (F03 + // single-store) and intern it, recycling an evicted id when available. + term_id = intern_owned_term(std::string(term)); + } else { + term_id = *it; // the set element IS the term-id + // G11: start the term's slot-index line fetch as soon as the intern + // probe resolves the id (see the pair-keyed path). + if (g11_prefetch_enabled()) { + __builtin_prefetch(slot_of_.data() + term_id); + } + } + accumulate(term_id, docid, pos); + // G04: amortized vocab-cap sweep step. Hooked on the string path too so + // synthetic bigram terms fed by tests / legacy callers behave identically to + // the piecewise add_bigram_token path. Two compares when the cap is off/idle. + maybe_sweep_bigram_vocab(term_id); + return term_id; +} + +void SpimiTermBuffer::add_bigram_token(std::string_view left, std::string_view right, + uint32_t docid, uint32_t pos) { + // Same OWNED-vocab-mode contract (and failure latch) as add_token(string_view): + // interning into a borrowed vocab would corrupt the id space. + if (vocab_ != &owned_vocab_) { + if (spill_status_.ok()) { + spill_status_ = Status::Error( + "spimi: add_bigram_token requires owned-vocab mode"); + } + return; + } + // G01 part C hot path: probe the intern set with the PIECEWISE key -- the + // transparent functors hash/compare marker + varint(len(left)) + left + right + // fragment by fragment against the stored composed strings, so a REPEAT word + // pair (the overwhelming majority of the ~per-token bigram stream) performs + // zero allocation and zero byte copies. The composed std::string is built + // exactly once, on first-time intern below. + const PhraseBigramTermView probe {left, right}; + auto it = intern_.find(probe); + uint32_t term_id; + if (it == intern_.end()) { + // The SOLE composition/materialization of this bigram's synthetic term + // (F03 single-store): hash_bigram_view(probe) == hash_term_bytes(composed) + // by construction (identical byte sequence through the same FNV update), + // so the interned id lands in the same bucket the probe searched. An + // EVICTED-then-reappearing pair misses the probe (eviction erased it) and + // re-interns here as a fresh term -- by then it is already in the + // ever-dropped bloom, so the flush drops it regardless of what its + // re-accumulated df grows to (the G04 completeness invariant). + term_id = intern_owned_term(format::make_phrase_bigram_term(left, right)); + } else { + term_id = *it; // the set element IS the term-id + } + accumulate(term_id, docid, pos); + // G04: amortized vocab-cap sweep step (bounded; a no-op two compares while + // the bigram intern storage is under the cap or the sweep is paused). + maybe_sweep_bigram_vocab(term_id); +} + +void SpimiTermBuffer::add_bigram_token(uint32_t left_id, uint32_t right_id, uint32_t docid, + uint32_t pos) { + // Same OWNED-vocab-mode contract (and failure latch) as the string paths. + if (vocab_ != &owned_vocab_) { + if (spill_status_.ok()) { + spill_status_ = Status::Error( + "spimi: add_bigram_token requires owned-vocab mode"); + } + return; + } + // G11: compute the pair key FIRST and prefetch the pair-map probe target + // (its ctrl + slot lines) so those DRAM fetches overlap the validation + // loads below (pair_of_[left_id] / pair_of_[right_id], themselves two + // scattered vocab-sized-array reads) instead of only starting once the + // probe executes. Pure hardware hint: no architectural side effects, the + // accumulated bytes are bit-identical. + const uint64_t pair_key = make_pair_key(left_id, right_id); + if (g11_prefetch_enabled()) { + bigram_pair_map_.prefetch(pair_key); + } + // The ids must name two ALREADY-INTERNED UNIGRAMS: an out-of-range id has no + // vocab string to materialize from, and a pair-term id as a constituent + // would compose a nested synthetic term. Both are caller bugs -- latch and + // ignore, mirroring add_token's out-of-range contract. (A string-keyed + // MARKER term as a constituent is equally wrong but costs a 20-byte memcmp + // to detect, so that stays a DCHECK: production feeds analyzer-token ids.) + if (left_id >= owned_vocab_.size() || right_id >= owned_vocab_.size() || + is_pair_term(left_id) || is_pair_term(right_id)) { + if (spill_status_.ok()) { + spill_status_ = Status::Error( + "spimi: add_bigram_token(left_id, right_id) requires interned unigram ids"); + } + return; + } + DCHECK(!format::is_phrase_bigram_term(owned_vocab_[left_id])); + DCHECK(!format::is_phrase_bigram_term(owned_vocab_[right_id])); + + // G05 hot path: ONE integer flat-map probe. No term bytes are hashed or + // compared, and nothing is allocated for a repeat pair (the ~per-token + // majority); a first-time pair interns a pair-map entry plus an EMPTY vocab + // slot -- the composed string is deferred to spill/flush materialization, + // and only for terms that must actually be emitted. + uint32_t term_id; + auto it = bigram_pair_map_.find(pair_key); + if (it == bigram_pair_map_.end()) { +#ifdef BE_TEST + // Seam only under BE_TEST: this is the per-token-pair hot path (billions of + // calls per import); an always-on shared atomic here cache-line-ping-pongs + // across concurrent writers (measured 35% of BE CPU on a 16-way load). + g_bigram_pair_map_misses.fetch_add(1, std::memory_order_relaxed); +#endif + // An EVICTED-then-reappearing pair misses here (eviction erased its + // entry) and re-interns as a fresh term -- by then its content hash is + // already in the ever-dropped bloom, so the flush drops it regardless of + // what its re-accumulated df grows to (the G04 completeness invariant, + // unchanged under pair keying). + term_id = intern_pair_term(pair_key); + } else { +#ifdef BE_TEST + g_bigram_pair_map_hits.fetch_add(1, std::memory_order_relaxed); +#endif + term_id = it->second; + // G11: start the term's slot-index line fetch the moment the id is + // known, ahead of accumulate()'s dependent slot_of_ -> slots_ -> arena + // chain walk. + if (g11_prefetch_enabled()) { + __builtin_prefetch(slot_of_.data() + term_id); + } + } + accumulate(term_id, docid, pos); + // G04: amortized vocab-cap sweep step, identical to the string bigram path. + maybe_sweep_bigram_vocab(term_id); +} + +// Shared owned-mode first-time-intern tail: stores `term_str` as the new id's +// vocab string (recycling an evicted id when one is free -- keeps the vocab +// vector, slot index and rank arrays from growing past the live vocabulary), +// inserts the id into the intern set, and accounts bigram intern storage +// against the G04 vocab cap. The string is stored BEFORE insert(term_id) so the +// set functors can hash owned_vocab_[term_id]; the set stores only the id, so a +// vocab reallocation never invalidates existing entries. G10 (flat set): an +// insert may REHASH the whole table -- re-hashing every STORED id through the +// functor -- which is safe because every stored id's string is live at every +// insert (eviction removes an id from the set before clearing its string) and a +// rehash relocates only the 4-byte id slots, never the strings themselves. +uint32_t SpimiTermBuffer::intern_owned_term(std::string&& term_str) { + uint32_t term_id; + if (!free_ids_.empty()) { + // Recycle an evicted id. Its old string was cleared at eviction and the + // id is in NO spill run (only never-spilled ids are evictable), so + // re-keying it cannot mis-attribute any run record. The in-place string + // change invalidates the cached lexicographic rank -> bump the epoch. + term_id = free_ids_.back(); + free_ids_.pop_back(); + owned_vocab_[term_id] = std::move(term_str); + ++vocab_epoch_; + } else { + term_id = static_cast(owned_vocab_.size()); + owned_vocab_.emplace_back(std::move(term_str)); + slot_of_.push_back(0); // vocab grows: new id starts with no live slot + } + // G08: credit the stored string's heap payload (0 for SSO) -- the header is + // charged via owned_vocab_.capacity(). evict_bigram_term debits it (symmetry). + owned_vocab_heap_bytes_ += StringHeapBytes(owned_vocab_[term_id]); + g_vocab_materializations.fetch_add(1, std::memory_order_relaxed); + intern_.insert(term_id); + if (bigram_diet_) { + const std::string& s = owned_vocab_[term_id]; + // The sentinel is exempt from the cap (never evictable; one per index). + if (format::is_phrase_bigram_term(s) && !format::is_phrase_bigram_sentinel_term(s)) { + bigram_intern_bytes_ += bigram_term_footprint(s); + } + } + return term_id; +} + +// G05 first-time intern of a pair key: assigns an owned-vocab id whose string +// stays EMPTY (the composed term is deferred to spill/flush materialization), +// records the id <-> pair-key mappings, and accounts the FIXED per-entry +// footprint against the vocab cap. NOT in intern_ (the content-keyed set) and +// NOT counted by the vocab-string-materialization seam -- no string exists yet. +// No vocab_epoch_ bump: a recycled id's string was already cleared (empty) at +// eviction and stays empty here, so no cached lexicographic rank changes. +uint32_t SpimiTermBuffer::intern_pair_term(uint64_t pair_key) { + uint32_t term_id; + if (!free_ids_.empty()) { + term_id = free_ids_.back(); + free_ids_.pop_back(); + DCHECK(owned_vocab_[term_id].empty()); + } else { + term_id = static_cast(owned_vocab_.size()); + owned_vocab_.emplace_back(); + slot_of_.push_back(0); // vocab grows: new id starts with no live slot + } + if (pair_of_.size() < owned_vocab_.size()) { + pair_of_.resize(owned_vocab_.size(), kNoPairKey); + } + pair_of_[term_id] = pair_key; + bigram_pair_map_.emplace(pair_key, term_id); + if (bigram_diet_) { + // Fixed bytes per pair-map entry (there is no string to measure); the + // pair key can never address the sentinel, so no exemption check. + bigram_intern_bytes_ += kBigramInternFixedOverheadBytes; + } + return term_id; +} + +uint64_t SpimiTermBuffer::pair_term_content_fnv(uint64_t pair_key) const { + const uint32_t left_id = static_cast(pair_key >> 32); + const uint32_t right_id = static_cast(pair_key); + // The unigram strings are pinned for the buffer's lifetime (only bigram + // terms are ever evicted/recycled), so resolving them here -- possibly long + // after the pair was interned -- is safe. + return bigram_view_fnv(PhraseBigramTermView {std::string_view(owned_vocab_[left_id]), + std::string_view(owned_vocab_[right_id])}); +} + +// The single point where a pair-keyed bigram term's on-disk string comes into +// existence. Bumps the vocab epoch (empty -> composed changes this id's +// lexicographic rank) and the materialization seam (this is the pair path's +// one-string-per-emitted-term analogue of intern_owned_term's bump). Cap +// accounting switches from the fixed pair footprint to the string footprint, +// mirroring what a string-keyed intern of the same term would carry. +void SpimiTermBuffer::materialize_pair_term(uint32_t id, uint64_t pair_key) { + const uint32_t left_id = static_cast(pair_key >> 32); + const uint32_t right_id = static_cast(pair_key); + owned_vocab_[id] = + format::make_phrase_bigram_term(owned_vocab_[left_id], owned_vocab_[right_id]); + // G08: the composed string's heap payload becomes resident here (the pair term + // owned no string before). Debited only when owned_vocab_ itself is released + // at the terminal drain -- materialized strings are pinned until then. + owned_vocab_heap_bytes_ += StringHeapBytes(owned_vocab_[id]); + if (bigram_diet_) { + const uint64_t fixed = kBigramInternFixedOverheadBytes; + bigram_intern_bytes_ = bigram_intern_bytes_ > fixed ? bigram_intern_bytes_ - fixed : 0; + bigram_intern_bytes_ += bigram_term_footprint(owned_vocab_[id]); + } + ++vocab_epoch_; + g_vocab_materializations.fetch_add(1, std::memory_order_relaxed); +} + +void SpimiTermBuffer::prepare_pair_terms_for_drain(bool evict_low_df_bigrams) { + if (bigram_pair_map_.empty()) { + return; + } + const bool evict = evict_low_df_bigrams && bigram_evict_enabled(); + // G06 drain-side df gate: the flush prune threshold process_term gates + // with -- EXACT by the time any final drain runs (LogicalIndexWriter:: + // build_blocks re-plumbs it), a safe lower bound of it during mid-feed + // spills (configure_bigram_diet). 0 == gate off: every live pair term + // materializes, the pre-G06 behavior. + const uint32_t min_df = bigram_drain_min_df_; + // Deferred mutation lists: evictions and df drops erase pair-map entries, + // so neither may run while the map is being iterated. Bounded by the live + // pair-term count. + std::vector evict_ids; + std::vector drop_ids; + for (const auto& [pair_key, id] : bigram_pair_map_) { + if (!owned_vocab_[id].empty()) { + continue; // materialized by an earlier spill; its string is final + } + if (slot_of_[id] == 0) { + continue; // defensive: no live postings to drain (see header note) + } + if (evict && bigram_evictable(id)) { + // Mid-feed spill: a df==1 pair term is EVICTED (bloomed, id + // recycled) instead of materialized -- writing it to the run would + // pin its id and (now) a composed string for the rest of the build. + // Identical rule, timing (spill) and bloom key (the synthetic term + // bytes' FNV) as the G04 string path's in-loop eviction. + evict_ids.push_back(id); + continue; + } + // G06: sink process_term's `df < bigram_prune_min_df` gate into the + // drain. Term::ndocs counts new-doc GROUPS: always >= the coalesced df, + // and EXACTLY the df process_term would compute while Term::sorted + // holds (a non-decreasing docid feed cannot revisit a docid + // non-adjacently). Dropping only when BOTH hold keeps the drop set + // identical to the flush gate's: + // * sorted && ndocs < min_df => final df == ndocs < min_df: the + // term exists only for process_term to prune AFTER paying its + // composed-string materialization, chain decode and emission (the + // wikipedia final-drain hot spot: ~hundreds of millions of df==1 / + // low-df pairs) -- drop it here without composing anything; + // * !sorted (rare out-of-order revisit feed, e.g. docids 5,1,5): + // ndocs may OVERCOUNT a revisited docid, so materialize + emit and + // let process_term gate the exact coalesced df -- correctness over + // savings. (Even trusting ndocs could never cause a WRONG drop -- + // it upper-bounds df, so ndocs < min_df implies df < min_df -- the + // sorted requirement is deliberate belt-and-braces so a drop only + // ever fires on a provably exact count.) + if (min_df > 0) { + const Term& t = slots_[slot_of_[id] - 1]; + if (t.sorted && t.ndocs < min_df) { + if (!evict_low_df_bigrams) { + // FINAL drain (terminal in-memory drain, or the residual + // spill inside merge_runs): no token can arrive after it, + // so the pair can never reappear -- drop WITHOUT bloom + // insertion, the same no-trace disposition process_term's + // df gate gives (no dict entry, no postings, no bloom + // membership; the reader's G01 pruned-segment contract + // covers the dict miss). + drop_ids.push_back(id); +#ifdef BE_TEST + g_bigram_drain_df_drops.fetch_add(1, std::memory_order_relaxed); +#endif + continue; + } + if (evict) { + // MID-FEED spill: same drop, but the pair MAY REAPPEAR + // after this spill with the docids dropped here missing -- + // record it in the ever-dropped bloom via the full G04 + // eviction (the rule that already governs the df==1 path + // above), so a reappearance is dropped at flush instead of + // materializing incomplete postings. + evict_ids.push_back(id); +#ifdef BE_TEST + g_bigram_drain_df_drops.fetch_add(1, std::memory_order_relaxed); +#endif + continue; + } + // Mid-feed spill WITHOUT the eviction machinery (no vocab + // cap): no bloom exists to make a reappearance safe, so fall + // through to materialize + emit and let process_term gate the + // final df -- correctness first. + } + } + // Survivor (df >= threshold, out-of-order-fed, or gate off): give it + // its on-disk bytes BEFORE any sort sees the id. + materialize_pair_term(id, pair_key); + } + for (uint32_t id : evict_ids) { + evict_bigram_term(id); + } + for (uint32_t id : drop_ids) { + release_pair_term(id, pair_of_[id]); + } +} + +namespace { + +// Reorders a term's flat arrays into ascending-docid order, COALESCING any +// same-docid groups so the result has exactly one entry per docid -- matching the +// k-way-merge path's boundary-doc coalescing and the writer's strictly-ascending +// precondition. Only invoked for the rare term that received out-of-order docids +// (the common ascending path leaves t.sorted true and skips it). +// +// A docid may REVISIT (e.g. feed 5,1,5): the chain holds two separate doc-groups +// for doc 5. A STABLE sort keeps equal-docid groups in arrival order, then the +// coalesce pass sums their freqs and concatenates their positions in that same +// (document/arrival) order -- so the merged positions stay consistent with the +// merged freqs, exactly as the run-order merge would have produced. +void SortByDocid(std::vector* docids, std::vector* freqs, + std::vector* positions_flat, bool has_positions) { + const size_t n = docids->size(); + std::vector order(n); + std::iota(order.begin(), order.end(), 0); + // STABLE so equal docids keep arrival order: their positions then concatenate in + // document order, the same order the merge path's run concatenation yields. + std::ranges::stable_sort(order, + [&](size_t a, size_t b) { return (*docids)[a] < (*docids)[b]; }); + + std::vector pos_off; + if (has_positions) { + pos_off.resize(n); + uint32_t running = 0; + for (size_t i = 0; i < n; ++i) { + pos_off[i] = running; + running += (*freqs)[i]; + } + } + std::vector nd, nf, np; + nd.reserve(n); + nf.reserve(n); + if (has_positions) { + np.reserve(positions_flat->size()); + } + for (size_t k : order) { + // Coalesce a revisited docid into the previous entry (it sorts adjacent now): + // sum freqs and append this group's positions right after the prior group's, + // so flat doc order stays partitioned by the merged freqs. + if (!nd.empty() && nd.back() == (*docids)[k]) { + nf.back() += (*freqs)[k]; + } else { + nd.push_back((*docids)[k]); + nf.push_back((*freqs)[k]); + } + if (has_positions) { + np.insert(np.end(), positions_flat->begin() + pos_off[k], + positions_flat->begin() + pos_off[k] + (*freqs)[k]); + } + } + *docids = std::move(nd); + *freqs = std::move(nf); + if (has_positions) { + *positions_flat = std::move(np); + } +} + +} // namespace + +namespace { + +// Decodes one varint from a pool chain cursor. The chain was written by +// encode_varint*, so the same LEB128 continuation-bit loop reconstructs it. +uint64_t DecodeChainVarint(CompactPostingPool::Cursor* c) { + uint64_t result = 0; + int shift = 0; + for (;;) { + const uint8_t b = c->next(); + result |= static_cast(b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + } + return result; +} + +void SkipChainVarint(CompactPostingPool::Cursor* c) { + DecodeChainVarint(c); +} + +} // namespace + +// Decodes a term's compact tagged chain back into a flat TermPostings (the exact +// docids/freqs/positions_flat the writer consumes), so the produced index is +// byte-identical to the legacy raw-uint32 accumulator. The chain holds one entry +// per token: varint((pos << 1) | new_doc); each new_doc entry is followed by a +// zigzag(docid-delta). A doc's freq is the run length of consecutive same-doc +// tokens; positions stream out in document order (empty when positions disabled). +// Stream positions for a sorted term whose token count exceeds this: such a term's +// flat positions buffer (uint32 per token) would be the peak-RSS transient (tens of +// MiB for the widest term). Below it, the flat buffer is cheap and simpler. +static constexpr uint32_t kStreamPositionsTokenThreshold = 1U << 16; // 65536 + +TermPostings SpimiTermBuffer::to_postings(std::string term, Term&& t, + bool allow_stream_positions) const { + TermPostings tp; + tp.term = std::move(term); + if (t.ntok == 0 || t.head == kNoChain) { + return tp; + } + + // Reserve docids/freqs by ndocs -- EXACT: the decode loop pushes one entry per + // new_doc tag, which is precisely what t.ndocs counts (SortByDocid may then + // coalesce a revisited docid, only ever shrinking). Replaces the former ntok + // upper-bound reserve (one slot per token, loose for freq>1 docs). + tp.docids.reserve(t.ndocs); + tp.freqs.reserve(t.ndocs); + + // Per-TERM positions capability: a G04 position-suppressed bigram term + // stored no position payload (its chain holds bare new_doc tags, identical + // to a positions-disabled encoding), so it decodes -- and is emitted -- as a + // docs+freq-only term: positions_flat stays empty and no pos_pump is ever + // built. The flush path never writes .prx for it (write_prx false whenever + // pruning is active), so nothing downstream misses the bytes. + const bool term_has_pos = has_positions_ && !t.pos_suppressed; + + // For a large SORTED term, stream positions on demand instead of materializing a + // multi-MiB flat buffer: the writer (prx builder) pulls them window by window via + // pos_pump, decoding straight from the still-resident arena chain. Out-of-order + // terms (rare, defensive) need a full sort, so they always use the flat path. + const bool stream_pos = allow_stream_positions && term_has_pos && t.sorted && + t.ntok >= kStreamPositionsTokenThreshold; + if (term_has_pos && !stream_pos) { + tp.positions_flat.reserve(t.ntok); + } + + CompactPostingPool::Cursor c = pool_.cursor(t.head, t.w.cur); + int64_t prev = 0; + for (uint32_t i = 0; i < t.ntok; ++i) { + const uint64_t tagged = DecodeChainVarint(&c); + const bool new_doc = (tagged & 1U) != 0; + if (new_doc) { + prev += zigzag_decode(DecodeChainVarint(&c)); + tp.docids.push_back(static_cast(prev)); + tp.freqs.push_back(0); + } + ++tp.freqs.back(); // count this token toward the current doc's freq + if (term_has_pos && !stream_pos) { + tp.positions_flat.push_back(static_cast(tagged >> 1)); + } + } + + // Decide the FINAL position handling now that df (= docids.size()) is known. + // pos_pump is honored ONLY by the windowed writer path (build_windowed_entry), + // taken when df >= kSlimDfThreshold. A SLIM term (df below it) goes through + // build_slim_entry, which reads positions_flat directly -- so streaming would + // leave it empty and crash. A high-ntok but low-df term (many repeats in few + // docs) therefore falls back to materializing its df-bounded positions here. + const bool windowed_path = tp.docids.size() >= format::kSlimDfThreshold; + if (stream_pos && windowed_path) { + // Hand the writer a sequential position source backed by a SECOND pass over the + // same chain (the chain stays resident in pool_ for the whole drain). The pump + // yields positions in document order -- identical to positions_flat -- so the + // produced .prx is byte-for-byte the same. The cursor is shared/advanced across + // calls (the writer pulls in order, exactly pos_total positions total). + tp.pos_total = t.ntok; + auto cur = std::make_shared(pool_.cursor(t.head, t.w.cur)); + tp.pos_pump = [cur](uint32_t* dst, size_t count) { + // Re-walk the tagged token stream, yielding one position per token. A new-doc + // token is followed by a zigzag docid-delta varint that must be consumed and + // discarded so the cursor stays aligned with the encoding. + for (size_t k = 0; k < count; ++k) { + const uint64_t tagged = DecodeChainVarint(cur.get()); + if ((tagged & 1U) != 0) { + SkipChainVarint(cur.get()); + } + dst[k] = static_cast(tagged >> 1); + } + }; + } else if (stream_pos) { + // Slim fallback: the decode loop skipped positions (stream candidate) but the + // term is slim, so materialize positions_flat in a second pass for build_slim. + // (stream_pos implies term_has_pos, so a suppressed term never lands here.) + tp.positions_flat.reserve(t.ntok); + CompactPostingPool::Cursor pc = pool_.cursor(t.head, t.w.cur); + for (uint32_t i = 0; i < t.ntok; ++i) { + const uint64_t tagged = DecodeChainVarint(&pc); + if ((tagged & 1U) != 0) { + SkipChainVarint(&pc); + } + tp.positions_flat.push_back(static_cast(tagged >> 1)); + } + } else if (!t.sorted) { + // Defensive reorder for the rare out-of-order-docid feed (merge of pre-sorted + // runs). The common ascending path leaves t.sorted true and skips it. + // term_has_pos (not the buffer flag): a suppressed term's positions_flat is + // empty, and indexing it by freqs would read out of range. + SortByDocid(&tp.docids, &tp.freqs, &tp.positions_flat, term_has_pos); + } + return tp; +} + +void SpimiTermBuffer::ensure_string_rank() const { + const std::vector& v = vocab(); + if (string_rank_.size() == v.size() && string_rank_epoch_ == vocab_epoch_) { + return; // already built (or empty vocab) and no id's string mutated since + } + // One full lexicographic sort of the vocabulary, amortized over every spill. + // Rebuilt when the vocab GREW or when G04 eviction/recycling mutated an + // existing id's string in place (epoch mismatch) -- a stale rank would order + // spill runs / the k-way merge by the id's OLD string. + std::vector order(v.size()); + std::iota(order.begin(), order.end(), 0U); + std::ranges::sort(order, [&](uint32_t a, uint32_t b) { return v[a] < v[b]; }); + string_rank_.assign(v.size(), 0U); + for (uint32_t rank = 0; rank < order.size(); ++rank) { + string_rank_[order[rank]] = rank; + } + string_rank_epoch_ = vocab_epoch_; +} + +std::vector SpimiTermBuffer::sorted_ids() const { + ensure_string_rank(); + std::vector ids = touched_ids_; + const std::vector& rank = string_rank_; + // Integer rank compare instead of full std::string compare: equal-string ids + // cannot occur for a dense vocab, so a strict rank order matches the original + // lexicographic order exactly. + std::ranges::sort(ids, [&](uint32_t a, uint32_t b) { return rank[a] < rank[b]; }); + return ids; +} + +void SpimiTermBuffer::release_term(uint32_t term_id) { + const uint32_t enc = slot_of_[term_id]; + if (enc == 0) { + return; // not live (defensive) + } + const uint32_t slot = enc - 1; + slots_[slot] = Term(); // free this term's arrays; the empty Term slot is reusable + free_slots_.push_back(slot); + slot_of_[term_id] = 0; + --live_term_count_; +} + +bool SpimiTermBuffer::bigram_evictable(uint32_t id) const { + if (id >= slot_of_.size()) { + return false; // defensive: slot index freed (post-drain) or stale cursor + } + const uint32_t enc = slot_of_[id]; + if (enc == 0) { + return false; // no live postings: already drained (in-run) or evicted + } + // ndocs can only OVERcount an out-of-order feed, so ndocs == 1 still means + // df == 1 exactly -- a df>=2 term can never look evictable. + if (slots_[enc - 1].ndocs != 1) { + return false; // df >= 2: past the Zipf tail, never evicted (hot/warm) + } + if (id < id_in_run_.size() && id_in_run_[id] != 0) { + return false; // a spill run references this id: string is pinned forever + } + // A G05 pair-keyed term is a real (non-sentinel) phrase bigram by + // construction -- the pair key can never address the sentinel -- and its + // vocab string is empty, so it is recognized by pair-map membership, not by + // the marker prefix. (A pair term MATERIALIZED by an earlier spill is + // in-run, caught above.) + if (is_pair_term(id)) { + return true; + } + const std::string& s = owned_vocab_[id]; + // The sentinel gates reader semantics ("bigram feature present") and must + // never be dropped; everything non-bigram is out of scope. + return format::is_phrase_bigram_term(s) && !format::is_phrase_bigram_sentinel_term(s); +} + +// Evicts one eligible bigram id. The caller established eligibility: either +// bigram_evictable (the G04 df==1 cap-sweep / spill rule) or the G06 mid-feed +// df-gate drop in prepare_pair_terms_for_drain (a live, never-spilled, +// not-yet-materialized PAIR term whose exact df -- possibly >= 2 -- is below +// the drain threshold; blooming it here is precisely what makes a +// post-spill reappearance safe). Steps: +// 1. record the term's SYNTHETIC BYTES in the ever-dropped bloom -- the +// flush-time process_term will drop this term even if the pair reappears +// and re-accumulates past the df threshold (its postings would be missing +// the docid dropped here). A string-keyed term inserts its stored string; +// a G05 pair-keyed term inserts the SAME key via the piecewise content +// FNV of its two unigram strings (never composing the term), so the +// flush-time string probe agrees bit-for-bit either way; +// 2. drop the in-memory postings (release_term; the term's arena chain bytes +// become dead until the next pool reset -- an amortized cost, reclaimed at +// the next spill/drain); +// 3. erase the id from its interning table (the content-keyed intern set for +// string terms, the pair map for pair terms), so a reappearing pair +// re-interns as a FRESH term instead of resurrecting the dropped one; +// 4. recycle the id (bounding owned_vocab_ / slot_of_ / string_rank_ to the +// live vocabulary). For a string term this also frees the vocab string and +// bumps the vocab epoch so the cached lexicographic rank is rebuilt before +// the next spill sort; a pair term's vocab string was empty and stays +// empty, so no rank-visible state changes and the epoch is untouched. +void SpimiTermBuffer::evict_bigram_term(uint32_t id) { + if (bigram_drop_filter_ == nullptr) { + // First eviction: size partition 0 from the cap -- roughly the live-term + // count the cap can hold at the fixed-overhead estimate, i.e. the upper + // bound on evictions a single full sweep can produce. + bigram_drop_filter_ = std::make_unique(bigram_vocab_cap_bytes_ / + kBigramInternFixedOverheadBytes); + } + if (is_pair_term(id)) { + const uint64_t pair_key = pair_of_[id]; + bigram_drop_filter_->insert_hashed(pair_term_content_fnv(pair_key)); + release_pair_term(id, pair_key); + g_bigram_evictions.fetch_add(1, std::memory_order_relaxed); + return; + } + std::string& s = owned_vocab_[id]; + bigram_drop_filter_->insert(s); + const uint64_t footprint = bigram_term_footprint(s); + bigram_intern_bytes_ = bigram_intern_bytes_ > footprint ? bigram_intern_bytes_ - footprint : 0; + // G08 symmetry: debit the heap payload intern_owned_term credited, BEFORE the + // swap below frees it (the intern_.erase debits its entry via intern_.size()). + const uint64_t heap = StringHeapBytes(s); + owned_vocab_heap_bytes_ = owned_vocab_heap_bytes_ > heap ? owned_vocab_heap_bytes_ - heap : 0; + // ORDER (G10 load-bearing): erase from the intern set while the string is + // still intact -- erase-by-key hashes owned_vocab_[id] through the functor to + // locate the slot (true for the old node set too; the flat set makes it worth + // stating). The erase tombstones one 4-byte slot and re-hashes nothing else. + intern_.erase(id); + release_term(id); + std::string().swap(s); // free the string payload (capacity 0) + free_ids_.push_back(id); + ++vocab_epoch_; + g_bigram_evictions.fetch_add(1, std::memory_order_relaxed); +} + +// Shared final release of a live, not-yet-materialized pair-keyed bigram term: +// the tail of a G04 pair eviction (the caller has ALREADY bloomed the content +// hash) and the WHOLE of a G06 final-drain df drop (deliberately no bloom -- +// nothing can reappear after the final drain, and blooming would only add +// false-positive pressure on surviving hot pairs). Un-accounts the fixed pair +// footprint, erases both pair mappings (so a later probe/intern of the pair +// key cannot resurrect the id), releases the slot/postings and recycles the +// id. The id's vocab string was empty and stays empty, so no cached +// lexicographic rank changes and vocab_epoch_ is untouched (mirrors +// intern_pair_term). +void SpimiTermBuffer::release_pair_term(uint32_t id, uint64_t pair_key) { + DCHECK(is_pair_term(id)); + DCHECK(owned_vocab_[id].empty()); + const uint64_t fixed = kBigramInternFixedOverheadBytes; + bigram_intern_bytes_ = bigram_intern_bytes_ > fixed ? bigram_intern_bytes_ - fixed : 0; + bigram_pair_map_.erase(pair_key); + pair_of_[id] = kNoPairKey; + release_term(id); + free_ids_.push_back(id); +} + +void SpimiTermBuffer::maybe_sweep_bigram_vocab(uint32_t just_touched_id) { + if (!bigram_evict_enabled() || bigram_intern_bytes_ <= bigram_vocab_cap_bytes_) { + return; // feature off or under the cap: two compares, no work + } + // Fruitless-lap pause: when a full lap over the vocabulary evicted nothing + // (everything over the cap is df>=2 or run-pinned -- nothing legal to drop), + // sweeping again is pure waste until the tail GROWS. sweep_rearm_bytes_ was + // set to current-bytes + a delta at pause time; stay parked until then. + if (bigram_intern_bytes_ < sweep_rearm_bytes_) { + return; + } + g_vocab_cap_sweeps.fetch_add(1, std::memory_order_relaxed); + const size_t vocab_size = owned_vocab_.size(); + uint64_t evicted = 0; + for (uint32_t scanned = 0; + scanned < kVocabSweepStride && bigram_intern_bytes_ > bigram_vocab_cap_bytes_; ++scanned) { + if (sweep_cursor_ >= vocab_size) { + sweep_cursor_ = 0; // circular scan; ids interned mid-lap are caught next lap + } + const uint32_t id = sweep_cursor_++; + // Never evict the term the CURRENT add just extended: its df may be + // about to grow (a recurring pair's occurrences often arrive + // back-to-back), and skipping it guarantees such a pair reaches df==2 + // immunity instead of being churned at df==1 by its own add's sweep -- + // decisive when the live vocabulary is no larger than the stride. + if (id == just_touched_id) { + continue; + } + if (bigram_evictable(id)) { + evict_bigram_term(id); + ++evicted; + } + } + if (evicted != 0) { + sweep_scanned_since_evict_ = 0; + sweep_rearm_bytes_ = 0; + return; + } + sweep_scanned_since_evict_ += kVocabSweepStride; + if (sweep_scanned_since_evict_ >= vocab_size) { + // One full fruitless lap: pause until the bigram intern storage grows by + // a meaningful delta (deterministic; scaled to the cap with a small floor + // so tiny test caps still re-arm). + const uint64_t delta = std::max(4096, bigram_vocab_cap_bytes_ / 64); + sweep_rearm_bytes_ = bigram_intern_bytes_ + delta; + sweep_scanned_since_evict_ = 0; + } +} + +Status SpimiTermBuffer::drain_sorted(const std::function& fn, + bool allow_stream_positions) { + // G05/G06: give every SURVIVING pair-keyed bigram term its composed on-disk + // string BEFORE sorted_ids() ranks anything. No bloomed eviction (this is + // the terminal in-memory drain), but the G06 df gate drops pair terms whose + // EXACT df is below the flush prune threshold right here -- without + // composing their strings or decoding their postings -- since process_term + // would prune them identically after paying for all of that. Everything + // else (string-keyed terms, out-of-order-fed pair terms) still reaches + // process_term's unchanged df gate. + prepare_pair_terms_for_drain(/*evict_low_df_bigrams=*/false); + const std::vector& v = vocab(); + for (uint32_t id : sorted_ids()) { + const uint32_t enc = slot_of_[id]; + if (enc == 0) { + // G04: touched_ids_ may hold ids whose slot is gone -- evicted bigram + // terms (dropped + bloom-recorded) and the stale first entry of a + // recycled id appearing twice (duplicates sort adjacently; the live + // slot drains exactly once). + continue; + } + Term term = slots_[enc - 1]; + release_term(id); // release this term's slot before building the next + // Allow streaming positions only when the caller consumes synchronously (the + // arena chain stays resident for the whole drain, so the pump can read from it). + TermPostings tp = to_postings(v[id], std::move(term), allow_stream_positions); + fn(std::move(tp)); + } + // Drop the arena + the slot pool (their bytes are fully decoded) and return the + // freed chunks to the OS so the process peak reflects only what survives the + // drain, not retained input-phase arena memory. The G05 pair map / reverse + // pair-key slots are equally dead after a terminal drain (every pair term is + // materialized into owned_vocab_, which the emitted strings copied from). + // G08: the terminal drain also releases every structure resident_bytes() now + // charges and that nothing consumes past this point -- the touched list, the + // cached rank, the G04 bookkeeping, and (owned mode; every emitted term COPIED + // its string above) the owned vocabulary + intern set. Only the drop bloom + // survives: the flush-time process_term backstop still probes it. + std::vector().swap(touched_ids_); + pool_.reset(); + std::vector().swap(slots_); + std::vector().swap(free_slots_); + std::vector().swap(slot_of_); + phmap::flat_hash_map().swap(bigram_pair_map_); + std::vector().swap(pair_of_); + std::vector().swap(free_ids_); + std::vector().swap(id_in_run_); + std::vector().swap(string_rank_); + intern_ = decltype(intern_)(0, OwnedVocabHash {&owned_vocab_}, OwnedVocabEq {&owned_vocab_}); + std::vector().swap(owned_vocab_); + owned_vocab_heap_bytes_ = 0; + TrimMalloc(); + // Everything charged is now freed: real resident ~0, so this emits the final + // negative that returns every reported byte (no leak after the in-memory drain). + report_arena_delta(); + return Status::OK(); +} + +Status SpimiTermBuffer::drain_to_writer(RunWriter* w, bool evict_low_df_bigrams) { + Status st = Status::OK(); + // G05: pair-keyed bigram terms must carry their composed string BEFORE + // sorted_ids() ranks this spill (run records pin the id; the k-way merge + // orders by the vocab string). On mid-feed spills (evict_low_df_bigrams) + // df==1 pair terms are evicted here instead of materialized -- the same + // rule the in-loop check below applies to string-keyed bigram terms. + prepare_pair_terms_for_drain(evict_low_df_bigrams); + const std::vector& v = vocab(); + const bool evict = evict_low_df_bigrams && bigram_evict_enabled(); + if (evict && id_in_run_.size() < v.size()) { + // Lazily size the run-pin map at the first evict-enabled spill (1 B/id, + // bounded because the cap bounds the vocabulary). + id_in_run_.resize(v.size(), 0); + } + // Spill writes by term-id (no string IO). Iterate touched ids in vocab-string + // order so each run is sorted; the k-way merge re-orders runs by the same key. + for (uint32_t id : sorted_ids()) { + const uint32_t enc = slot_of_[id]; + if (enc == 0) { + continue; // evicted mid-feed / stale duplicate of a recycled id (see drain_sorted) + } + if (evict && bigram_evictable(id)) { + // G04 spill-side eviction: a df==1 bigram written to this run would + // PIN its vocab string for the rest of the build (run records key on + // the id, so an in-run id can never be evicted or recycled) -- across + // tens of spills the pinned tail would defeat the cap entirely. Drop + // it here instead: bloom-recorded, so if the pair reappears later its + // re-interned term is dropped at flush; if it never reappears the + // flush-time df threshold would have dropped it anyway (df 1 < any + // active threshold's minimum of 1... except threshold==1, which the + // bloom drop covers -- over-drop, safe under the fallback contract). + evict_bigram_term(id); + continue; + } + if (evict) { + id_in_run_[id] = 1; // this run now references the id: pinned for good + } + Term term = slots_[enc - 1]; + release_term(id); + // Spill path: the run codec serializes positions_flat directly, so positions + // must be materialized (no streaming pump). A G04 position-suppressed + // bigram term serializes an EMPTY position block (n_pos == 0), which the + // run codec and merge handle per-term. + TermPostings tp = to_postings(v[id], std::move(term), /*allow_stream=*/false); + if (st.ok()) { + st = w->write_term(id, tp); + } + } + touched_ids_.clear(); + pool_.reset(); // all chains decoded into the run; free the arena for the refill + // The spill returns the arena to 0; slot_of_ keeps its capacity (survives + // the spill). Report the arena-drop negative now so the gate-2 spill is balanced + // immediately, not deferred to the next token. + report_arena_delta(); + return st; +} + +Status SpimiTermBuffer::compact_runs() { + if (run_paths_.size() < 2) { + return Status::OK(); + } + // The compaction keys its k-way heap on the same term-id -> lexicographic + // rank the per-spill sorts and the final merge use. Safe across spills: + // every id present in a run file is PINNED (never evicted/recycled -- + // id_in_run_ / the pair-term materialize-at-spill rule), so its vocab + // string -- and therefore its rank order relative to every other in-run + // id -- is immutable from the moment it was first written. + ensure_string_rank(); + const std::string out_path = MakeRunPath(resolve_temp_dir()); + Status s = CompactRuns(run_paths_, string_rank_, has_positions_, out_path); + if (!s.ok()) { + std::remove(out_path.c_str()); // drop the partial output; inputs intact + return s; + } + // The compacted run REPLACES its inputs at the FRONT of the run order: + // it holds exactly runs [0..n) merged in run order, and any later run only + // covers strictly-later docids, so per-term run-order concatenation (the + // k-way merge invariant) is preserved. + for (const std::string& p : run_paths_) { + std::remove(p.c_str()); + } + run_paths_.clear(); + run_paths_.push_back(out_path); + g_run_compactions.fetch_add(1, std::memory_order_relaxed); + return Status::OK(); +} + +Status SpimiTermBuffer::spill_to_run(bool evict_low_df_bigrams) { + // G09 run-file cap: a buffer must never accumulate unbounded run files -- + // the final k-way merge (re)opens ALL of them simultaneously and holds + // the fds for its whole duration, so unbounded runs across ~100 + // concurrent writers exhausted the BE nofile rlimit ('Too many open + // files' at run reopen). At the cap, merge-compact the existing runs into + // one before cutting the new run: the merge fan-in (and its fd count) is + // bounded by cap + 1 per buffer. + if (max_run_files_ != 0 && run_paths_.size() >= max_run_files_) { + RETURN_IF_ERROR(compact_runs()); + } + const std::string dir = resolve_temp_dir(); + // Best-effort space pre-check: fail with a clear, early error rather than a + // mid-write IoError that leaves a half-written run. Best-effort only (TOCTOU; on + // tmpfs this reports RAM). The ARENA -- not full resident_bytes(), which since + // G08 also charges vocab/pair structures a run never contains -- is what the + // run re-encodes, and its block slack makes it a conservative over-estimate of + // the run's on-disk size. + const uint64_t arena = pool_.arena_bytes(); + const uint64_t avail = temp_dir_available_bytes(dir); + if (avail < arena) { + return Status::Error( + "spimi: insufficient temp space in '" + dir + "' to spill ~" + + std::to_string(arena) + " B (~" + std::to_string(avail) + + " B free); set SNII_TEMP_DIR/TMPDIR to a larger disk"); + } + const std::string path = MakeRunPath(dir); + RunWriter w; + RETURN_IF_ERROR(w.open(path)); + run_paths_.push_back(path); // tracked for cleanup even if a later step fails + RETURN_IF_ERROR(drain_to_writer(&w, evict_low_df_bigrams)); + // drain emptied touched_ids_ and freed each term's arrays; terms_/present_ keep + // their (vocab-sized) capacity so the next fill reuses the dense slots with no + // re-allocation. present_ is already all-zero after release_term per id. + return w.close(); +} + +Status SpimiTermBuffer::merge_runs(const std::function& fn, + bool allow_stream_positions) { + // Flush whatever is still resident as one final sorted run so the k-way merge + // sees a uniform set of run files (and never holds two term sources at once). + // NO eviction on this FINAL residual spill: no token can arrive after it, so + // its df==1 bigrams are dropped by the flush-time df threshold regardless; + // blooming them here would only inflate the filter (false-positive pressure + // on hot survivors) and, at an explicit threshold of 1, wrongly drop terms + // the control build would materialize. The G06 drain-side df gate DOES apply + // (evict_low_df_bigrams=false selects its final-drain arm): never-spilled + // pair terms whose exact df is below the flush threshold are dropped + // unbloomed instead of being materialized into the run only for + // process_term to prune them after the merge. Pair terms materialized by an + // EARLIER spill flow through the runs untouched -- their per-run partial dfs + // only process_term can total, so it stays their gate. + if (!touched_ids_.empty()) { + Status s = spill_to_run(/*evict_low_df_bigrams=*/false); + if (!s.ok() && spill_status_.ok()) { + spill_status_ = s; + } + } + if (!spill_status_.ok()) { + return spill_status_; // a spill or add_token error; emit nothing + } + // All terms are now spilled; the merge reads runs and never touches the + // accumulators. Free the pool + the vocab-sized slot index so the merge phase + // holds none of the input-side arrays resident -- keeps spill-mode peak RSS + // down. The G05 pair map is dead too: the residual spill materialized every + // live pair term (no more adds can arrive), so only the (still-needed) vocab + // strings survive into the merge. malloc_trim(0) returns the freed glibc + // arenas to the OS so the peak RSS measurement reflects the merge transient, + // not retained input-phase chunks. + std::vector().swap(slots_); + std::vector().swap(free_slots_); + std::vector().swap(slot_of_); + phmap::flat_hash_map().swap(bigram_pair_map_); + std::vector().swap(pair_of_); + // G08: the touched list (cleared by the residual spill but capacity-retained) + // and the G04 bookkeeping are dead here too. The owned vocab, intern set and + // rank array must SURVIVE into MergeRuns (it keys the heap on them) -- they + // are released right after the merge below. + std::vector().swap(touched_ids_); + std::vector().swap(free_ids_); + std::vector().swap(id_in_run_); + TrimMalloc(); + // pool_ was already reset by the final spill_to_run -> drain_to_writer (reported + // there); these swaps free the slot index + bookkeeping, so report the remaining + // negative now. The vocab-side remainder is reported after the merge. + report_arena_delta(); + // The k-way merge keys its heap/gather on the term-id -> lexicographic rank array + // instead of comparing vocab strings. Build it explicitly here (idempotent -- every + // spill already builds it via sorted_ids(), and merge_runs is only reached after at + // least one spill, but the explicit call keeps the rank fresh and sized to the vocab + // even if a future caller path reaches the merge without a prior spill). + ensure_string_rank(); + Status s = MergeRuns(run_paths_, vocab(), string_rank_, has_positions_, fn, + allow_stream_positions); + // G08: the merge was the LAST consumer of the vocab strings, the intern set + // and the rank array (every emitted term copied its string). Release them and + // report the final negative so the post-drain resident (and the writer's + // unified total) reflects only what actually survives -- the drop bloom, which + // the flush-time process_term backstop still probes, stays. + intern_ = decltype(intern_)(0, OwnedVocabHash {&owned_vocab_}, OwnedVocabEq {&owned_vocab_}); + std::vector().swap(owned_vocab_); + owned_vocab_heap_bytes_ = 0; + std::vector().swap(string_rank_); + report_arena_delta(); + // The merge churns one large coalesced TermPostings per term (the widest term's + // arrays are tens of MiB) plus per-run reader windows; on completion glibc + // retains those freed chunks in its arenas. Trim again so the post-merge resident + // set (and thus the process peak high-water if a later phase allocates) reflects + // only live state, not merge-transient retention. + TrimMalloc(); + return s; +} + +Status SpimiTermBuffer::for_each_term_sorted(const std::function& fn) { + // Single-drain contract: a second call would re-merge the (still-present) run + // files and re-emit every term, or emit nothing in the in-memory path. Return + // an error and emit NOTHING rather than produce a wrong second stream. + if (drained_) { + return Status::Error( + "spimi: already drained (single-drain contract)"); + } + drained_ = true; + // The callback is invoked synchronously while the arena is resident, so large + // sorted terms may stream positions via pos_pump (peak-RSS win for the writer). + if (run_paths_.empty() && spill_status_.ok()) { + return drain_sorted(fn, /*allow_stream_positions=*/true); // pure in-memory path + } + // Spilled path (or add_token latched a validation error): the merge may STREAM + // a wide term's positions via pos_pump (fn consumes each term synchronously + // while the run readers stay parked). merge_runs returns the I/O status + // directly; add_token validation errors surface via spill_status_ inside it. + return merge_runs(fn, /*allow_stream_positions=*/true); +} + +std::vector SpimiTermBuffer::finalize_sorted() { + std::vector out; + // Single-drain contract (mirrors for_each_term_sorted): a second drain (including + // a finalize_sorted after a for_each_term_sorted, or vice versa) would re-emit or + // emit nothing. Latch an error and return EMPTY rather than a wrong result. + if (drained_) { + if (spill_status_.ok()) { + spill_status_ = Status::Error( + "spimi: already drained (single-drain contract)"); + } + return out; + } + drained_ = true; + out.reserve(touched_ids_.size()); + // RETAINS each TermPostings past the drain, so positions must be MATERIALIZED + // (a streamed pos_pump would reference the arena, freed when the drain ends). + if (run_paths_.empty() && spill_status_.ok()) { + Status s = drain_sorted([&out](TermPostings&& tp) { out.push_back(std::move(tp)); }, + /*allow_stream_positions=*/false); + if (!s.ok() && spill_status_.ok()) { + spill_status_ = s; + } + } else { + // RETAINS each TermPostings past the merge, so positions MUST be materialized + // (a streamed pos_pump would reference run readers freed when the merge ends). + Status s = merge_runs([&out](TermPostings&& tp) { out.push_back(std::move(tp)); }, + /*allow_stream_positions=*/false); + if (!s.ok() && spill_status_.ok()) { + spill_status_ = s; + } + } + return out; +} + +void SpimiTermBuffer::cleanup_runs() { + for (const std::string& p : run_paths_) { + std::remove(p.c_str()); + } + run_paths_.clear(); +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/spimi_term_buffer.h b/be/src/storage/index/snii/writer/spimi_term_buffer.h new file mode 100644 index 00000000000000..9f8c82987f046b --- /dev/null +++ b/be/src/storage/index/snii/writer/spimi_term_buffer.h @@ -0,0 +1,1032 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/writer/bigram_drop_filter.h" +#include "storage/index/snii/writer/compact_posting_pool.h" +#include "storage/index/snii/writer/memory_reporter.h" + +namespace doris::snii::writer { + +// G11: compiled-in marker for the per-token prefetch candidate (the locality +// bench keys its in-process A/B test off this). +#define SNII_G11_PREFETCH 1 + +class GlobalMemoryLimiter; // G09 process-wide build-RAM registry (see below) + +// Heterogeneous probe key for a hidden phrase-bigram term (G01 part C): the +// synthetic term make_phrase_bigram_term(left, right) addressed by its two +// fragments, WITHOUT composing the string. The intern-set functors hash and +// compare it piecewise (marker / varint(len(left)) / left / right), so a repeat +// word pair costs zero allocation and zero byte copies on the add hot path. +struct PhraseBigramTermView { + std::string_view left; + std::string_view right; +}; + +// One term's posting list: docids ascending, with parallel freqs and (when +// positions are enabled) a single FLAT positions buffer. +// +// positions_flat holds every position for the term in document order, partitioned +// by freqs: doc i owns the next freqs[i] entries. This is the SAME layout the +// accumulator stores natively, so no per-doc vector-of-vectors is ever built on +// the build/merge hot path (that vector-of-vectors was the dominant peak-RSS +// driver for high-df terms). doc_positions(i) returns a non-owning span view of +// doc i's positions for consumers that want per-doc access (e.g. the prx window +// builder, tests). positions_flat is empty when positions are disabled. +struct TermPostings { + std::string term; + std::vector docids; + std::vector freqs; + std::vector positions_flat; // empty when positions disabled + + // OPTIONAL streamed-positions source (peak-RSS optimization for very-high-df + // terms). When set, positions_flat is left EMPTY and the writer pulls positions + // SEQUENTIALLY in document order via pos_pump(dst, n) -- filling `dst[0..n)` with + // the next n positions -- one window at a time, so the term's full flat positions + // buffer (tens of MiB for the widest term) is never materialized. The yielded + // bytes are byte-identical to building from positions_flat (same values, same + // order). pos_total is the total number of positions the pump will yield (== + // sum(freqs)); it lets the writer validate without a flat buffer. When pos_pump + // is null, positions come from positions_flat as before. Only the writer's prx + // builders consume this; all other consumers use positions_flat. + // + // OWNERSHIP CONTRACT (synchronous-consume-once): a streamed pos_pump captures + // references into the producer's stack and its parked run readers/arena, valid ONLY + // for the duration of the synchronous fn(TermPostings&&) call that delivered this + // TermPostings. The consumer MUST pull all positions inside fn() and MUST NOT store + // the TermPostings or invoke pos_pump after fn() returns. Callers that retain the + // TermPostings pass allow_stream_positions=false, which materializes positions into + // positions_flat instead (no pump). As a safety net, a deferred call to a streamed + // pump throws std::logic_error rather than dereferencing freed state. + std::function pos_pump; + uint64_t pos_total = 0; + + // Byte offset of doc i's first position within positions_flat (prefix sum of + // freqs). O(i) -- callers iterating all docs should track a running offset. + size_t pos_offset(size_t doc_index) const { + size_t off = 0; + for (size_t i = 0; i < doc_index; ++i) { + off += freqs[i]; + } + return off; + } + // Non-owning view of doc i's positions (length freqs[i]) into positions_flat. + std::span doc_positions(size_t doc_index) const { + const size_t off = pos_offset(doc_index); + return {positions_flat.data() + off, freqs[doc_index]}; + } + + // Rebuilds the per-doc position lists (for callers/tests wanting per-doc access) + // from positions_flat partitioned by freqs. O(total positions); allocates. + std::vector> positions_per_doc() const { + std::vector> out(freqs.size()); + size_t off = 0; + for (size_t i = 0; i < freqs.size(); ++i) { + out[i].assign(positions_flat.begin() + off, positions_flat.begin() + off + freqs[i]); + off += freqs[i]; + } + return out; + } + + // Sets the flat positions from per-doc lists (convenience for tests / callers + // that produce per-doc positions). Does NOT touch freqs; the caller is expected + // to keep freqs[i] == per_doc[i].size() consistent (the writer validates this). + void set_positions_per_doc(const std::vector>& per_doc) { + positions_flat.clear(); + for (const auto& d : per_doc) { + positions_flat.insert(positions_flat.end(), d.begin(), d.end()); + } + } +}; + +// In-memory SPIMI (Single-Pass In-Memory Indexing) accumulator for one logical +// index. Records term occurrences and produces lexicographically sorted terms +// with ascending-docid posting lists. +// +// TERM-ID ACCUMULATION (no per-token string work): tokens are accumulated by an +// INTEGER term-id, not by hashing/constructing a std::string per token. The +// caller supplies a VOCABULARY mapping term-id -> term string; the buffer keeps +// a DENSE std::vector indexed by term-id, so the hot add_token path is a +// vector index + a couple of pushes -- no hashing, no allocation per token. The +// vocabulary is resolved to strings only once per distinct term at finalize. +// +// Two construction modes: +// * BORROWED vocab (the fast path): pass a non-null `vocab` that the caller +// owns and keeps alive; add_token(term_id, ...) indexes straight into it. +// * OWNED vocab (compatibility): pass a null `vocab`; the string-keyed +// add_token(string_view, ...) interns each new term into an internal owned +// vocabulary (assigning ids in first-seen order) and forwards to the id +// path. Existing callers that feed strings keep working unchanged. +// +// SPILL / K-WAY MERGE (out-of-core, bounds input RAM): when a non-zero +// spill_threshold_bytes is set, the REAL resident accumulator size (see +// resident_bytes(): the posting arena PLUS every live vocab / pair-map / slot / +// rank structure, G08) is compared against the threshold as tokens arrive; once +// it crosses the threshold the buffer SORTS its current terms, +// writes a self-describing sorted RUN to a temp file, and CLEARS memory. Each +// run record is keyed by the TERM-ID (varint); the k-way merge orders runs by +// the id's VOCAB STRING so the merged stream stays lexicographic. Because +// tokens arrive in globally ascending docid order, a term that reappears in a +// later run only covers strictly-later docids, so concatenating its postings in +// run order during the final merge keeps docids ascending. for_each_term_sorted +// flushes the residual buffer as a final run, then k-way merges all runs +// materializing only ONE merged term at a time -> peak memory stays bounded by +// the threshold (plus the widest single term), NOT by total postings. With the +// default threshold 0 (unlimited) the path is exactly the in-memory behavior. +// +// Internal representation is a COMPACT TAGGED VARINT byte stream per term, held in +// a shared SEGMENTED ARENA (CompactPostingPool), NOT per-term uint32 vectors. Each +// term owns ONE arena chain holding a stream of per-TOKEN entries in arrival +// order: every token contributes varint((pos << 1) | new_doc_bit); when new_doc_bit +// is set, the token's doc differs from the previous one, so a zigzag-varint(docid - +// prev_docid) immediately follows. Frequencies are NOT stored -- a doc's freq is +// the count of consecutive same-doc tokens, recovered while decoding. This drops +// the entire freq stream and the second (positions) chain versus a freq/prox split, +// so the payload is ~3.4x smaller than raw uint32 docids/freqs/positions, and the +// shared arena removes per-vector doubling slack and per-term vector headers. Each +// append writes straight into the chain (no deferred per-doc flush): the only live +// per-term state is the current doc id (to detect a doc change) and the delta base. +// to_postings() decodes a term's chain back to the SAME flat TermPostings the +// writer consumes, so the produced .idx is BYTE-IDENTICAL. positions_flat stays +// empty (and pos is tagged as 0) when positions are disabled; freq still counts. +// +// Duplicate vocab strings: the vocab is assumed to map each id to a DISTINCT +// string (a dense vocabulary). If two ids share a string they sort adjacently +// but are emitted as two separate terms; callers must not rely on coalescing. +class SpimiTermBuffer { +public: + // BORROWED-vocab constructor: `vocab` maps term-id -> term string and is + // borrowed (NOT owned) -- the caller must keep it alive for the buffer's + // lifetime. add_token(term_id, ...) accumulates by id with no string work. + // spill_threshold_bytes is the gate-2 internal buffer cap (e.g. 512 MiB), + // sourced from config; == 0 means unlimited (pure in-memory, default). A + // positive value caps the REAL resident accumulator size (resident_bytes(): + // arena + every live vocab/pair-map/slot/rank structure, G08), triggering a + // spill when that crosses the cap -- NOT the old per-token estimate. + // `reporter` is the OPTIONAL writer-level build-RAM reporter (null off-Doris / + // unit tests). When non-null, the accumulator reports its REAL resident-byte + // deltas -- resident_bytes() diffs -- positive on grow, negative on every + // reset/free, exactly once. NEVER reports live_bytes_ (a gated estimate that + // feeds only the spill threshold). + explicit SpimiTermBuffer(const std::vector* vocab, bool has_positions, + size_t spill_threshold_bytes = 0, MemoryReporter* reporter = nullptr); + + // OWNED-vocab (compatibility) constructor: no external vocab. The string-keyed + // add_token interns terms into an internal vocabulary on first occurrence. + explicit SpimiTermBuffer(bool has_positions, size_t spill_threshold_bytes = 0, + MemoryReporter* reporter = nullptr); + + ~SpimiTermBuffer(); + + SpimiTermBuffer(const SpimiTermBuffer&) = delete; + SpimiTermBuffer& operator=(const SpimiTermBuffer&) = delete; + + // Records one token by TERM-ID: term `term_id` occurs in `docid` at `pos`. + // `term_id` must be in [0, vocab_size). An out-of-range id latches an + // InvalidArgument into status() and is ignored. For a given term, docids are + // expected to arrive in non-decreasing order, and positions within a docid in + // ascending order; out-of-order docids (INCLUDING a REVISITED docid -- the same + // docid appearing again after a different one) are tolerated and reordered at + // finalize: SortByDocid stably sorts by docid and COALESCES same-docid groups + // (summing freqs, concatenating positions in document order), so the emitted + // postings have exactly ONE strictly-ascending entry per docid -- matching the + // k-way merge path and the writer's strictly-ascending precondition. + void add_token(uint32_t term_id, uint32_t docid, uint32_t pos); + + // Compatibility overload: records one token by TERM STRING. Valid ONLY on an + // OWNED-vocab buffer (constructed without an external vocab); interns `term` + // into the internal vocabulary on first occurrence, then forwards by id. Called + // on a BORROWED-vocab buffer it is REJECTED (latches InvalidArgument, token + // ignored) -- interning would grow the owned vocab out of step with the borrowed + // one and corrupt the build. Interning probes a heterogeneous (string_view-keyed) + // set, so a repeat token for an already-seen term allocates NOTHING; a std::string + // is materialized only on a term's FIRST occurrence (stored once in owned_vocab_). + // The id overload remains the hot path (no hashing at all); prefer that and + // reserve this for tests / legacy string-fed callers. + void add_token(std::string_view term, uint32_t docid, uint32_t pos); + + // Sentinel "no term id" return of add_token_returning_id (a real id would + // require a four-billion-string vocabulary). Returned only when the token was + // REJECTED (wrong vocab mode; the error is latched into status()). + static constexpr uint32_t kInvalidTermId = 0xFFFFFFFFU; + + // Same contract/behavior as add_token(std::string_view, ...) but returns the + // OWNED-vocab term-id the token was interned/accumulated under (G05): the SNII + // column writer captures each unigram's id here and feeds adjacent pairs to + // the id-keyed add_bigram_token below, so the per-pair hot path never hashes + // or compares term BYTES again. The returned id is stable for the buffer's + // lifetime for every NON-bigram term (only marker-prefixed bigram terms are + // ever evicted/recycled by the G04 diet). Returns kInvalidTermId (and latches + // InvalidArgument) on a borrowed-vocab buffer. + uint32_t add_token_returning_id(std::string_view term, uint32_t docid, uint32_t pos); + + // ZERO-ALLOC hidden phrase-bigram token (G01 part C): records one occurrence + // of the synthetic term make_phrase_bigram_term(left, right) WITHOUT ever + // composing that string on the hot path. The intern probe hashes and + // compares the term piecewise (marker + varint(len(left)) + left + right) + // via the transparent PhraseBigramTermView overloads below; the owned + // std::string is constructed EXACTLY ONCE, on the pair's first-time intern. + // Byte-identical accumulation to + // add_token(format::make_phrase_bigram_term(left, right), docid, pos) + // (the two entry points intern to the SAME term-id -- one shared content + // hash/equality), and the same OWNED-vocab-mode-only contract: called on a + // borrowed-vocab buffer it latches InvalidArgument and ignores the token. + void add_bigram_token(std::string_view left, std::string_view right, uint32_t docid, + uint32_t pos); + + // G05 PAIR-KEYED hidden phrase-bigram token: records one occurrence of the + // synthetic term make_phrase_bigram_term(vocab[left_id], vocab[right_id]) + // keyed by the uint64 (left_id << 32 | right_id) PAIR KEY -- no byte hashing, + // no fragment compares, no string storage on the accumulation path at all. + // `left_id`/`right_id` are the UNIGRAM ids add_token_returning_id handed the + // caller for the two constituent tokens (both already interned, so their + // vocab strings are pinned for the buffer's lifetime -- only bigram terms are + // ever evicted). The pair's Term accumulates under an owned-vocab id whose + // vocab string stays EMPTY until the composed on-disk term string is + // materialized from the two unigram strings at the LAST possible moment: + // * at a gate-2 spill, for pair terms actually written to the run (df>=2; + // df==1 pair terms are evicted instead, exactly like G04), because run + // records pin the id and the k-way merge orders by the vocab string; + // * at drain start (flush), for every still-live pair term, BEFORE any + // sort -- so the emitted term order is identical to the composed-string + // order the string-keyed path produces. + // Byte-identical accumulation and drain output to + // add_bigram_token(vocab[left_id], vocab[right_id], docid, pos). + // OWNED-vocab mode only (latches InvalidArgument otherwise); ids must be + // in-range non-bigram terms (else latches InvalidArgument, token ignored). + // MIXING CONTRACT: within one buffer a given pair must be fed EITHER through + // this id-keyed path OR through the string paths above, never both -- the two + // interning tables cannot see each other's entries, so mixing would emit the + // same composed term twice (the documented duplicate-vocab-string caveat). + // The production writer only ever uses this path; the string paths remain + // for tests/legacy callers. + void add_bigram_token(uint32_t left_id, uint32_t right_id, uint32_t docid, uint32_t pos); + + // G04 "bigram diet" phase 2. Call ONCE, before any add, on an OWNED-vocab + // buffer, and ONLY when the flush-time G01 bigram df-prune WILL be active + // (config snii_bigram_prune_min_df != 0) -- the caller (SNII column writer) + // owns that gate. Enables, for every hidden phrase-bigram term (marker + // prefix, sentinel included): + // (a) POSITION SUPPRESSION: bigram tokens accumulate DOCS+FREQ ONLY. Their + // positions were pure dead bytes since G01 (surviving bigrams are + // written docs+freq with no .prx; pruned ones vanish), yet were still + // varint-encoded per token into the arena, spilled into every run and + // re-materialized at drain. Suppressed terms emit an EMPTY + // positions_flat (and never a pos_pump); freqs are unaffected. The + // flush-time validate accepts this because write_prx is false for + // bigrams whenever pruning is active. Unigrams are untouched. + // (b) VOCAB-CAP EVICTION (when vocab_cap_bytes > 0): once the live bigram + // intern storage (string capacity + a fixed per-term overhead + // estimate) exceeds vocab_cap_bytes, bigram terms whose CURRENT df is + // exactly 1 (the Zipf long tail; never the sentinel) are evicted + // INCREMENTALLY -- a bounded sweep step per bigram add, plus a pass at + // each gate-2 spill -- releasing their intern string, intern-set entry, + // term slot and postings, and recycling their term-id for the next + // intern. Every evicted term is recorded in the ever-dropped bloom + // (bigram_dropped_filter()); the flush-time process_term drops any + // bigram the bloom may contain IN ADDITION to the df threshold, so an + // evicted-then-reappearing pair (whose re-interned postings would be + // INCOMPLETE) can never materialize. Bloom false positives only + // over-drop, which the G01 reader fallback contract makes safe. + // Ids already written to a spill run are NEVER evicted or recycled + // (the run records reference them; their vocab strings are immutable + // from first spill on). + // (c) G06 DRAIN-SIDE DF GATE (when bigram_drain_min_df > 0): sinks the + // flush-time phrase-bigram df prune (LogicalIndexWriter::process_term's + // `df < bigram_prune_min_df` gate) into the buffer drain for PAIR-KEYED + // bigram terms, so the Zipf low-df tail is dropped WITHOUT ever + // composing its term strings or decoding its postings (on wikipedia + // that tail's materialize+emit was ~1/3 of build CPU). See + // prepare_pair_terms_for_drain for the exact drop rule and its + // exactness guarantee. `bigram_drain_min_df` passed HERE governs + // MID-FEED spill drains, which run before the final doc count (and + // thus the exact flush threshold) exists -- pass the flush value when + // the config fixes it, or a LOWER BOUND of it (e.g. the auto formula's + // 0-doc floor) otherwise; every mid-feed drop is bloom-recorded, so + // even an over-estimate only ever over-drops safely (the G01 reader + // fallback contract). The FINAL drain must gate with the EXACT + // process_term threshold: set_bigram_drain_min_df below re-plumbs it + // at flush (LogicalIndexWriter::build_blocks does this) before the + // terminal drain runs. 0 (default) disables the gate entirely: every + // live pair term materializes at drain, exactly the pre-G06 behavior. + // Calling this on a BORROWED-vocab buffer latches InvalidArgument and is + // ignored (the diet needs the owned intern table). + void configure_bigram_diet(uint64_t vocab_cap_bytes, uint32_t bigram_drain_min_df = 0); + bool bigram_diet_configured() const { return bigram_diet_; } + + // G06: (re)sets the drain-side phrase-bigram df gate to `min_df` -- the + // EXACT effective flush threshold process_term will gate with (0 disables + // the gate). LogicalIndexWriter::build_blocks calls this with its resolved + // bigram_prune_min_df right before draining the term source, so the final + // drain's drop set is byte-identical to what process_term would have pruned. + // Plain state update, safe on any buffer in any mode: the gate only ever + // examines pair-map entries, so a borrowed-vocab / no-pair-term buffer is + // unaffected regardless of the value. + void set_bigram_drain_min_df(uint32_t min_df) { bigram_drain_min_df_ = min_df; } + + // Ever-dropped bloom over evicted bigram terms; nullptr until the first + // eviction (so the flush path probes nothing when the feature never fired). + // Borrowed; valid for the buffer's lifetime. + const BigramDropFilter* bigram_dropped_filter() const { return bigram_drop_filter_.get(); } + + // Live bigram intern storage estimate the vocab cap is enforced against: + // sum over live non-sentinel bigram terms of (string capacity + + // kBigramInternFixedOverheadBytes) for string-keyed terms, or exactly + // kBigramInternFixedOverheadBytes for a G05 pair-keyed term (it owns NO + // string until materialization -- the footprint is the fixed pair-map / + // vocab-slot overhead). Exposed for the cap-bound tests / observability. + uint64_t bigram_intern_bytes() const { return bigram_intern_bytes_; } + + // Fixed per-term overhead added to a bigram string's capacity when + // accounting intern storage against the vocab cap: the std::string header in + // owned_vocab_, the intern-set entry (or the G05 pair-map entry + reverse + // pair-key slot), and the 4 B slot/rank entries. An estimate (allocator + // slack varies); deterministic so tests can reason about the cap exactly. + // A G05 pair-keyed bigram term's WHOLE footprint is this constant (fixed + // bytes per pair-map entry; no string). + static constexpr uint64_t kBigramInternFixedOverheadBytes = 64; + + // TEST-ONLY: number of live pair-keyed bigram terms (pair-map entries, + // including entries whose id was materialized+pinned by a spill). Not part + // of the production API. + size_t bigram_pair_terms_for_test() const { return bigram_pair_map_.size(); } + + // G09: joins the PROCESS-WIDE build-RAM registry. Registers this buffer's + // current resident bytes with `limiter` and forwards every subsequent + // (debounced, see report_arena_delta) resident total to it; the destructor + // un-registers. When the registered sum across ALL of the process's live + // buffers exceeds the limiter's budget, the limiter may set this buffer's + // ADVISORY spill-request flag from ANOTHER thread; the flag is observed -- + // and the forced spill run ON THIS BUFFER'S OWN THREAD -- by the next + // add_token's maybe_spill_after_token (see there for the honor rule). + // Call at most once, right after construction (extra calls are ignored); + // `limiter` must outlive this buffer. Null / never attached = the G08 + // per-writer behavior, byte-identical. + void attach_global_limiter(GlobalMemoryLimiter* limiter); + + // TEST-ONLY: G09 advisory-flag observability -- read the pending flag, and + // plant a request directly (what the limiter does cross-thread) so the + // owner-honors-at-next-token contract is testable without a registry. + bool global_spill_requested_for_test() const { + return global_spill_requested_.load(std::memory_order_relaxed); + } + void request_global_spill_for_test() { + global_spill_requested_.store(true, std::memory_order_relaxed); + } + + // G09 forced-spill floor (config snii_forced_spill_min_arena_bytes): a + // pending process-wide forced-spill request is honored only once the + // reclaimable posting arena holds at least this much (never below one + // arena block, so a run is always writable). A request planted while the + // arena is below the floor is a NO-OP that stays PENDING -- it is NOT + // retried as a spill every token -- and is honored when the arena regrows + // past the floor. Without the floor, an unreachable global budget (the + // persistent vocab/pair structures of all writers alone exceeding it) + // re-flagged every buffer on every report and each honored with a single + // 32 KiB arena block: thousands of tiny runs per buffer, EMFILE at the + // k-way merge reopen, failed loads (the conc=16 wikipedia field storm). + static constexpr uint64_t kDefaultForcedSpillMinArenaBytes = 64ULL << 20; // 64 MiB + void set_forced_spill_min_arena_bytes(uint64_t bytes) { forced_spill_min_arena_bytes_ = bytes; } + uint64_t forced_spill_min_arena_bytes() const { return forced_spill_min_arena_bytes_; } + + // G09 run-file cap (config snii_spill_max_run_files_per_buffer): when a + // new spill would grow the accumulated run-file count past this cap, the + // existing runs are first MERGE-COMPACTED into one (a k-way merge of the + // run files back into a single fresh run; term stream byte-identical, the + // old files deleted) so the buffer never holds more than the cap + 1 run + // files. Bounds both the final k-way merge's fan-in and -- decisively -- + // its OPEN FILE DESCRIPTORS: every run of a buffer is (re)opened + // simultaneously and held open for the whole merge, so unbounded run + // counts across ~100 concurrent writers exhausted the BE nofile rlimit + // ('Too many open files' at run reopen). 0 disables the cap. + static constexpr size_t kDefaultMaxRunFilesPerBuffer = 64; + void set_max_run_files(size_t cap) { max_run_files_ = cap; } + size_t max_run_files() const { return max_run_files_; } + + // Number of DISTINCT terms accumulated so far (touched ids still resident). + size_t unique_terms() const; + uint64_t total_tokens() const { return total_tokens_; } + bool has_positions() const { return has_positions_; } + + // OK unless an add_token validation error (out-of-range term-id, wrong vocab + // mode) was latched. for_each_term_sorted now returns its own I/O Status + // directly; callers that use add_token's latch-and-report pattern MUST check + // this after draining to surface input-side validation errors. + [[nodiscard]] Status status() const { return spill_status_; } + + // TEST-ONLY: number of spill run files currently HELD (== 0 in pure + // in-memory mode). Lets tests assert that a gate-2 spill actually fired + // once the REAL resident size crossed the configured cap. NOTE: a G09 + // run-cap merge-compaction (see set_max_run_files) collapses the list to + // ONE file, so the count is not monotonic. Not part of the production API. + size_t run_count_for_test() const { return run_paths_.size(); } + + // TEST-ONLY: the REAL resident accumulator bytes the gate-2 trigger and the + // MemoryReporter see (resident_bytes()). Lets the G08 accounting tests assert + // coverage (>= the externally-measured pair-map/vector footprint) and + // monotonicity without widening access to the private accounting. Not part of + // the production API. + uint64_t resident_bytes_for_test() const { return resident_bytes(); } + + // Materializes all terms sorted lexicographically; each term's docids are + // ascending. Convenience wrapper around for_each_term_sorted that keeps the + // whole result alive at once. Prefer for_each_term_sorted for low peak memory. + // MUST be called at most once: it drains internal state. A SECOND drain (a + // repeat call, or a finalize_sorted after a for_each_term_sorted, or vice versa) + // returns EMPTY and latches an error into status() rather than re-emitting. + std::vector finalize_sorted(); + + // Streams terms to `fn` in lexicographic order, building ONE transient + // TermPostings at a time and freeing that term's accumulated arrays before + // moving to the next. This keeps at most a single term's postings duplicated, + // avoiding the input+output coexistence peak. MUST be called at most once: it + // drains internal state. A SECOND drain invokes `fn` zero times and returns + // an Internal error (a re-merge of the still-present run files would otherwise + // re-emit every term). Returns non-OK on spill/merge I/O or corruption errors, + // or if a prior add_token latched a validation error into status(). + Status for_each_term_sorted(const std::function& fn); + +private: + // Compact per-term accumulator: ONE tagged-varint arena chain plus a few cursors. + // Every token is appended immediately (no deferred flush), so the only running + // state is the current doc id and the delta base. A sentinel chain head of + // kNoChain marks a term that has not started its chain yet (so an all-empty term + // costs no arena bytes). ntok / ndocs bound the decode loop and size reserves. + // Total ~36 B per live term. + static constexpr uint32_t kNoChain = 0xFFFFFFFFU; + struct Term { + uint32_t head = kNoChain; // chain read entry point + CompactPostingPool::SliceWriter w; // append cursor for the chain (8 B) + uint32_t ntok = 0; // total tokens (entries) in the chain + uint32_t cur_docid = 0; // most-recent doc id: detects doc change AND + // is the zigzag delta base for the next doc + // G04/G06: EXACT count of new-doc GROUPS in the chain (one per new_doc + // tag, i.e. per maximal same-docid token run). Always an UPPER BOUND on + // the coalesced df (distinct docids), and EQUAL to it while `sorted` + // holds -- a non-decreasing docid feed cannot revisit a docid + // non-adjacently, so every group is a distinct doc. ndocs == 1 answers + // the G04 eviction's "df == 1 vs >= 2" question exactly even when + // unsorted (an overcount can never make a df>=2 term look like 1), and + // the G06 drain-side bigram df gate drops a pair term only when + // `sorted` is set, i.e. when this IS the df process_term would compute. + // Replaces the former u8 saturating-at-2 counter; the u32 lands in + // Term's padding, so sizeof(Term) is unchanged. + uint32_t ndocs = 0; + uint8_t level = 0; // current slice level of w (packed here, not in w) + bool started = false; // false until the first token is appended + bool sorted = true; // false if a docid arrived out of ascending order + // G04 part (b of G01 phase-2 note): true for hidden phrase-bigram terms + // when the diet is configured -- the chain then stores the new_doc tag + // ONLY (no position payload; identical bytes to a positions-disabled + // encoding) and to_postings emits an empty positions_flat. + bool pos_suppressed = false; + }; + static_assert(sizeof(CompactPostingPool::SliceWriter) == 8, + "SliceWriter must stay 8 bytes to keep Term compact"); + + // The active vocabulary (term-id -> string): either the borrowed pointer or, + // in owned mode, &owned_vocab_. Always non-null after construction. + const std::vector& vocab() const { return *vocab_; } + + // Accumulates one already-validated token into the per-id Term. + void accumulate(uint32_t term_id, uint32_t docid, uint32_t pos); + + // Per-token gate-2 tail of accumulate(): reports the token's resident growth, + // then spills when the unified cap / local threshold fires with a worthwhile + // reclaimable arena (the G08 anti-churn floor), when the G09 process-wide + // limiter's advisory request flag is pending (honored here, on the owner's + // own thread; bypasses the G08 floor but requires one allocated arena block + // so a run is writable), or when the arena nears its hard 4 GiB offset + // limit. Every add path funnels through accumulate(), so every add path + // observes this gate. + void maybe_spill_after_token(); + + // Decodes `t`'s compact chain into a TermPostings (the exact docids/freqs/ + // positions the writer consumes), sorting by docid first if `t.sorted` is false. + // When `allow_stream_positions` is true (the in-memory drain path), a large + // sorted term's positions are provided via TermPostings::pos_pump instead of a + // materialized positions_flat (peak-RSS win). The spill path passes false so the + // run codec always sees a fully-materialized positions_flat. + TermPostings to_postings(std::string term, Term&& t, bool allow_stream_positions) const; + + // Returns the touched term-ids sorted by their vocab string (lexicographic). + // Sorts by a PRECOMPUTED integer string-rank (term-id -> lexicographic rank), + // not by full std::string compare: a single std::string sort over the whole + // vocabulary is amortized across every spill, so each spill's sort is an + // integer compare instead of paying a fresh O(touched * strcmp) on every spill. + std::vector sorted_ids() const; + // Builds string_rank_ (term-id -> lexicographic rank) once, lazily. Idempotent. + void ensure_string_rank() const; + // Streams the in-memory terms in sorted order, draining the slot pool (the + // in-memory single-pass path). When `allow_stream_positions` is true, large + // sorted terms stream positions via pos_pump (valid only because the callback + // consumes each term synchronously while the arena is still resident); callers + // that RETAIN the TermPostings past the drain (finalize_sorted) must pass false. + Status drain_sorted(const std::function& fn, bool allow_stream_positions); + // Spills the current buffer to a fresh sorted run file and clears memory. + // `evict_low_df_bigrams`: gate-2 mid-feed spills pass true so df==1 bigram + // terms are EVICTED (bloom-recorded, id recycled) instead of being written + // to the run -- a run record would otherwise pin the term's vocab string + // forever (in-run ids are never evictable). The FINAL residual spill inside + // merge_runs passes false: no token can arrive after it, so its df==1 + // bigrams are already doomed by the flush-time df threshold and blooming + // them would only add false-positive pressure. + Status spill_to_run(bool evict_low_df_bigrams); + // G09 run-file cap enforcement (see set_max_run_files): merge-compacts the + // current run files into ONE fresh run (same term stream, ids ordered by + // the current string rank -- in-run ids' vocab strings are pinned, so the + // rank order over them never changes between spills), deletes the old + // files and replaces run_paths_ with the compacted one. Called by + // spill_to_run before opening a new run once the cap is reached. + Status compact_runs(); + // Writes all current terms (sorted) to an already-open RunWriter, draining. + // Skips ids whose slot is gone (evicted mid-feed) and, when + // `evict_low_df_bigrams`, evicts eligible df==1 bigrams instead of writing + // them; every id actually written is marked in-run (never evictable after). + Status drain_to_writer(class RunWriter* w, bool evict_low_df_bigrams); + // REAL resident accumulator bytes -- the single source of truth for the gate-2 + // spill trigger and every MemoryReporter delta. G08: sums EVERY live input-side + // structure -- the posting arena (unigram AND bigram chains, docs+prx payload) + // plus the vocab-sized slot index, the Term slot pool + free/touched lists, the + // owned vocabulary (headers by capacity + string heap payloads via + // owned_vocab_heap_bytes_) and its intern set, the G05 pair map (capacity * + // slot size) + reverse pair-key slots, and the G04/rank bookkeeping arrays + // (free_ids_ / id_in_run_ / string_rank_). Replaces the pre-G05 arena + + // slot-index-only figure, which left the whole pair-key/vocab machinery + // invisible to the gate (the wikipedia ~20 GiB overshoot). Capacity, not size, + // throughout: the reserved tail is resident RSS and survives spills. + uint64_t resident_bytes() const; + // Reports the signed change in REAL resident bytes (resident_bytes()) to + // mem_reporter_ since the previous call, then caches the new total. + // Single-source diff: every grow/reset/free emits EXACTLY ONE delta + // (self-balancing -> impossible to double-count or miss a negative). No-op when + // mem_reporter_ is null. + void report_arena_delta(); + // Final k-way merge over the spilled runs (+ the residual flushed as a run). + // When `allow_stream_positions` is true (the streaming for_each path), a wide + // merged term streams positions via pos_pump (valid only because fn consumes + // synchronously while the run readers stay parked); callers that RETAIN the + // TermPostings past the merge (finalize_sorted) MUST pass false. + Status merge_runs(const std::function& fn, bool allow_stream_positions); + // Deletes every temp run file; called from the destructor (RAII cleanup). + void cleanup_runs(); + // Frees a drained term's accumulator (id leaves the touched set). + void release_term(uint32_t term_id); + + // ---- G04 bigram vocab-cap eviction (owned-vocab mode only) --------------- + // Shared first-time-intern tail for both owned-mode add paths: recycles a + // freed (evicted) term-id when one is available -- otherwise appends a fresh + // id -- stores `term_str` as the id's vocab string, inserts the id into the + // intern set, and does the bigram intern-storage accounting. Returns the id. + uint32_t intern_owned_term(std::string&& term_str); + // True when eviction can fire: diet configured with a non-zero cap on an + // owned-vocab buffer. + bool bigram_evict_enabled() const { + return bigram_diet_ && bigram_vocab_cap_bytes_ != 0 && vocab_ == &owned_vocab_; + } + // Cap-accounted footprint of one bigram term's intern storage. + static uint64_t bigram_term_footprint(const std::string& s) { + return s.capacity() + kBigramInternFixedOverheadBytes; + } + // Is `id` evictable right now: live slot, current df == 1 (ndocs == 1), + // never written to a spill run, and a non-sentinel phrase-bigram term. + bool bigram_evictable(uint32_t id) const; + // Evicts one eligible id: records the term in the ever-dropped bloom, + // releases its slot/postings, erases it from the intern set, frees its vocab + // string and recycles the id. Bumps the eviction seam (+ the vocab epoch for + // string-keyed terms). Eligibility is the caller's job: bigram_evictable + // (the df==1 cap sweep / spill rule) OR the G06 mid-feed df-gate drop + // (prepare_pair_terms_for_drain: a never-spilled pair term with exact + // df < the drain threshold -- possibly df >= 2 -- whose reappearance the + // bloom must cover). + void evict_bigram_term(uint32_t id); + // Bounded incremental sweep step (amortized, never stop-the-world): when the + // bigram intern storage is over the cap (and the sweep is not paused after a + // fruitless full lap), scans up to kVocabSweepStride ids from a persistent + // cursor, evicting every eligible one EXCEPT `just_touched_id` -- the term + // the current add extended. Skipping it costs one compare and guarantees a + // recurring pair whose occurrences arrive back-to-back reaches df==2 (and + // thus permanent immunity) instead of being churned at df==1 by its own + // add's sweep -- decisive when the vocabulary is no larger than the stride. + // Called once per owned-mode add. + void maybe_sweep_bigram_vocab(uint32_t just_touched_id); + + // ---- G05 pair-keyed bigram terms (owned-vocab mode only) ----------------- + // Composes the uint64 pair key add_bigram_token(left_id, right_id) interns + // bigram terms under. + static uint64_t make_pair_key(uint32_t left_id, uint32_t right_id) { + return (static_cast(left_id) << 32) | right_id; + } + // True when `id` is a live pair-keyed bigram term that has NOT yet had its + // composed string materialized OR was materialized by a spill (the pair-map + // entry outlives materialization so later occurrences keep finding the id); + // false for every unigram / string-keyed / recycled id. + bool is_pair_term(uint32_t id) const { + return id < pair_of_.size() && pair_of_[id] != kNoPairKey; + } + // First-time intern of a pair key: recycles a freed id when available (else + // appends a fresh id whose vocab string stays EMPTY), records the reverse + // id -> pair-key mapping, inserts the pair-map entry, and accounts the FIXED + // per-entry footprint against the vocab cap. Never bumps vocab_epoch_ (the + // id's vocab string is empty before and after). + uint32_t intern_pair_term(uint64_t pair_key); + // FNV-1a 64 of the pair's synthetic term bytes, computed piecewise from the + // two unigram vocab strings (== fnv of the composed string; the drop-filter + // key for pair-term evictions). + uint64_t pair_term_content_fnv(uint64_t pair_key) const; + // Composes and stores the pair's on-disk term string into owned_vocab_[id] + // (the single materialization point), switches the cap accounting from the + // fixed pair footprint to the string footprint, and bumps vocab_epoch_ so + // the cached lexicographic rank is rebuilt over the COMPOSED string before + // any subsequent sort. The pair-map entry is kept (later adds of the pair + // must keep resolving to this id). + void materialize_pair_term(uint32_t id, uint64_t pair_key); + // MUST run at the start of every drain that sorts (drain_sorted and + // drain_to_writer), BEFORE sorted_ids(): pair-keyed terms carry EMPTY vocab + // strings, which would rank first en bloc and emit the run / stream out of + // lexicographic order. For every live not-yet-materialized pair term this + // does exactly one of: + // * EVICT it (mid-feed spills only, df==1 -- same rule and bloom as the + // G04 string path); + // * G06 DF-GATE DROP it (bigram_drain_min_df_ > 0 only): a pair term + // whose EXACT df is provably below the flush prune threshold is dropped + // WITHOUT materialization and WITHOUT emission -- the same no-trace + // disposition process_term's df gate would give it (no dict entry, no + // postings, no bloom membership; the reader's G01 pruned-segment + // contract covers the miss). "Provably exact" means Term::sorted still + // holds, so Term::ndocs == the coalesced df process_term would compute; + // an out-of-order-fed term is NEVER dropped here (it materializes and + // process_term gates it on the exact coalesced df). On MID-FEED spills + // the drop additionally records the term in the ever-dropped bloom via + // the full G04 eviction (the pair may reappear with these docids + // missing) and therefore requires the eviction machinery (vocab cap + // on); on the FINAL drain (terminal in-memory drain / the residual + // spill inside merge_runs) nothing can reappear, so the drop skips the + // bloom entirely and needs no cap. + // * MATERIALIZE its composed string (everything else). + // Afterwards every id the sort can see ranks by its final on-disk bytes, so + // the emitted order is identical to the string-keyed build's. Flush-side df + // pruning and bloom drops stay in LogicalIndexWriter::process_term as the + // unchanged backstop for whatever still reaches it (string-keyed terms, + // spill-materialized pair terms, out-of-order-fed pair terms). + void prepare_pair_terms_for_drain(bool evict_low_df_bigrams); + // Shared release tail for a live not-yet-materialized pair-keyed bigram + // term: un-accounts the fixed pair footprint, erases the pair-map entry and + // the reverse id -> pair-key mapping, releases the term's slot/postings and + // recycles the id. Deliberately does NOT touch the drop bloom: the G04 + // eviction path blooms BEFORE calling this; the G06 final-drain df drop + // calls it alone (no bloom -- nothing reappears after the final drain, and + // blooming would only add false-positive pressure on surviving hot pairs). + // Never bumps vocab_epoch_ (the id's vocab string was empty and stays so). + void release_pair_term(uint32_t id, uint64_t pair_key); + + const std::vector* vocab_; // active vocab (borrowed or &owned_) + std::vector owned_vocab_; // owned mode: interned term strings + + // G08: running sum of the owned vocab strings' HEAP payloads (0 for SSO + // strings -- their bytes live inside the headers owned_vocab_.capacity() + // already charges; capacity+1 for heap strings). Maintained incrementally so + // resident_bytes() stays O(1): intern_owned_term and materialize_pair_term + // CREDIT a stored string, evict_bigram_term DEBITS it before freeing (the + // credit/debit symmetry), and the terminal drains zero it when owned_vocab_ + // is released. + uint64_t owned_vocab_heap_bytes_ = 0; + + // G08: fixed per-entry estimate for one intern-set entry. Sized for the + // pre-G10 NODE-based set (16 B next-ptr+id node, its malloc chunk rounding, + // and an amortized bucket-array share) and deliberately UNCHANGED by the G10 + // swap to the flat set: resident_bytes() feeds the gate-2 spill trigger, so + // keeping the constant keeps the resident-byte sequence -- and therefore + // every spill point and the drained output -- bit-identical to the prior + // build. For the flat set it OVER-approximates the real cost (~5 B per slot + // by capacity), which can only fire the gate earlier, never overshoot. + // Deterministic so the accounting tests can reason about it, and ZERO for an + // empty set so an untouched (borrowed-mode) buffer charges nothing for it. + static constexpr uint64_t kInternEntryEstimateBytes = 48; + + // Heterogeneous (is_transparent) functors backing the owned-mode interning set. + // The set stores ONLY 4-byte term-ids; each id's string lives EXACTLY ONCE in + // owned_vocab_ (F03 single-store -- no second owned-string map key). Both functors + // hold a back-pointer to owned_vocab_ and dereference a stored id to + // owned_vocab_[id] for content hashing/equality. + // + // CONTENT HASH: a byte-INCREMENTAL FNV-1a 64 (fnv_update below) replaces + // std::hash so the SAME hash is computable either over a whole + // string (id / string_view probes) or fragment-by-fragment over a + // PhraseBigramTermView probe (marker, varint(len(left)), left, right) WITHOUT + // composing the synthetic bigram string -- the G01 zero-alloc requirement. + // std::hash cannot be computed piecewise (implementation-defined), which is + // exactly why it was replaced; equal content ALWAYS hashes equal across all + // three key forms because every form feeds the identical byte sequence. + // + // BOTH functors MUST be transparent (P0919/P1690): a transparent hash alone + // does NOT enable heterogeneous find(); the equal functor must be transparent + // too, and both must accept every probe key form. + static constexpr uint64_t kFnvOffsetBasis = 1469598103934665603ULL; + static constexpr uint64_t kFnvPrime = 1099511628211ULL; + static uint64_t fnv_update(uint64_t h, std::string_view bytes) noexcept { + for (const char c : bytes) { + h ^= static_cast(c); + h *= kFnvPrime; + } + return h; + } + static size_t hash_term_bytes(std::string_view s) noexcept { + return static_cast(fnv_update(kFnvOffsetBasis, s)); + } + // Full 64-bit piecewise FNV of the synthetic bigram term addressed by `v`. + // Fragment order mirrors make_phrase_bigram_term's byte layout exactly: + // marker ++ varint(len(left)) ++ left ++ right -- so the value EQUALS the + // FNV-1a 64 of the composed string (the intern-set content hash AND the + // BigramDropFilter key hash; the G05 pair-keyed eviction blooms with this, + // never composing the string). + static uint64_t bigram_view_fnv(const PhraseBigramTermView& v) noexcept { + uint64_t h = fnv_update(kFnvOffsetBasis, format::kPhraseBigramTermMarker); + char varint_buf[5]; + const size_t n = format::encode_phrase_bigram_varint32(static_cast(v.left.size()), + varint_buf); + h = fnv_update(h, std::string_view(varint_buf, n)); + h = fnv_update(h, v.left); + h = fnv_update(h, v.right); + return h; + } + static size_t hash_bigram_view(const PhraseBigramTermView& v) noexcept { + return static_cast(bigram_view_fnv(v)); + } + struct OwnedVocabHash { + using is_transparent = void; + const std::vector* vocab = nullptr; + size_t operator()(std::string_view s) const noexcept { return hash_term_bytes(s); } + size_t operator()(uint32_t id) const noexcept { + return hash_term_bytes(std::string_view((*vocab)[id])); + } + size_t operator()(const PhraseBigramTermView& v) const noexcept { + return hash_bigram_view(v); + } + }; + struct OwnedVocabEq { + using is_transparent = void; + const std::vector* vocab = nullptr; + bool operator()(uint32_t a, uint32_t b) const noexcept { return a == b; } + bool operator()(uint32_t a, std::string_view s) const noexcept { + return std::string_view((*vocab)[a]) == s; + } + bool operator()(std::string_view s, uint32_t a) const noexcept { + return std::string_view((*vocab)[a]) == s; + } + // Piecewise fragment compare against the stored composed string -- no + // temporary composition on either side. + bool operator()(uint32_t a, const PhraseBigramTermView& v) const noexcept { + return format::phrase_bigram_term_equals(std::string_view((*vocab)[a]), v.left, + v.right); + } + bool operator()(const PhraseBigramTermView& v, uint32_t a) const noexcept { + return format::phrase_bigram_term_equals(std::string_view((*vocab)[a]), v.left, + v.right); + } + }; + // Owned mode only: interns each distinct term to a term-id on first occurrence. + // Keyed by term-id (NOT std::string) so the vocab string is stored exactly once + // (in owned_vocab_); probed heterogeneously with a string_view so a repeat token + // costs no temporary std::string. + // + // G10/G11: a FLAT (open-addressing, SwissTable) set, not std::unordered_set. + // The node-based set chased one heap node per chained probe -- a cache miss + // per node through _M_find_before_node_tr (~7% of wiki-import CPU; the G11 + // in-process locality bench measures the swap at -29% on the unigram add + // path, -14% on the whole feed) -- while the flat set probes a contiguous + // control-byte group. phmap honors the functors' heterogeneous probes: + // raw_hash_set templates find()/erase() on the key argument whenever BOTH + // Hash and Eq expose is_transparent (its KeyArgImpl is + // KeyArg && IsTransparent>), and evaluates equality + // as eq(stored_id, probe) -- overloads OwnedVocabEq provides for all three + // key forms. phmap post-mixes the functor's hash (phmap_mix) identically on + // the insert, find and rehash paths, so content-equal key forms still + // resolve to the same slot group. + // + // SAFETY under the intern_owned_term invariant (ids hash owned_vocab_[id]): + // * a REHASH (insert-triggered growth / tombstone compaction) re-hashes the + // STORED ids -- dereferencing owned_vocab_[id] -- and moves only the + // 4-byte id slots, NEVER the strings. Every stored id's string is live at + // every insert (evict_bigram_term erases the id BEFORE clearing its + // string), so no rehash can read a freed string. + // * erase(id) hashes owned_vocab_[id] to locate the slot (same ordering + // requirement, already satisfied), then tombstones that one slot; no + // other element moves or is re-hashed. + // * flat-hash iterator/reference invalidation (every insert may invalidate + // ALL of them, unlike unordered_set's stable nodes) is moot here: no code + // iterates intern_ or holds an iterator/reference across a mutation -- + // both find() results are copied out (`term_id = *it`) immediately. + phmap::flat_hash_set intern_; + + bool has_positions_; + size_t spill_threshold_bytes_; // 0 => unlimited (no spilling) + uint64_t total_tokens_ = 0; + + // POOLED accumulators (replaces a dense vocab-sized std::vector, which + // cost ~80 B per vocab id even for the ~empty majority -- the single largest + // input-phase memory line). slot_of_ is the only vocab-sized array: a 4 B index + // per id (0 == no live Term; otherwise slot index + 1). slots_ holds ONE Term + // per CURRENTLY-LIVE id, so its size tracks the live touched count, not the + // vocabulary. On first touch an id claims a slot (reusing a freed one from + // free_slots_ when available, else appending). release_term frees the slot back + // to the pool and clears slot_of_[id]. touched_ids_ lists every live id so + // finalize/spill iterate touched ids without scanning the whole vocabulary. + // present_[id] is now (slot_of_[id] != 0). The hot add path is still a vector + // index + a couple of pushes: no hashing, no per-token allocation. + std::vector slot_of_; // vocab-sized: id -> slot index + 1 (0=empty) + std::vector slots_; // live Term pool (size ~ live touched count) + std::vector free_slots_; // recycled slot indices (drained terms) + std::vector touched_ids_; + size_t live_term_count_ = 0; // present (non-drained) terms; == unique_terms() + + // Shared arena backing every live term's DOC and POS varint byte chains. Holds + // the bulk of the accumulator's memory in a few large blocks (no per-term vector + // headers, no per-vector doubling slack) -- the compact-RSS win. + CompactPostingPool pool_; + + // Optional writer-level build-RAM reporter (null off-Doris / unit tests) and the + // last resident-byte total it was told about. report_arena_delta() diffs the live + // total (arena_bytes() + slot_of_.capacity()*4) against reported_resident_. + MemoryReporter* mem_reporter_ = nullptr; + int64_t reported_resident_ = 0; + + // ---- G09 process-wide limiter hookup (null / false = feature off) -------- + // The registry this buffer joined via attach_global_limiter (borrowed; must + // outlive the buffer), and the ADVISORY forced-spill request flag the + // limiter sets from other threads (only ever under the registry mutex; the + // owner reads it relaxed on its own thread each token). The flag pointer + // doubles as the buffer's registry identity. + GlobalMemoryLimiter* global_limiter_ = nullptr; + std::atomic global_spill_requested_ {false}; + // G09 forced-spill floor / run-file cap (see the public setters above). + uint64_t forced_spill_min_arena_bytes_ = kDefaultForcedSpillMinArenaBytes; + size_t max_run_files_ = kDefaultMaxRunFilesPerBuffer; + + // Returns the live Term for `term_id`, claiming a pool slot on first touch. + Term& term_slot(uint32_t term_id, bool* new_term); + + // Appends one byte / one varint to a term's tagged chain, lazily starting the + // chain on first use (so an untouched term costs no arena bytes). + void put_byte(Term* t, uint8_t b); + void put_varint(Term* t, uint64_t v); + + // ---- G05 pair-keyed bigram state (owned-vocab mode only) ----------------- + // pair key (left_id << 32 | right_id) -> owned-vocab term-id. The bigram + // accumulation hot path is ONE integer-keyed flat-map probe; no term bytes + // are hashed, compared or stored. Entries are erased on eviction and RETAINED + // across materialization (an in-run materialized id keeps accumulating later + // occurrences of its pair). + phmap::flat_hash_map bigram_pair_map_; + // Reverse map id -> pair key (kNoPairKey = not a pair term), sized lazily to + // the vocab on the first pair intern; the sweep/evict/materialize paths need + // id-first access. kNoPairKey is unreachable as a real key: left_id == + // 0xFFFFFFFF would require a four-billion-entry vocabulary. + static constexpr uint64_t kNoPairKey = 0xFFFFFFFFFFFFFFFFULL; + std::vector pair_of_; + + // ---- G04 bigram diet state (owned-vocab mode only) ----------------------- + bool bigram_diet_ = false; // position suppression + (cap) eviction on + uint64_t bigram_vocab_cap_bytes_ = 0; // 0 = no eviction (suppression only) + // G06 drain-side phrase-bigram df gate (0 = off). Mid-feed spill drains see + // the configure_bigram_diet value (the flush threshold or a lower bound of + // it); the final drain sees the EXACT flush threshold, re-plumbed by + // LogicalIndexWriter::build_blocks via set_bigram_drain_min_df. + uint32_t bigram_drain_min_df_ = 0; + uint64_t bigram_intern_bytes_ = 0; // live bigram intern storage (cap metric) + std::vector free_ids_; // evicted ids awaiting recycling + std::vector id_in_run_; // 1 = id written to some spill run (pinned); + // sized lazily at the first evict-enabled spill + std::unique_ptr bigram_drop_filter_; // created on first eviction + uint32_t sweep_cursor_ = 0; // next vocab id the incremental sweep examines + uint64_t sweep_scanned_since_evict_ = 0; // fruitless-lap detector + uint64_t sweep_rearm_bytes_ = 0; // paused sweep re-arms once bytes reach this + // Bumped whenever an EXISTING vocab id's string changes (eviction clears it, + // recycling re-assigns it). string_rank_ caches this epoch: a recycled id + // would otherwise keep its stale rank (the vocab SIZE alone cannot detect an + // in-place string change), and a stale rank would emit spill runs out of + // lexicographic order -- corrupting the k-way merge. + uint64_t vocab_epoch_ = 0; + + std::vector run_paths_; // spilled run temp files (deleted in dtor) + Status spill_status_; // first spill / range error, at finalize + bool drained_ = false; // set once finalize_sorted/for_each_term_sorted has run; + // a second drain would (spilled path) re-merge the run + // files and re-emit every term, or (in-memory path) emit + // nothing -- both wrong. Guard against the double-drain. + + // Lazily-built vocab-sized map: term-id -> its lexicographic rank among all + // vocab strings. Computed once (one full std::string sort of the vocabulary) + // on the first sorted_ids() call, then reused by every spill's id sort UNTIL + // the vocab grows OR an existing id's string mutates (G04 eviction/recycling; + // detected via string_rank_epoch_ != vocab_epoch_). mutable so the const + // sorted_ids() can fill it on demand. + mutable std::vector string_rank_; + mutable uint64_t string_rank_epoch_ = 0; // vocab_epoch_ the rank was built at +}; + +// TEST-ONLY observability seam (mirrors the reader-side decode-counter pattern). +// Counts how many times a vocabulary string is MATERIALIZED into owned_vocab_ during +// owned-mode interning. With single-store interning this is bumped EXACTLY ONCE per +// DISTINCT term (the owned_vocab_.emplace_back) and NEVER per token -- so feeding the +// same term M times still materializes it once, and the per-token temporary probe +// string is gone entirely. Writer tests use it for deterministic allocation +// assertions (count == distinct terms). Process-global; reset between tests. Not part +// of the production API. +namespace testing { +// G11 bench seam (honored under BE_TEST only): disables the add-path +// prefetch hints so the locality bench can A/B them within ONE process. +// Production builds prefetch unconditionally. +void set_bench_disable_g11_prefetch(bool disabled); + +uint64_t vocab_string_materialization_count(); +void reset_vocab_string_materialization_count(); + +// G04 bigram vocab-cap observability seams (same always-on relaxed-atomic +// pattern). Deterministic on the single-threaded build path; reset between +// tests. Not part of the production API. +// bigram_evictions : terms evicted from the intern table (cap sweep + spill +// pass combined); each was recorded in the drop bloom. +// vocab_cap_sweeps : bounded incremental sweep STEPS executed (each scans at +// most kVocabSweepStride vocabulary ids). +uint64_t bigram_evictions(); +uint64_t vocab_cap_sweeps(); +void reset_bigram_vocab_cap_counters(); + +// G05 pair-keyed bigram observability seams (same always-on relaxed-atomic +// pattern; deterministic on the single-threaded build path; reset between +// tests). One of the two is bumped per add_bigram_token(left_id, right_id) +// call that reaches the pair map: +// bigram_pair_map_hits : the pair key was already interned (the +// overwhelming majority of the per-token stream); +// bigram_pair_map_misses : first-time intern of the pair key (== distinct +// pairs interned, counting re-interns after an +// eviction). +uint64_t bigram_pair_map_hits(); +uint64_t bigram_pair_map_misses(); +void reset_bigram_pair_map_counters(); + +// G06 drain-side df-gate seam: pair-keyed bigram terms dropped by +// prepare_pair_terms_for_drain's df gate WITHOUT materialization -- final-drain +// drops (no bloom) and mid-feed df-gate drops (bloomed via the G04 eviction) +// both count; plain df==1 evictions (the pre-G06 G04 rule, taken before the +// gate is consulted) do NOT. Like the pair-map seams above, the increment is +// compiled ONLY under BE_TEST: the final drain executes it once per live pair +// term (hundreds of millions per wikipedia segment, across concurrent +// writers), where an always-on shared atomic would cache-line ping-pong. +// Deterministic on the single-threaded build path; reset between tests. Not +// part of the production API. +uint64_t bigram_drain_df_drops(); +void reset_bigram_drain_df_drops(); + +// G09 process-wide limiter seam: spills that observed -- and cleared -- a +// PENDING global forced-spill request at the moment they fired (whether or not +// the per-writer gate would also have spilled that token; the request was +// consumed either way). Incremented under BE_TEST only, matching the pair-map +// seams' contention rationale (the check sits on the per-token path of every +// concurrent writer). Deterministic on the single-threaded build path; reset +// between tests. Not part of the production API. +uint64_t global_forced_spills(); +void reset_global_forced_spills(); + +// G09 run-file cap seam: merge-compactions of a buffer's accumulated spill +// runs (each collapses the whole run list into one file). Always-on relaxed +// atomic (a compaction is rare -- at most once per cap-many spills -- so +// contention is a non-issue, unlike the per-token seams above). Deterministic +// on the single-threaded build path; reset between tests. Not part of the +// production API. +uint64_t run_compactions(); +void reset_run_compactions(); +} // namespace testing + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/temp_dir.cpp b/be/src/storage/index/snii/writer/temp_dir.cpp new file mode 100644 index 00000000000000..23da77c6a2ecff --- /dev/null +++ b/be/src/storage/index/snii/writer/temp_dir.cpp @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/temp_dir.h" + +#include +#include +#include + +#include "runtime/exec_env.h" +#include "storage/index/index_writer.h" // segment_v2::TmpFileDirs (full definition) + +namespace doris::snii::writer { + +std::string resolve_temp_dir() { + // Use Doris's configured spill/scratch dirs (the same source the inverted-index + // writer uses; see index_file_writer.cpp). SNII spills/section temp files live in + // a dedicated "snii" subdirectory so they do not crowd the tmp root alongside + // every other component's files. + auto dir = ExecEnv::GetInstance()->get_tmp_file_dirs()->get_tmp_file_dir(); + dir /= "snii"; + std::error_code ec; + std::filesystem::create_directories(dir, ec); + return dir.native(); +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/index/snii/writer/temp_dir.h b/be/src/storage/index/snii/writer/temp_dir.h new file mode 100644 index 00000000000000..1b249087cbd213 --- /dev/null +++ b/be/src/storage/index/snii/writer/temp_dir.h @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +namespace doris::snii::writer { + +// Scratch directory for spill runs and section temp files. Uses Doris's configured +// tmp_file_dirs (ExecEnv::get_tmp_file_dirs) in production; when those are not +// initialized (unit tests / standalone) it falls back to SNII_TEMP_DIR -> TMPDIR -> +// /tmp. Defined in temp_dir.cpp to keep exec_env.h out of this header. +// +// The fallback (SNII_TEMP_DIR / TMPDIR) should point at a REAL disk (SSD/NVMe): +// /tmp is often tmpfs (RAM-backed), where spilling does NOT reduce RSS. +std::string resolve_temp_dir(); + +// Best-effort free bytes on the filesystem backing `dir`. Returns UINT64_MAX when +// statvfs fails, so a caller's space pre-check never false-positives on an +// unstattable path. CAVEATS: this is best-effort only -- it is subject to TOCTOU +// (free space can drop before/while the write runs), and on tmpfs it reports +// RAM-backed space (use the temp-dir config to avoid tmpfs in the first place). +inline uint64_t temp_dir_available_bytes(const std::string& dir) { + struct statvfs vfs; + if (::statvfs(dir.c_str(), &vfs) != 0) return UINT64_MAX; + return static_cast(vfs.f_bavail) * static_cast(vfs.f_frsize); +} + +} // namespace doris::snii::writer diff --git a/be/src/storage/rowset/beta_rowset.cpp b/be/src/storage/rowset/beta_rowset.cpp index 5017c0f136e377..0d575feca665a5 100644 --- a/be/src/storage/rowset/beta_rowset.cpp +++ b/be/src/storage/rowset/beta_rowset.cpp @@ -829,6 +829,9 @@ Status BetaRowset::show_nested_index_file(rapidjson::Value* rowset_value, case InvertedIndexStorageFormatPB::V3: format_str = "V3"; break; + case InvertedIndexStorageFormatPB::SNII: + format_str = "SNII"; + break; default: return Status::InternalError("inverted index storage format error"); break; @@ -838,6 +841,19 @@ Status BetaRowset::show_nested_index_file(rapidjson::Value* rowset_value, rowset_value->AddMember("index_storage_format", rapidjson::Value(format_str.c_str(), allocator), allocator); rapidjson::Value segments(rapidjson::kArrayType); + auto add_file_info_to_json = [&](const std::string& path, + rapidjson::Value& json_value) -> Status { + json_value.AddMember("idx_file_path", rapidjson::Value(path.c_str(), allocator), allocator); + int64_t idx_file_size = 0; + auto st = fs->file_size(path, &idx_file_size); + if (st != Status::OK()) { + LOG(WARNING) << "show nested index file get file size error, file: " << path + << ", error: " << st.msg(); + return st; + } + json_value.AddMember("idx_file_size", rapidjson::Value(idx_file_size).Move(), allocator); + return Status::OK(); + }; for (int seg_id = 0; seg_id < num_segments(); ++seg_id) { rapidjson::Value segment(rapidjson::kObjectType); segment.AddMember("segment_id", rapidjson::Value(seg_id).Move(), allocator); @@ -848,24 +864,20 @@ Status BetaRowset::show_nested_index_file(rapidjson::Value* rowset_value, fs, std::string(index_file_path_prefix), storage_format, InvertedIndexFileInfo(), _rowset_meta->tablet_id()); RETURN_IF_ERROR(index_file_reader->init()); + if (storage_format == InvertedIndexStorageFormatPB::SNII) { + rapidjson::Value index_file(rapidjson::kObjectType); + auto index_file_path = + InvertedIndexDescriptor::get_index_file_path_v2(index_file_path_prefix); + RETURN_IF_ERROR(add_file_info_to_json(index_file_path, index_file)); + segment.AddMember("index_files", rapidjson::Value(rapidjson::kArrayType).Move(), + allocator); + auto& index_files = segment["index_files"]; + index_files.PushBack(index_file, allocator); + segments.PushBack(segment, allocator); + continue; + } auto dirs = index_file_reader->get_all_directories(); - auto add_file_info_to_json = [&](const std::string& path, - rapidjson::Value& json_value) -> Status { - json_value.AddMember("idx_file_path", rapidjson::Value(path.c_str(), allocator), - allocator); - int64_t idx_file_size = 0; - auto st = fs->file_size(path, &idx_file_size); - if (st != Status::OK()) { - LOG(WARNING) << "show nested index file get file size error, file: " << path - << ", error: " << st.msg(); - return st; - } - json_value.AddMember("idx_file_size", rapidjson::Value(idx_file_size).Move(), - allocator); - return Status::OK(); - }; - auto process_files = [&allocator, &index_file_reader](auto& index_meta, rapidjson::Value& indices, rapidjson::Value& index) -> Status { diff --git a/be/src/storage/segment/column_reader.cpp b/be/src/storage/segment/column_reader.cpp index f15e7462e6c11a..e41dfcdcbb0700 100644 --- a/be/src/storage/segment/column_reader.cpp +++ b/be/src/storage/segment/column_reader.cpp @@ -56,6 +56,7 @@ #include "storage/index/index_reader.h" #include "storage/index/inverted/analyzer/analyzer.h" #include "storage/index/inverted/inverted_index_reader.h" +#include "storage/index/snii/snii_index_reader.h" #include "storage/index/zone_map/zone_map_index.h" #include "storage/iterators.h" #include "storage/olap_common.h" @@ -722,6 +723,17 @@ Status ColumnReader::_load_index(const std::shared_ptr& index_f } IndexReaderPtr index_reader; + if (index_file_reader->get_storage_format() == InvertedIndexStorageFormatPB::SNII) { + if (!is_string_type(type)) { + return Status::Error( + "SNII inverted index storage format does not support BKD index type {}", type); + } + auto reader_type = should_analyzer ? InvertedIndexReaderType::FULLTEXT + : InvertedIndexReaderType::STRING_TYPE; + index_reader = SniiIndexReader::create_shared(index_meta, index_file_reader, reader_type); + _index_readers[index_meta->index_id()] = index_reader; + return Status::OK(); + } if (is_string_type(type)) { if (should_analyzer) { diff --git a/be/src/storage/segment/column_writer.cpp b/be/src/storage/segment/column_writer.cpp index 7d604897494f7d..369c325293f6d0 100644 --- a/be/src/storage/segment/column_writer.cpp +++ b/be/src/storage/segment/column_writer.cpp @@ -565,6 +565,13 @@ Status ScalarColumnWriter::init() { RETURN_IF_ERROR(IndexColumnWriter::create( get_column(), &_inverted_index_builders[i], _opts.index_file_writer, _opts.inverted_indexes[i])); + // After create() (which runs the writer's init()) and before any + // value lands: forward the write type UNCONDITIONALLY so + // write-type-aware index builds (SNII bigram deferral) can + // distinguish loads from compaction / schema change -- and the + // documented "forwarded to every created IndexColumnWriter" + // contract holds for both values. + _inverted_index_builders[i]->set_direct_load(_opts.is_direct_load); } } while (false); } @@ -1056,6 +1063,9 @@ Status ArrayColumnWriter::init() { RETURN_IF_ERROR(IndexColumnWriter::create(get_column(), &_inverted_index_writer, _opts.index_file_writer, _opts.inverted_indexes[0])); + // Same unconditional forwarding as the scalar path: after + // create()/init(), before any array value is added. + _inverted_index_writer->set_direct_load(_opts.is_direct_load); } } if (_opts.need_ann_index) { diff --git a/be/src/storage/segment/column_writer.h b/be/src/storage/segment/column_writer.h index 50822c945c1bb7..75df145b08699d 100644 --- a/be/src/storage/segment/column_writer.h +++ b/be/src/storage/segment/column_writer.h @@ -78,6 +78,13 @@ struct ColumnWriterOptions { BloomFilterOptions bf_options; std::vector inverted_indexes; IndexFileWriter* index_file_writer = nullptr; + // The owning segment serves a direct load (stream/broker load, + // DataWriteType::TYPE_DIRECT) rather than compaction / schema change. Set + // once by the segment writer and propagated to variant subcolumn writers; + // forwarded to every created IndexColumnWriter via set_direct_load() so a + // write-type-aware index build (SNII bigram deferral) can tell loads apart + // without plumbing DataWriteType itself down here. + bool is_direct_load = false; SegmentFooterPB* footer = nullptr; io::FileWriter* file_writer = nullptr; diff --git a/be/src/storage/segment/count_on_index_fastpath.h b/be/src/storage/segment/count_on_index_fastpath.h new file mode 100644 index 00000000000000..30b0ccf7218157 --- /dev/null +++ b/be/src/storage/segment/count_on_index_fastpath.h @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +// G02 count-only fast-path caller guard (functional core, unit-testable +// without a SegmentIterator). +// +// COUNT_ON_INDEX counts the rows of THIS segment that match the pushed-down +// predicates MINUS deleted rows: SegmentIterator seeds _row_bitmap with +// [0, num_rows), intersects the index result bitmap into it, then subtracts +// the MOW delete bitmap and applies delete predicates / row ranges, and the +// scan emits |_row_bitmap| default-valued rows that the agg counts. The SNII +// fast path replaces the index result with a FABRICATED [0, df) bitmap whose +// cardinality is exact but whose row ids are not real. That is only equal in +// observable behavior when, for this segment iterator: +// 1. only the COUNT matters (COUNT_ON_INDEX agg pushdown), +// 2. the single pushed-down MATCH predicate is the ONLY filter (nothing else +// may intersect _row_bitmap before or after the index apply), +// 3. no rows are deleted (mirror of the V3 handling: _lazy_init subtracts +// the per-segment delete bitmap AFTER the index apply, and delete +// predicates filter later -- a fabricated id range cannot participate in +// either subtraction), +// 4. nothing consumes REAL row ids (rowid recording, ANN topn, BM25 +// scoring, virtual columns), and +// 5. rows are emitted as defaults without reading column data at the +// fabricated ids (the COUNT_ON_INDEX no-read-data contract must be +// active: enable_no_need_read_data_opt + DUP_KEYS or MOW). +// +// SegmentIterator fills the facts from its state right before applying the +// index and only sets IndexQueryContext::count_on_index_fastpath when this +// predicate holds. +namespace doris::segment_v2 { + +struct CountOnIndexFastpathFacts { + // (1) count-only context. + bool is_count_on_index_agg = false; + // (2) single MATCH predicate, no other conjuncts. + bool has_column_predicates = false; + size_t common_expr_count = 0; + bool single_expr_is_match_pred = false; + bool has_virtual_column_exprs = false; + // (3) deletes must be absent for this segment. + bool has_delete_predicates = false; + bool segment_delete_bitmap_empty = false; + // (2 cont.) nothing else may prune the row space around the index apply. + bool has_col_id_predicates = false; + bool has_topn_filters = false; + bool has_external_row_ranges = false; + bool row_bitmap_is_full = false; + // (4) consumers of real row ids. + bool record_rowids = false; + bool has_ann_topn = false; + bool has_score_runtime = false; + // (5) rows must be emitted as defaults (no data read at fabricated ids). + bool no_need_read_data_opt_enabled = false; + bool keys_type_supported = false; +}; + +inline bool count_on_index_fastpath_safe(const CountOnIndexFastpathFacts& f) { + return f.is_count_on_index_agg && !f.has_column_predicates && f.common_expr_count == 1 && + f.single_expr_is_match_pred && !f.has_virtual_column_exprs && !f.has_delete_predicates && + f.segment_delete_bitmap_empty && !f.has_col_id_predicates && !f.has_topn_filters && + !f.has_external_row_ranges && f.row_bitmap_is_full && !f.record_rowids && + !f.has_ann_topn && !f.has_score_runtime && f.no_need_read_data_opt_enabled && + f.keys_type_supported; +} + +// G03 count-emission shortcut guard (functional core, unit-testable without a +// SegmentIterator). +// +// After the G02 fast path answered the single MATCH predicate with a +// count-shaped bitmap, the only remaining work of the scan is to emit +// |_row_bitmap| default-valued rows batch by batch: the per-batch rowid +// iteration over the fabricated bitmap and the per-column no-read checks are +// pure overhead. The shortcut replaces them with a countdown that fills the +// block columns with defaults directly, in VStatisticsIterator-sized batches. +// +// That replacement is byte-for-byte equal to today's emission only when, at +// the end of _lazy_init: +// 1. the reader ACTUALLY answered from df (count_fastpath_hit) -- a mere +// guard pass with a row-accurate decode keeps today's path untouched, +// 2. no evaluation stage survives (vec/short-circuit/expr eval, leftover +// column predicates or common exprs, delete predicates, lazy +// materialization), +// 3. nothing consumes real row ids or per-row values (virtual columns, +// rowid recording), +// 4. batch accounting is a pure countdown (no read limit, no reverse +// key-ordered read, no condition-cache writes), and +// 5. the block is exactly the read schema and EVERY column would take a +// defaults fill in _read_columns_by_index (no real column read, no +// storage->schema cast, no version/lsn/tso rewrite) -- checked +// per-column by the iterator and summarized in one fact. +// +// Every fact is re-verified from live iterator state even though the G02 +// facts guard already implies most of them: the shortcut independently +// refuses on any drift, falling through to today's emission (which is always +// count-exact). +struct CountEmitShortcutFacts { + // (1) reader answered with a fabricated count bitmap. + bool count_fastpath_hit = false; + // (2) nothing evaluates or filters rows after the index apply. + bool needs_vec_eval = false; + bool needs_short_eval = false; + bool needs_expr_eval = false; + bool has_remaining_col_predicates = false; + bool has_remaining_common_exprs = false; + bool has_delete_predicates = false; + bool lazy_materialization_read = false; + // (3) consumers of real row ids / per-row values. + bool has_virtual_columns = false; + bool record_rowids = false; + // (4) batch accounting must be a pure countdown. + bool has_read_limit = false; + bool read_orderby_key_reverse = false; + bool has_condition_cache_digest = false; + // (5) emitted block == read schema, all columns defaults-fillable. + bool block_shape_matches_schema = false; + bool all_columns_emit_defaults = false; +}; + +inline bool count_emit_shortcut_safe(const CountEmitShortcutFacts& f) { + return f.count_fastpath_hit && !f.needs_vec_eval && !f.needs_short_eval && !f.needs_expr_eval && + !f.has_remaining_col_predicates && !f.has_remaining_common_exprs && + !f.has_delete_predicates && !f.lazy_materialization_read && !f.has_virtual_columns && + !f.record_rowids && !f.has_read_limit && !f.read_orderby_key_reverse && + !f.has_condition_cache_digest && f.block_shape_matches_schema && + f.all_columns_emit_defaults; +} + +} // namespace doris::segment_v2 + +// Deterministic seam for the G03 count-emission shortcut, mirroring the SNII +// query seam (storage/index/snii/query/internal/query_test_counters.h): +// - count_emit_shortcut_hits : engage decisions that ADMITTED the +// shortcut (incremented inside +// _should_engage_count_emit_shortcut, which +// _lazy_init calls once per iterator). Guard +// fall-throughs leave it unchanged. +// - count_emit_shortcut_batches : default-rows batches emitted by the +// shortcut (== ceil(count / 65535) per +// engaged iterator with a non-zero count). +// +// Active only under BE_TEST (library-wide define of doris_be_test); in a +// release build the macro expands to ((void)0): zero overhead, no global +// mutable state on the production path. The singleton is intentionally +// unsynchronized -- single-threaded test-only seam; reset between cases with +// `count_emit_test_counters() = {}`. +#if defined(BE_TEST) && !defined(SNII_COUNT_EMIT_TEST_COUNTERS) +#define SNII_COUNT_EMIT_TEST_COUNTERS +#endif + +#ifdef SNII_COUNT_EMIT_TEST_COUNTERS + +namespace doris::segment_v2::internal { + +struct CountEmitTestCounters { + uint64_t count_emit_shortcut_hits = 0; + uint64_t count_emit_shortcut_batches = 0; +}; + +// `inline` gives a single shared instance across all TUs that include this +// header, so counter increments made in segment_iterator.cpp are visible to +// the test that reads them. +inline CountEmitTestCounters& count_emit_test_counters() { + static CountEmitTestCounters counters; + return counters; +} + +} // namespace doris::segment_v2::internal + +#define SNII_COUNT_EMIT_COUNT(field) \ + (++::doris::segment_v2::internal::count_emit_test_counters().field) + +#else + +#define SNII_COUNT_EMIT_COUNT(field) ((void)0) + +#endif diff --git a/be/src/storage/segment/row_ranges.h b/be/src/storage/segment/row_ranges.h index 0f44f599695b8c..9d98f02e245b15 100644 --- a/be/src/storage/segment/row_ranges.h +++ b/be/src/storage/segment/row_ranges.h @@ -247,7 +247,7 @@ class RowRanges { size_t count() { return _count; } - bool is_empty() { return _count == 0; } + bool is_empty() const { return _count == 0; } bool contain(rowid_t from, rowid_t to) { // binary search diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index f667efc0db14cc..134678f67ba533 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -103,6 +103,7 @@ #include "storage/segment/column_reader.h" #include "storage/segment/column_reader_cache.h" #include "storage/segment/condition_cache.h" +#include "storage/segment/count_on_index_fastpath.h" #include "storage/segment/row_ranges.h" #include "storage/segment/segment.h" #include "storage/segment/segment_prefetcher.h" @@ -614,6 +615,16 @@ Status SegmentIterator::_lazy_init(Block* block) { _init_segment_prefetchers(); + // G03: engage the count-emission shortcut. All inputs are final here (the + // index apply ran, _row_bitmap saw every subtraction/intersection above, + // _vec_init_lazy_materialization fixed the eval flags), so the post-apply + // cardinality IS the exact row count today's batch loop would emit; the + // shortcut only changes how fast those default rows are produced. + _count_emit_shortcut = _should_engage_count_emit_shortcut(block); + if (_count_emit_shortcut) { + _count_emit_rows_remaining = _row_bitmap.cardinality(); + } + return Status::OK(); } @@ -825,6 +836,18 @@ Status SegmentIterator::_get_row_ranges_by_column_conditions() { (has_index_in_iterators() || !_common_expr_ctxs_push_down.empty())) { SCOPED_RAW_TIMER(&_opts.stats->inverted_index_filter_timer); size_t input_rows = _row_bitmap.cardinality(); + // G02 count-only fast path handshake: only while the single + // pushed-down MATCH predicate of a provably filter-free + // COUNT_ON_INDEX scan is evaluated may a reader answer with a + // count-shaped bitmap (see count_on_index_fastpath.h). The reply + // direction (did the reader actually fabricate one?) is captured + // into _count_fastpath_hit and both flags are reset on every exit + // path so no later read_from_index call can observe or forge them. + if (_index_query_context != nullptr) { + _index_query_context->count_on_index_fastpath = _count_on_index_fastpath_safe(); + _index_query_context->count_on_index_fastpath_hit = false; + } + DEFER({ _capture_count_fastpath_hit(); }); // Only apply column-level inverted index if we have iterators if (has_index_in_iterators()) { RETURN_IF_ERROR(_apply_inverted_index()); @@ -1337,6 +1360,146 @@ Status SegmentIterator::_apply_index_expr() { return Status::OK(); } +bool SegmentIterator::_count_on_index_fastpath_safe() const { + CountOnIndexFastpathFacts facts; + facts.is_count_on_index_agg = _opts.push_down_agg_type_opt == TPushAggOp::COUNT_ON_INDEX; + facts.has_column_predicates = !_col_predicates.empty(); + facts.common_expr_count = _common_expr_ctxs_push_down.size(); + facts.single_expr_is_match_pred = + _common_expr_ctxs_push_down.size() == 1 && + _common_expr_ctxs_push_down.front()->root() != nullptr && + _common_expr_ctxs_push_down.front()->root()->node_type() == TExprNodeType::MATCH_PRED; + facts.has_virtual_column_exprs = !_virtual_column_exprs.empty(); + facts.has_delete_predicates = _opts.delete_condition_predicates != nullptr && + _opts.delete_condition_predicates->num_of_column_predicate() > 0; + // Mirror of the _lazy_init delete-bitmap subtraction: the fast path is only + // sound when there is nothing to subtract for THIS segment. + const auto delete_bitmap_it = _opts.delete_bitmap.find(segment_id()); + facts.segment_delete_bitmap_empty = delete_bitmap_it == _opts.delete_bitmap.end() || + delete_bitmap_it->second == nullptr || + delete_bitmap_it->second->isEmpty(); + facts.has_col_id_predicates = !_opts.col_id_to_predicates.empty(); + facts.has_topn_filters = !_opts.topn_filter_source_node_ids.empty(); + facts.has_external_row_ranges = !_opts.row_ranges.is_empty(); + // Catches every earlier pruning source in one check (condition cache, key + // ranges): the fabricated [0, df) range only counts correctly against a + // full [0, num_rows) bitmap. + facts.row_bitmap_is_full = _row_bitmap.cardinality() == uint64_t(num_rows()); + facts.record_rowids = _opts.record_rowids; + facts.has_ann_topn = _opts.ann_topn_runtime != nullptr; + facts.has_score_runtime = _score_runtime != nullptr; + // Mirror of the _need_read_data preamble: rows must be emitted as defaults, + // never materialized from the fabricated row ids. + facts.no_need_read_data_opt_enabled = + _opts.runtime_state == nullptr || + _opts.runtime_state->query_options().enable_no_need_read_data_opt; + facts.keys_type_supported = _opts.tablet_schema->keys_type() == KeysType::DUP_KEYS || + (_opts.tablet_schema->keys_type() == KeysType::UNIQUE_KEYS && + _opts.enable_unique_key_merge_on_write); + return count_on_index_fastpath_safe(facts); +} + +void SegmentIterator::_capture_count_fastpath_hit() { + if (_index_query_context == nullptr) { + return; + } + _count_fastpath_hit = _index_query_context->count_on_index_fastpath_hit; + _index_query_context->count_on_index_fastpath = false; + _index_query_context->count_on_index_fastpath_hit = false; +} + +bool SegmentIterator::_column_emits_defaults_for_count(ColumnId cid) { + // Mirror of the per-batch fill in _read_columns_by_index: a column's batch + // content is reproducible by the emission shortcut iff the column takes + // the _no_need_read_key_data defaults fill or the _prune_column defaults + // fill. Anything else -- a real column read, a storage->schema cast in + // _init_current_block/_convert_to_expected_type (whose CAST(default) need + // not equal the schema-type default), or a version/lsn/tso rewrite source + // (those columns are never no-read, so the type/read checks below already + // veto them) -- must keep today's path. + if (_is_pred_column[cid]) { + return false; + } + const auto* column_desc = _schema->column(cid); + if (column_desc == nullptr) { + return false; + } + const auto& file_column_type = _storage_name_and_type[cid].second; + DataTypePtr expected_type = Schema::get_data_type_ptr(*column_desc); + if (file_column_type == nullptr || expected_type == nullptr || + !file_column_type->equals(*expected_type)) { + return false; + } + return _no_need_read_key_data_eligible(cid) || !_need_read_data(cid); +} + +bool SegmentIterator::_should_engage_count_emit_shortcut(const Block* block) { + if (!_count_fastpath_hit) { + return false; + } + CountEmitShortcutFacts facts; + facts.count_fastpath_hit = _count_fastpath_hit; + facts.needs_vec_eval = _is_need_vec_eval; + facts.needs_short_eval = _is_need_short_eval; + facts.needs_expr_eval = _is_need_expr_eval; + facts.has_remaining_col_predicates = !_col_predicates.empty(); + facts.has_remaining_common_exprs = !_common_expr_ctxs_push_down.empty(); + facts.has_delete_predicates = _opts.delete_condition_predicates != nullptr && + _opts.delete_condition_predicates->num_of_column_predicate() > 0; + facts.lazy_materialization_read = _lazy_materialization_read; + facts.has_virtual_columns = !_virtual_column_exprs.empty(); + facts.record_rowids = _opts.record_rowids || _record_rowids; + facts.has_read_limit = _opts.read_limit > 0; + facts.read_orderby_key_reverse = _opts.read_orderby_key_reverse; + facts.has_condition_cache_digest = _opts.condition_cache_digest != 0; + facts.block_shape_matches_schema = block->columns() == _schema->num_column_ids(); + facts.all_columns_emit_defaults = true; + for (size_t i = 0; i < _schema->num_column_ids() && facts.all_columns_emit_defaults; ++i) { + facts.all_columns_emit_defaults = _column_emits_defaults_for_count(_schema->column_id(i)); + } + const bool engage = count_emit_shortcut_safe(facts); + if (engage) { + SNII_COUNT_EMIT_COUNT(count_emit_shortcut_hits); + } + return engage; +} + +Status SegmentIterator::_emit_count_shortcut_batch(Block* block) { + if (_count_emit_rows_remaining == 0) { + // EOF twin of _process_eof: shortcut batches never move the block's + // columns into _current_return_columns, so there is nothing to + // restore; deliver the empty block and release iterator memory the + // same way. + block->clear_column_data(); + _column_iterators.clear(); + _index_iterators.clear(); + return Status::EndOfFile("no more data in segment"); + } + const auto rows = static_cast( + std::min(_count_emit_rows_remaining, kCountEmitBatchRows)); + block->clear_column_data(_schema->num_column_ids()); + for (size_t i = 0; i < _schema->num_column_ids(); ++i) { + MutableColumnPtr column = std::move(*block->get_by_position(i).column).mutate(); + // Same fill as the defaults branches of _read_columns_by_index + // (_no_need_read_key_data / _prune_column): NOT-NULL defaults. + // ColumnNullable::insert_many_defaults would insert NULLs and break + // count(col) parity. + if (is_column_nullable(*column)) { + auto* nullable_col_ptr = reinterpret_cast(column.get()); + nullable_col_ptr->get_null_map_column().insert_many_defaults(rows); + nullable_col_ptr->get_nested_column_ptr()->insert_many_defaults(rows); + } else { + column->insert_many_defaults(rows); + } + block->replace_by_position(i, std::move(column)); + } + _count_emit_rows_remaining -= rows; + _opts.stats->blocks_load += 1; + _opts.stats->raw_rows_read += rows; + SNII_COUNT_EMIT_COUNT(count_emit_shortcut_batches); + return Status::OK(); +} + bool SegmentIterator::_downgrade_without_index(Status res, bool need_remaining) { bool is_fallback = _opts.runtime_state->query_options().enable_fallback_on_missing_inverted_index; @@ -2937,6 +3100,14 @@ Status SegmentIterator::_next_batch_internal(Block* block) { SCOPED_RAW_TIMER(&_opts.stats->block_load_ns); + // G03: a count-fastpath scan emits the remaining count as default rows in + // statistics-sized batches; the row-bitmap iterator and the per-rowid + // machinery below are never touched. Engaged only when read_limit == 0, + // so the limit check below stays unreachable for shortcut scans. + if (_count_emit_shortcut) { + return _emit_count_shortcut_batch(block); + } + if (_opts.read_limit > 0 && _rows_returned >= _opts.read_limit) { return _process_eof(block); } @@ -3540,8 +3711,7 @@ void SegmentIterator::_calculate_common_expr_index_exec_status() { } } -bool SegmentIterator::_no_need_read_key_data(ColumnId cid, MutableColumnPtr& column, - size_t nrows_read) { +bool SegmentIterator::_no_need_read_key_data_eligible(ColumnId cid) { if (_opts.runtime_state && !_opts.runtime_state->query_options().enable_no_need_read_data_opt) { return false; } @@ -3568,6 +3738,14 @@ bool SegmentIterator::_no_need_read_key_data(ColumnId cid, MutableColumnPtr& col return false; } + return true; +} + +bool SegmentIterator::_no_need_read_key_data(ColumnId cid, MutableColumnPtr& column, + size_t nrows_read) { + if (!_no_need_read_key_data_eligible(cid)) { + return false; + } insert_many_not_null_defaults(column, nrows_read); return true; } diff --git a/be/src/storage/segment/segment_iterator.h b/be/src/storage/segment/segment_iterator.h index 29735a16badfa9..c8949cb82669e2 100644 --- a/be/src/storage/segment/segment_iterator.h +++ b/be/src/storage/segment/segment_iterator.h @@ -198,6 +198,32 @@ class SegmentIterator : public RowwiseIterator { bool* continue_apply); [[nodiscard]] Status _apply_ann_topn_predicate(); [[nodiscard]] Status _apply_index_expr(); + // G02: true iff answering the single pushed-down MATCH predicate by its + // match COUNT alone is indistinguishable from the row-accurate bitmap for + // this COUNT_ON_INDEX scan (no deletes, no other filters, full row bitmap, + // no row-id consumers). Gates IndexQueryContext::count_on_index_fastpath; + // the decision predicate itself lives in count_on_index_fastpath.h. + bool _count_on_index_fastpath_safe() const; + // G03: teardown of the G02 handshake. Captures whether the reader answered + // with a fabricated count bitmap into _count_fastpath_hit and clears both + // context flags so no later read_from_index call can observe or forge + // them. Runs on every exit of the index-apply scope. + void _capture_count_fastpath_hit(); + // G03: true iff the per-batch defaults fill of _read_columns_by_index + // would apply to `cid` (the _no_need_read_key_data or _prune_column + // branch) AND the block column needs no storage->schema cast, i.e. the + // emission shortcut can reproduce the column's batch content exactly. + bool _column_emits_defaults_for_count(ColumnId cid); + // G03: fills CountEmitShortcutFacts from live iterator state at the end of + // _lazy_init and returns the pure-guard verdict; the decision predicate + // itself lives in count_on_index_fastpath.h. + bool _should_engage_count_emit_shortcut(const Block* block); + // G03: one emission-shortcut batch: min(remaining, kCountEmitBatchRows) + // default rows filled straight into the block (NOT-NULL defaults for + // nullable columns, mirroring _prune_column), then EOF once the countdown + // reaches zero. Replaces the whole per-rowid _next_batch_internal body for + // engaged scans. + Status _emit_count_shortcut_batch(Block* block); bool _column_has_fulltext_index(int32_t cid); bool _column_has_ann_index(int32_t cid); @@ -315,6 +341,10 @@ class SegmentIterator : public RowwiseIterator { Status _convert_to_expected_type(const std::vector& col_ids); bool _no_need_read_key_data(ColumnId cid, MutableColumnPtr& column, size_t nrows_read); + // Side-effect-free eligibility half of _no_need_read_key_data (no column + // fill); shared by the per-batch fill and the G03 engage-time per-column + // proof so the two can never drift. + bool _no_need_read_key_data_eligible(ColumnId cid); bool _has_delete_predicate(ColumnId cid); bool _can_skip_reading_extra_column(ColumnId cid); @@ -478,6 +508,23 @@ class SegmentIterator : public RowwiseIterator { IndexQueryContextPtr _index_query_context; + // G03 count-emission shortcut state (see count_on_index_fastpath.h). + // _count_fastpath_hit: the reader answered the single MATCH predicate with + // a fabricated count bitmap (captured from the G02 handshake reply). + // _count_emit_shortcut: engaged at the end of _lazy_init when + // count_emit_shortcut_safe holds; every subsequent batch is emitted by + // _emit_count_shortcut_batch from _count_emit_rows_remaining (initialized + // to the post-apply _row_bitmap cardinality) without touching the row + // bitmap iterator. + bool _count_fastpath_hit = false; + bool _count_emit_shortcut = false; + uint64_t _count_emit_rows_remaining = 0; + // Batch size for shortcut emission: VStatisticsIterator's + // MAX_ROW_SIZE_IN_COUNT, the largest default-rows block shape already + // proven through every consumer above the segment iterator by the plain + // COUNT pushdown (rowset reader, collect iterator, block reader, scanner). + static constexpr uint64_t kCountEmitBatchRows = 65535; + // key is column uid, value is the sparse column cache std::unordered_map _variant_sparse_column_cache; diff --git a/be/src/storage/segment/segment_writer.cpp b/be/src/storage/segment/segment_writer.cpp index 6f4ecef1140d67..f338a26f221edc 100644 --- a/be/src/storage/segment/segment_writer.cpp +++ b/be/src/storage/segment/segment_writer.cpp @@ -227,6 +227,9 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co if (_opts.write_type == DataWriteType::TYPE_DIRECT && schema->skip_write_index_on_load()) { skip_inverted_index = true; } + // let index column writers distinguish direct load (stream/broker load) + // from compaction / schema change (SNII bigram deferral hint) + opts.is_direct_load = _opts.write_type == DataWriteType::TYPE_DIRECT; // indexes for this column if (!skip_inverted_index) { auto inverted_indexs = schema->inverted_indexs(column); diff --git a/be/src/storage/segment/variant/variant_column_writer_impl.cpp b/be/src/storage/segment/variant/variant_column_writer_impl.cpp index 1e8610f42d886f..165d9ed1ca9293 100644 --- a/be/src/storage/segment/variant/variant_column_writer_impl.cpp +++ b/be/src/storage/segment/variant/variant_column_writer_impl.cpp @@ -455,6 +455,8 @@ Status prepare_materialized_subcolumn_writer( opts.rowset_ctx = base_opts.rowset_ctx; opts.file_writer = base_opts.file_writer; opts.storage_format = base_opts.storage_format; + // keep the segment writer's direct-load marking for subcolumn index writers + opts.is_direct_load = base_opts.is_direct_load; std::unique_ptr writer; variant_util::inherit_column_attributes(parent_column, tablet_column); @@ -1482,6 +1484,8 @@ Status prepare_subcolumn_writer_target( opts.rowset_ctx = base_opts.rowset_ctx; opts.file_writer = base_opts.file_writer; opts.storage_format = base_opts.storage_format; + // keep the segment writer's direct-load marking for subcolumn index writers + opts.is_direct_load = base_opts.is_direct_load; variant_util::inherit_column_attributes(parent_column, tablet_column); bool need_record_none_null_value_size = diff --git a/be/src/storage/segment/vertical_segment_writer.cpp b/be/src/storage/segment/vertical_segment_writer.cpp index f5b60ebadb756c..535b9150746cf4 100644 --- a/be/src/storage/segment/vertical_segment_writer.cpp +++ b/be/src/storage/segment/vertical_segment_writer.cpp @@ -237,6 +237,9 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo tablet_schema->skip_write_index_on_load()) { skip_inverted_index = true; } + // let index column writers distinguish direct load (stream/broker load) + // from compaction / schema change (SNII bigram deferral hint) + opts.is_direct_load = _opts.write_type == DataWriteType::TYPE_DIRECT; if (!skip_inverted_index) { auto inverted_indexs = tablet_schema->inverted_indexs(column); if (!inverted_indexs.empty()) { diff --git a/be/src/storage/tablet/tablet_meta.cpp b/be/src/storage/tablet/tablet_meta.cpp index 2fe9fd2dde7bc8..7d81e4b976d002 100644 --- a/be/src/storage/tablet/tablet_meta.cpp +++ b/be/src/storage/tablet/tablet_meta.cpp @@ -101,6 +101,9 @@ TabletMetaSharedPtr TabletMeta::create( case TInvertedIndexStorageFormat::V2: inverted_index_file_storage_format = TInvertedIndexFileStorageFormat::V2; break; + case TInvertedIndexStorageFormat::SNII: + inverted_index_file_storage_format = TInvertedIndexFileStorageFormat::SNII; + break; default: break; } @@ -495,6 +498,9 @@ void TabletMeta::init_schema_from_thrift(const TTabletSchema& tablet_schema, case TInvertedIndexFileStorageFormat::V3: tablet_schema_pb->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V3); break; + case TInvertedIndexFileStorageFormat::SNII: + tablet_schema_pb->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::SNII); + break; default: tablet_schema_pb->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V3); break; diff --git a/be/src/storage/task/index_builder.cpp b/be/src/storage/task/index_builder.cpp index ef49626e143ab5..0e0ffeeb1d1036 100644 --- a/be/src/storage/task/index_builder.cpp +++ b/be/src/storage/task/index_builder.cpp @@ -338,6 +338,13 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta if (_is_drop_op) { const auto& output_rs_tablet_schema = output_rowset_meta->tablet_schema(); + if (output_rs_tablet_schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::SNII) { + LOG(INFO) << "skip physical SNII inverted index rewrite for drop index. tablet_id=" + << _tablet->tablet_id() + << " rowset_id=" << output_rowset_meta->rowset_id().to_string(); + return Status::OK(); + } if (output_rs_tablet_schema->get_inverted_index_storage_format() != InvertedIndexStorageFormatPB::V1) { const auto& fs = output_rowset_meta->fs(); @@ -421,6 +428,11 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta _olap_data_convertor->reserve(_alter_inverted_indexes.size()); std::unique_ptr index_file_writer = nullptr; + if (output_rowset_schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::SNII) { + return Status::Error( + "BUILD INDEX is not supported for SNII inverted index storage format yet"); + } if (output_rowset_schema->get_inverted_index_storage_format() >= InvertedIndexStorageFormatPB::V2) { auto idx_file_reader_iter = _index_file_readers.find( diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index 9df8dcb54f4934..e7878003702762 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -365,6 +365,25 @@ TEST(FileScannerV2Test, FileCacheStatisticsArePublishedToScannerProfile) { file_cache_statistics.bytes_read_from_remote = 13; file_cache_statistics.bytes_read_from_peer = 17; file_cache_statistics.bytes_write_into_cache = 19; + file_cache_statistics.segment_footer_index_num_local_io_total = 23; + file_cache_statistics.segment_footer_index_num_remote_io_total = 29; + file_cache_statistics.segment_footer_index_num_peer_io_total = 31; + file_cache_statistics.segment_footer_index_bytes_read_from_local = 37; + file_cache_statistics.segment_footer_index_bytes_read_from_remote = 41; + file_cache_statistics.segment_footer_index_bytes_read_from_peer = 43; + file_cache_statistics.segment_footer_index_local_io_timer = 47; + file_cache_statistics.segment_footer_index_remote_io_timer = 53; + file_cache_statistics.segment_footer_index_peer_io_timer = 59; + file_cache_statistics.num_cross_cg_peer_io_total = 61; + file_cache_statistics.bytes_read_from_cross_cg_peer = 67; + file_cache_statistics.cross_cg_peer_io_timer = 71; + file_cache_statistics.num_same_cg_peer_io_total = 73; + file_cache_statistics.bytes_read_from_same_cg_peer = 79; + file_cache_statistics.same_cg_peer_io_timer = 83; + file_cache_statistics.num_peer_race_peer_win = 89; + file_cache_statistics.num_peer_race_s3_win = 97; + file_cache_statistics.num_peer_lazy_fetch = 101; + file_cache_statistics.peer_lazy_fetch_timer = 103; file_cache_statistics.peer_hosts = {"peer-a", "peer-b"}; FileScannerV2::TEST_report_file_cache_profile(&profile, file_cache_statistics); @@ -377,6 +396,25 @@ TEST(FileScannerV2Test, FileCacheStatisticsArePublishedToScannerProfile) { EXPECT_EQ(profile.get_counter("BytesScannedFromRemote")->value(), 13); EXPECT_EQ(profile.get_counter("BytesScannedFromPeer")->value(), 17); EXPECT_EQ(profile.get_counter("BytesWriteIntoCache")->value(), 19); + EXPECT_EQ(profile.get_counter("SegmentFooterIndexNumLocalIOTotal")->value(), 23); + EXPECT_EQ(profile.get_counter("SegmentFooterIndexNumRemoteIOTotal")->value(), 29); + EXPECT_EQ(profile.get_counter("SegmentFooterIndexNumPeerIOTotal")->value(), 31); + EXPECT_EQ(profile.get_counter("SegmentFooterIndexBytesScannedFromCache")->value(), 37); + EXPECT_EQ(profile.get_counter("SegmentFooterIndexBytesScannedFromRemote")->value(), 41); + EXPECT_EQ(profile.get_counter("SegmentFooterIndexBytesScannedFromPeer")->value(), 43); + EXPECT_EQ(profile.get_counter("SegmentFooterIndexLocalIOUseTimer")->value(), 47); + EXPECT_EQ(profile.get_counter("SegmentFooterIndexRemoteIOUseTimer")->value(), 53); + EXPECT_EQ(profile.get_counter("SegmentFooterIndexPeerIOUseTimer")->value(), 59); + EXPECT_EQ(profile.get_counter("CrossCGPeerIOTotal")->value(), 61); + EXPECT_EQ(profile.get_counter("CrossCGPeerBytesRead")->value(), 67); + EXPECT_EQ(profile.get_counter("CrossCGPeerIOTime")->value(), 71); + EXPECT_EQ(profile.get_counter("SameCGPeerIOTotal")->value(), 73); + EXPECT_EQ(profile.get_counter("SameCGPeerBytesRead")->value(), 79); + EXPECT_EQ(profile.get_counter("SameCGPeerIOTime")->value(), 83); + EXPECT_EQ(profile.get_counter("PeerRaceWin")->value(), 89); + EXPECT_EQ(profile.get_counter("S3RaceWin")->value(), 97); + EXPECT_EQ(profile.get_counter("PeerLazyFetch")->value(), 101); + EXPECT_EQ(profile.get_counter("PeerLazyFetchTime")->value(), 103); ASSERT_NE(profile.get_info_string("PeerCacheNodes"), nullptr); EXPECT_EQ(*profile.get_info_string("PeerCacheNodes"), "peer-a, peer-b"); } diff --git a/be/test/io/cache/block_file_cache_profile_reporter_test.cpp b/be/test/io/cache/block_file_cache_profile_reporter_test.cpp index e74ad758ac1db3..4e7bb6bb1d05a4 100644 --- a/be/test/io/cache/block_file_cache_profile_reporter_test.cpp +++ b/be/test/io/cache/block_file_cache_profile_reporter_test.cpp @@ -52,6 +52,10 @@ io::FileCacheStatistics make_file_cache_stats(int64_t multiplier) { stats.inverted_index_remote_io_timer = multiplier * 26; stats.inverted_index_peer_io_timer = multiplier * 27; stats.inverted_index_io_timer = multiplier * 28; + stats.inverted_index_request_bytes = multiplier * 29; + stats.inverted_index_read_bytes = multiplier * 30; + stats.inverted_index_range_read_count = multiplier * 31; + stats.inverted_index_serial_read_rounds = multiplier * 32; return stats; } @@ -89,6 +93,10 @@ void expect_file_cache_stats_eq(const io::FileCacheStatistics& actual, EXPECT_EQ(actual.inverted_index_remote_io_timer, expected.inverted_index_remote_io_timer); EXPECT_EQ(actual.inverted_index_peer_io_timer, expected.inverted_index_peer_io_timer); EXPECT_EQ(actual.inverted_index_io_timer, expected.inverted_index_io_timer); + EXPECT_EQ(actual.inverted_index_request_bytes, expected.inverted_index_request_bytes); + EXPECT_EQ(actual.inverted_index_read_bytes, expected.inverted_index_read_bytes); + EXPECT_EQ(actual.inverted_index_range_read_count, expected.inverted_index_range_read_count); + EXPECT_EQ(actual.inverted_index_serial_read_rounds, expected.inverted_index_serial_read_rounds); } } // namespace @@ -134,6 +142,14 @@ TEST(FileCacheProfileReporterTest, ReporterAggregatesDeltaReportsToExactFinalTot EXPECT_EQ(profile->get_counter("CacheGetOrSetTimer")->value(), after_second_report.cache_get_or_set_timer); EXPECT_EQ(profile->get_counter("LockWaitTimer")->value(), after_second_report.lock_wait_timer); + EXPECT_EQ(profile->get_counter("InvertedIndexRequestBytes")->value(), + after_second_report.inverted_index_request_bytes); + EXPECT_EQ(profile->get_counter("InvertedIndexReadBytes")->value(), + after_second_report.inverted_index_read_bytes); + EXPECT_EQ(profile->get_counter("InvertedIndexRangeReadCount")->value(), + after_second_report.inverted_index_range_read_count); + EXPECT_EQ(profile->get_counter("InvertedIndexSerialReadRounds")->value(), + after_second_report.inverted_index_serial_read_rounds); } } // namespace doris diff --git a/be/test/storage/index/snii/common/single_flight_test.cpp b/be/test/storage/index/snii/common/single_flight_test.cpp new file mode 100644 index 00000000000000..37998fef8ba061 --- /dev/null +++ b/be/test/storage/index/snii/common/single_flight_test.cpp @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/common/single_flight.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace doris::snii { + +// A lone caller leads, a second concurrent caller follows, and the follower reuses the +// leader's published result. The in-flight entry is cleared on publish. +TEST(SniiSingleFlight, LeadFollowReuse) { + SingleFlight sf; + + auto leader = sf.join_or_lead("k"); + EXPECT_FALSE(leader.has_value()); // first caller leads + EXPECT_EQ(sf.inflight_size(), 1U); + + auto follower = sf.join_or_lead("k"); + ASSERT_TRUE(follower.has_value()); // second caller follows the in-flight leader + EXPECT_EQ(sf.inflight_size(), 1U); // still one in-flight key + + sf.publish("k", 7); + EXPECT_EQ(follower->get(), 7); // follower reuses the leader's result + EXPECT_EQ(sf.inflight_size(), 0U); +} + +// Different keys execute independently -- each first caller leads. +TEST(SniiSingleFlight, DistinctKeysLeadIndependently) { + SingleFlight sf; + EXPECT_FALSE(sf.join_or_lead("a").has_value()); + EXPECT_FALSE(sf.join_or_lead("b").has_value()); + EXPECT_EQ(sf.inflight_size(), 2U); + sf.publish("a", 1); + sf.publish("b", 2); + EXPECT_EQ(sf.inflight_size(), 0U); +} + +// After a key is published, the next caller of the same key leads a fresh execution. +TEST(SniiSingleFlight, ReLeadAfterPublish) { + SingleFlight sf; + EXPECT_FALSE(sf.join_or_lead("k").has_value()); + sf.publish("k", 1); + EXPECT_EQ(sf.inflight_size(), 0U); + EXPECT_FALSE(sf.join_or_lead("k").has_value()); // leads again + EXPECT_EQ(sf.inflight_size(), 1U); + sf.publish("k", 2); +} + +// The motivating scenario: many threads issue the same key concurrently. Exactly one leads; +// all followers receive the single shared result (one shared_ptr payload, shared read-only). +TEST(SniiSingleFlight, ConcurrentCollapsesToOneLeader) { + constexpr int kThreads = 16; + SingleFlight> sf; + + std::atomic leader_count {0}; + std::latch joined(kThreads); + std::vector> results(kThreads); + auto payload = std::make_shared(42); + + std::vector threads; + threads.reserve(kThreads); + for (int i = 0; i < kThreads; ++i) { + threads.emplace_back([&, i] { + auto follow = sf.join_or_lead("hot-segment"); + const bool is_leader = !follow.has_value(); + joined.count_down(); + if (is_leader) { + leader_count.fetch_add(1, std::memory_order_relaxed); + joined.wait(); // ensure every thread has joined before we publish + sf.publish("hot-segment", payload); + } else { + results[i] = follow->get(); + } + }); + } + for (auto& t : threads) { + t.join(); + } + + EXPECT_EQ(leader_count.load(), 1); // only one open/execute happened + EXPECT_EQ(sf.inflight_size(), 0U); + for (int i = 0; i < kThreads; ++i) { + if (results[i] != nullptr) { + EXPECT_EQ(results[i], payload); // same shared object, not a recomputed copy + EXPECT_EQ(*results[i], 42); + } + } +} + +// A leader that fails publishes its failure; followers observe it (and would then fall back +// to computing independently in the production path). +TEST(SniiSingleFlight, ErrorResultPropagates) { + SingleFlight> sf; + auto leader = sf.join_or_lead("k"); + ASSERT_FALSE(leader.has_value()); + auto follower = sf.join_or_lead("k"); + ASSERT_TRUE(follower.has_value()); + + sf.publish("k", std::make_pair(false, 0)); // leader failed + auto [ok, value] = follower->get(); + EXPECT_FALSE(ok); + EXPECT_EQ(value, 0); +} + +} // namespace doris::snii diff --git a/be/test/storage/index/snii/common/slice_test.cpp b/be/test/storage/index/snii/common/slice_test.cpp new file mode 100644 index 00000000000000..a86f0a9d7b9aa1 --- /dev/null +++ b/be/test/storage/index/snii/common/slice_test.cpp @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/common/slice.h" + +#include + +#include + +#include "common/status.h" + +using doris::snii::Slice; + +TEST(SniiSlice, BasicAccess) { + const uint8_t buf[] = {1, 2, 3, 4}; + Slice s(buf, 4); + EXPECT_EQ(s.size(), 4U); + EXPECT_EQ(s[2], 3U); + Slice sub = s.subslice(1, 2); + EXPECT_EQ(sub.size(), 2U); + EXPECT_EQ(sub[0], 2U); +} + +TEST(SniiSlice, EmptyDefault) { + Slice s; + EXPECT_TRUE(s.empty()); + EXPECT_EQ(s.size(), 0U); +} diff --git a/be/test/storage/index/snii/common/uninitialized_buffer_test.cpp b/be/test/storage/index/snii/common/uninitialized_buffer_test.cpp new file mode 100644 index 00000000000000..bc498c83e64860 --- /dev/null +++ b/be/test/storage/index/snii/common/uninitialized_buffer_test.cpp @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/common/uninitialized_buffer.h" + +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/zstd_codec.h" +#include "storage/index/snii/format/prx_pod.h" + +using doris::snii::ByteSink; +using doris::snii::ByteSource; +using doris::snii::Slice; +using doris::snii::zstd_compress; +using doris::snii::zstd_decompress; +using doris::snii::format::build_prx_window; +using doris::snii::format::read_prx_window_csr; + +namespace { + +using PerDoc = std::vector>; + +// The uninitialized_vector overload must NOT value-initialize the regrown tail: +// after a sentinel fill + clear + regrow into the SAME storage, the object +// representation still holds the sentinel bytes (proves no zero-fill pass). +TEST(SniiUninitializedBuffer, UninitVectorGrowSkipsZeroFill) { + constexpr size_t kN = 64; + doris::snii::uninitialized_vector v; + doris::snii::resize_uninitialized(v, kN); + for (size_t i = 0; i < kN; ++i) { + v[i] = 0xAAAAAAAAU; + } + const uint32_t* old_data = v.data(); + v.clear(); // trivial element destruction is a no-op; storage/capacity retained + doris::snii::resize_uninitialized(v, kN); + ASSERT_EQ(v.data(), old_data) << "regrow within capacity must not reallocate"; + // Examining the object representation through unsigned char* is well-defined. + const auto* bytes = reinterpret_cast(v.data()); + bool all_sentinel = true; + for (size_t i = 0; i < kN * sizeof(uint32_t); ++i) { + if (bytes[i] != 0xAA) { + all_sentinel = false; + break; + } + } + EXPECT_TRUE(all_sentinel) << "uninitialized_vector regrow must default-init (no zero-fill)"; +} + +// Contrast: a plain std::vector DOES value-initialize the regrown tail to zero. +TEST(SniiUninitializedBuffer, StdVectorGrowZeroFillsForContrast) { + constexpr size_t kN = 64; + std::vector v; + doris::snii::resize_uninitialized(v, kN); // std::vector overload == plain resize + for (size_t i = 0; i < kN; ++i) { + v[i] = 0xAAAAAAAAU; + } + v.clear(); + doris::snii::resize_uninitialized(v, kN); + for (size_t i = 0; i < kN; ++i) { + EXPECT_EQ(v[i], 0U) << "std::vector regrow value-initializes to 0"; + } +} + +// Shrinking a warm buffer must reuse storage (no realloc). +TEST(SniiUninitializedBuffer, ResizeUninitializedShrinkKeepsCapacity) { + std::vector v; + doris::snii::resize_uninitialized(v, 1000); + const size_t cap = v.capacity(); + doris::snii::resize_uninitialized(v, 10); + EXPECT_EQ(v.capacity(), cap) << "shrink must not reallocate"; +} + +// Warm-reused CSR decode (large window then small window into the SAME buffers) +// must equal a fresh decode of the small window -- no stale tail leaks past size(). +TEST(SniiUninitializedBuffer, CsrWarmReuseLargeThenSmallNoStaleTail) { + PerDoc large(64); + for (auto& doc : large) { + uint32_t p = 0; + for (int i = 0; i < 256; ++i) { + doc.push_back(p += 1); + } + } + const PerDoc small = {{1, 2, 3}, {10}, {7, 9}}; + + ByteSink large_sink; + ByteSink small_sink; + ASSERT_TRUE(build_prx_window(large, -1, &large_sink).ok()); + ASSERT_TRUE(build_prx_window(small, -1, &small_sink).ok()); + + std::vector pos_flat; + std::vector pos_off; + { + ByteSource src(large_sink.view()); + ASSERT_TRUE(read_prx_window_csr(&src, &pos_flat, &pos_off).ok()); + } + { + ByteSource src(small_sink.view()); + ASSERT_TRUE(read_prx_window_csr(&src, &pos_flat, &pos_off).ok()); + } + std::vector exp_flat; + std::vector exp_off; + { + ByteSource src(small_sink.view()); + ASSERT_TRUE(read_prx_window_csr(&src, &exp_flat, &exp_off).ok()); + } + EXPECT_EQ(pos_flat, exp_flat) << "warm-reused decode must equal a fresh decode"; + EXPECT_EQ(pos_off, exp_off); + EXPECT_EQ(pos_flat.size(), 6U) << "small window total_pos = 3 + 1 + 2"; +} + +// zstd decode into a reused buffer must not reallocate and stays byte-identical. +TEST(SniiUninitializedBuffer, ZstdReusedBufferNoReallocAndByteIdentical) { + std::string payload(50000, '\0'); + for (size_t i = 0; i < payload.size(); ++i) { + payload[i] = static_cast(i * 7 + 3); + } + std::vector comp; + ASSERT_TRUE(zstd_compress(Slice(payload), 3, &comp).ok()); + + std::vector out; + ASSERT_TRUE(zstd_decompress(Slice(comp), payload.size(), &out).ok()); + ASSERT_EQ(out.size(), payload.size()); + EXPECT_EQ(0, std::memcmp(out.data(), payload.data(), payload.size())); + const size_t cap = out.capacity(); + + ASSERT_TRUE(zstd_decompress(Slice(comp), payload.size(), &out).ok()); + EXPECT_EQ(out.capacity(), cap) << "warm-reused decompress must not reallocate"; + EXPECT_EQ(0, std::memcmp(out.data(), payload.data(), payload.size())); +} + +} // namespace diff --git a/be/test/storage/index/snii/encoding/byte_sink_test.cpp b/be/test/storage/index/snii/encoding/byte_sink_test.cpp new file mode 100644 index 00000000000000..41d1c9acbcd9f6 --- /dev/null +++ b/be/test/storage/index/snii/encoding/byte_sink_test.cpp @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/byte_sink.h" + +#include + +#include + +#include "common/status.h" +#include "storage/index/snii/encoding/varint.h" + +using namespace doris::snii; + +TEST(SniiByteSink, Fixed32LittleEndian) { + ByteSink s; + s.put_fixed32(0x04030201U); + ASSERT_EQ(s.size(), 4U); + const auto& b = s.buffer(); + EXPECT_EQ(b[0], 0x01); + EXPECT_EQ(b[1], 0x02); + EXPECT_EQ(b[3], 0x04); +} + +TEST(SniiByteSink, Fixed64LittleEndian) { + ByteSink s; + s.put_fixed64(0x0807060504030201ULL); + ASSERT_EQ(s.size(), 8U); + EXPECT_EQ(s.buffer()[0], 0x01); + EXPECT_EQ(s.buffer()[7], 0x08); +} + +TEST(SniiByteSink, VarintThenBytes) { + ByteSink s; + s.put_varint32(300); + const uint8_t payload[] = {0xAA, 0xBB}; + s.put_bytes(Slice(payload, 2)); + EXPECT_EQ(s.size(), varint_len(300) + 2); +} diff --git a/be/test/storage/index/snii/encoding/byte_source_test.cpp b/be/test/storage/index/snii/encoding/byte_source_test.cpp new file mode 100644 index 00000000000000..660771b7fa5c76 --- /dev/null +++ b/be/test/storage/index/snii/encoding/byte_source_test.cpp @@ -0,0 +1,241 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/byte_source.h" + +#include + +#include + +#include "common/status.h" +#include "storage/index/snii/encoding/byte_sink.h" + +using namespace doris::snii; + +TEST(SniiByteSource, RoundTripWithSink) { + ByteSink s; + s.put_fixed32(0xDEADBEEF); + s.put_varint64(123456789); + s.put_zigzag(-42); + ByteSource src(s.view()); + uint32_t a; + uint64_t b; + int64_t c; + ASSERT_TRUE(src.get_fixed32(&a).ok()); + EXPECT_EQ(a, 0xDEADBEEFU); + ASSERT_TRUE(src.get_varint64(&b).ok()); + EXPECT_EQ(b, 123456789U); + ASSERT_TRUE(src.get_zigzag(&c).ok()); + EXPECT_EQ(c, -42); + EXPECT_TRUE(src.eof()); +} + +TEST(SniiByteSource, GetBytesAdvances) { + ByteSink s; + const uint8_t p[] = {1, 2, 3, 4, 5}; + s.put_bytes(Slice(p, 5)); + ByteSource src(s.view()); + Slice got; + ASSERT_TRUE(src.get_bytes(3, &got).ok()); + ASSERT_EQ(got.size(), 3U); + EXPECT_EQ(got[0], 1U); + EXPECT_EQ(src.remaining(), 2U); +} + +TEST(SniiByteSource, OverrunFails) { + uint8_t one[1] = {0x01}; + ByteSource src(Slice(one, 1)); + uint32_t a; + EXPECT_FALSE(src.get_fixed32(&a).ok()); +} + +// skip_varints must advance the cursor EXACTLY as decoding the same varints +// would -- across 1..5-byte encodings and back-to-back values -- so a CSR +// selective decode can skip non-candidate docs' position deltas losslessly. +TEST(SniiByteSource, SkipVarintsAdvancesLikeDecode) { + ByteSink s; + const uint64_t vals[] = {0, 1, 127, 128, 300, 16384, 0xFFFFFFFFULL, 42, 1u << 21, 5}; + for (uint64_t v : vals) s.put_varint64(v); + s.put_u8(0xAB); // sentinel after the varints + + // Decode-advance reference: consume all values, record end position. + ByteSource ref(s.view()); + for (size_t i = 0; i < std::size(vals); ++i) { + uint64_t v = 0; + ASSERT_TRUE(ref.get_varint64(&v).ok()); + EXPECT_EQ(v, vals[i]); + } + const size_t decoded_pos = ref.position(); + + // Skip-advance: skipping the same count must land at the identical position. + ByteSource skp(s.view()); + ASSERT_TRUE(skp.skip_varints(std::size(vals)).ok()); + EXPECT_EQ(skp.position(), decoded_pos); + uint8_t sentinel = 0; + ASSERT_TRUE(skp.get_u8(&sentinel).ok()); + EXPECT_EQ(sentinel, 0xAB); + + // Partial skip then decode the rest interleaves correctly. + ByteSource mix(s.view()); + ASSERT_TRUE(mix.skip_varints(3).ok()); // skip 0,1,127 + uint64_t v = 0; + ASSERT_TRUE(mix.get_varint64(&v).ok()); + EXPECT_EQ(v, 128U); // 4th value decoded intact +} + +// A long run of single-byte varints interleaved with multi-byte values. +// skip_varints must land exactly where a full decode would, for every prefix +// count -- covers long runs and mixed widths in one sweep. +TEST(SniiByteSource, SkipVarintsLongRunMatchesDecode) { + ByteSink s; + std::vector vals; + // 40 one-byte values, then a 5-byte value (forces a straddle), then more. + for (int i = 0; i < 40; ++i) { + vals.push_back(static_cast(i % 100)); + } + vals.push_back(0xFFFFFFFFULL); // 5 bytes + for (int i = 0; i < 40; ++i) { + vals.push_back(static_cast((i * 7) % 120)); + } + vals.push_back(300); // 2 bytes + for (int i = 0; i < 20; ++i) { + vals.push_back(1); + } + for (uint64_t v : vals) { + s.put_varint64(v); + } + s.put_u8(0x3C); // sentinel + + // Reference decode end-position after consuming the first k values. + for (size_t k = 0; k <= vals.size(); ++k) { + ByteSource ref(s.view()); + for (size_t i = 0; i < k; ++i) { + uint64_t v = 0; + ASSERT_TRUE(ref.get_varint64(&v).ok()); + } + const size_t want = ref.position(); + ByteSource skp(s.view()); + ASSERT_TRUE(skp.skip_varints(k).ok()) << "k=" << k; + EXPECT_EQ(skp.position(), want) << "k=" << k; + } + // Full skip then read the sentinel. + ByteSource full(s.view()); + ASSERT_TRUE(full.skip_varints(vals.size()).ok()); + uint8_t sentinel = 0; + ASSERT_TRUE(full.get_u8(&sentinel).ok()); + EXPECT_EQ(sentinel, 0x3C); +} + +TEST(SniiByteSource, SkipVarintsTruncationFails) { + // A dangling continuation byte (high bit set, no terminator) must error, not + // run off the buffer. + uint8_t buf[2] = {0x80, 0x80}; + ByteSource src(Slice(buf, 2)); + EXPECT_FALSE(src.skip_varints(1).ok()); + // Zero-count skip is a no-op and always succeeds. + ByteSource empty(Slice(buf, 0)); + EXPECT_TRUE(empty.skip_varints(0).ok()); +} + +// decode_delta_run must produce the SAME running prefix-sum values as the +// per-value get_varint32 loop it replaces in the CSR reader -- across 1..5-byte +// delta encodings -- and advance the cursor identically. +TEST(SniiByteSource, DecodeDeltaRunMatchesManualPrefixSum) { + ByteSink s; + // Deltas chosen to span every varint width and to sum to a >32-bit-looking + // running value only if mis-summed; the running total stays within uint32. + const uint32_t deltas[] = {5, 0, 1, 127, 128, 300, 16384, 1U << 21, 7, 2}; + for (uint32_t d : deltas) { + s.put_varint64(d); + } + s.put_u8(0xCD); // sentinel + + // Manual reference: decode each delta, prefix-sum from 0. + std::vector expect; + uint32_t running = 0; + for (uint32_t d : deltas) { + running += d; + expect.push_back(running); + } + + std::vector got; + ByteSource src(s.view()); + ASSERT_TRUE(src.decode_delta_run(std::size(deltas), &got).ok()); + EXPECT_EQ(got, expect); + // Cursor lands exactly on the sentinel. + uint8_t sentinel = 0; + ASSERT_TRUE(src.get_u8(&sentinel).ok()); + EXPECT_EQ(sentinel, 0xCD); +} + +TEST(SniiByteSource, DecodeDeltaRunAppendsAndZeroCount) { + ByteSink s; + s.put_varint64(10); + s.put_varint64(20); + std::vector out {999}; // pre-existing content must be preserved + ByteSource src(s.view()); + // Zero-count run is a no-op that leaves the cursor and output untouched. + ASSERT_TRUE(src.decode_delta_run(0, &out).ok()); + EXPECT_EQ(out.size(), 1U); + EXPECT_EQ(src.position(), 0U); + // A real run APPENDS running values after the existing entry. + ASSERT_TRUE(src.decode_delta_run(2, &out).ok()); + ASSERT_EQ(out.size(), 3U); + EXPECT_EQ(out[0], 999U); + EXPECT_EQ(out[1], 10U); + EXPECT_EQ(out[2], 30U); +} + +// get_varint32_fast must decode identically to get_varint32 across the 1-byte +// fast path, the multi-byte fallback, and the boundary at 127/128. +TEST(SniiByteSource, GetVarint32FastMatchesSlow) { + ByteSink s; + const uint32_t vals[] = {0, 1, 127, 128, 129, 255, 300, 16384, 0xFFFFFFFFU, 42}; + for (uint32_t v : vals) { + s.put_varint64(v); + } + s.put_u8(0x5A); // sentinel + + ByteSource fast(s.view()); + for (uint32_t expect : vals) { + uint32_t v = 0; + ASSERT_TRUE(fast.get_varint32_fast(&v).ok()); + EXPECT_EQ(v, expect); + } + uint8_t sentinel = 0; + ASSERT_TRUE(fast.get_u8(&sentinel).ok()); + EXPECT_EQ(sentinel, 0x5A); + + // Empty buffer -> fast path bails to the checked slow decoder, which errors. + ByteSource empty(Slice(nullptr, 0)); + uint32_t v = 0; + EXPECT_FALSE(empty.get_varint32_fast(&v).ok()); +} + +TEST(SniiByteSource, DecodeDeltaRunTruncationFails) { + // Dangling continuation byte -> corruption, no buffer overrun. + uint8_t buf[2] = {0x80, 0x80}; + ByteSource src(Slice(buf, 2)); + std::vector out; + EXPECT_FALSE(src.decode_delta_run(1, &out).ok()); + // Asking for more values than present also fails cleanly. + ByteSink s; + s.put_varint64(3); + ByteSource src2(s.view()); + std::vector out2; + EXPECT_FALSE(src2.decode_delta_run(2, &out2).ok()); +} diff --git a/be/test/storage/index/snii/encoding/crc32c_test.cpp b/be/test/storage/index/snii/encoding/crc32c_test.cpp new file mode 100644 index 00000000000000..2913d9b1516ce4 --- /dev/null +++ b/be/test/storage/index/snii/encoding/crc32c_test.cpp @@ -0,0 +1,336 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/crc32c.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii_query_test_util.h" + +// Integrated crc32c.h pulls in the thirdparty `namespace crc32c`, so a blanket +// `using namespace doris::snii;` makes a bare crc32c() call ambiguous; pull in only the +// Slice type and qualify the doris::snii::crc32c free functions explicitly. +using doris::snii::Slice; + +// leveldb/RocksDB standard CRC32C(Castagnoli) test vectors. +TEST(SniiCrc32c, KnownVectors) { + std::vector zeros(32, 0x00); + EXPECT_EQ(doris::snii::crc32c(Slice(zeros)), 0x8a9136aaU); + std::vector ff(32, 0xff); + EXPECT_EQ(doris::snii::crc32c(Slice(ff)), 0x62a8ab43U); + std::vector ramp(32); + for (int i = 0; i < 32; ++i) { + ramp[i] = static_cast(i); + } + EXPECT_EQ(doris::snii::crc32c(Slice(ramp)), 0x46dd794eU); +} + +TEST(SniiCrc32c, ExtendEqualsContiguous) { + std::vector v {1, 2, 3, 4, 5, 6, 7, 8}; + uint32_t whole = doris::snii::crc32c(Slice(v)); + uint32_t part = doris::snii::crc32c(Slice(v.data(), 4)); + part = doris::snii::crc32c_extend(part, Slice(v.data() + 4, 4)); + EXPECT_EQ(whole, part); +} + +namespace { + +// Independent byte-at-a-time CRC32C reference (bit-reflected Castagnoli). Used to +// pin the optimized slice-by-8 / hardware path to the canonical scalar result. +uint32_t crc32c_ref(const std::vector& data) { + static constexpr uint32_t kPoly = 0x82F63B78U; + uint32_t crc = ~0U; + for (uint8_t b : data) { + crc ^= b; + for (int k = 0; k < 8; ++k) { + crc = (crc & 1) ? (kPoly ^ (crc >> 1)) : (crc >> 1); + } + } + return ~crc; +} + +} // namespace + +// Sweep every length 0..2048: stresses the 8-byte main loop plus the 0..7 residue +// tail (and the hardware u32/u8 tails) against the scalar reference. Catches any +// slice-by-8 table or unaligned-load bug that the fixed-size vectors above miss. +TEST(SniiCrc32c, MatchesScalarReferenceAllLengths) { + std::vector data; + data.reserve(2048); + uint32_t x = 0x12345678U; + for (size_t len = 0; len <= 2048; ++len) { + EXPECT_EQ(doris::snii::crc32c(Slice(data)), crc32c_ref(data)) << "len=" << len; + // Pseudo-random next byte (xorshift) so the stream is non-trivial. + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + data.push_back(static_cast(x)); + } +} + +// Splitting a buffer at an arbitrary boundary and extending must equal the +// one-shot crc -- the property the windowed/region layout relies on. +TEST(SniiCrc32c, ExtendAcrossArbitrarySplits) { + std::vector data(300); + for (size_t i = 0; i < data.size(); ++i) { + data[i] = static_cast(i * 7 + 1); + } + const uint32_t whole = doris::snii::crc32c(Slice(data)); + for (size_t split : {0U, 1U, 7U, 8U, 9U, 16U, 100U, 299U, 300U}) { + uint32_t c = doris::snii::crc32c(Slice(data.data(), split)); + c = doris::snii::crc32c_extend(c, Slice(data.data() + split, data.size() - split)); + EXPECT_EQ(c, whole) << "split=" << split; + } +} + +// ============================================================================= +// T21 -- CRC32C 3-way interleaved hardware reference paths. +// +// Production crc32c()/crc32c_extend() delegate to the bundled Google crc32c +// thirdparty (already hardware-accelerated + interleaved). The doris::snii::detail +// seam below exposes three reference sub-paths -- portable slice-by-8, serial +// SSE4.2 hardware, and the 3-way interleaved SSE4.2 hardware algorithm with a +// GF(2) shift-combine -- so these tests can prove, byte-for-byte, that the +// production path equals the canonical CRC32C across every size/alignment and that +// the hardware path is engaged. detail::* is compiled only under BE_TEST. +// ============================================================================= + +namespace { + +// Alias only the detail seam; the free crc32c()/crc32c_extend() are qualified at +// every call site below because a bare `crc32c` is ambiguous with the thirdparty +// `crc32c` namespace pulled in by (see the file-header note). +namespace detail = doris::snii::detail; + +// Deterministic pseudo-random buffer with `pad` leading bytes, so a Slice can start +// at an arbitrary alignment offset into the heap allocation. The exact bytes are +// irrelevant to the equivalence asserts (all paths hash the same buffer); the +// seeded engine only keeps runs reproducible. +std::vector make_padded_bytes(uint64_t seed, size_t pad, size_t n) { + std::vector v(pad + n); + std::mt19937_64 rng(seed); + for (auto& b : v) { + b = static_cast(rng() & 0xFFU); + } + return v; +} + +// Sizes spanning the 8-byte main loop, every <8 tail, the 4-byte step, and buffers +// straddling the 1024-byte interleave threshold (below/at/above and exact 3x). +constexpr size_t kSweepSizes[] = {0, 1, 7, 8, 9, 15, 16, 255, + 256, 1023, 1024, 1025, 3072, 4096, 65536}; +constexpr size_t kAlignments[] = {0, 1, 2, 3, 4, 7}; + +} // namespace + +// FV-1: hw3 == slice8 == hw_serial == production crc32c across all sizes/alignments. +TEST(SniiCrc32cTest, Hw3MatchesSlice8AcrossSizesAndAlignments) { + for (size_t n : kSweepSizes) { + for (size_t align : kAlignments) { + const std::vector buf = + make_padded_bytes(0x9E3779B97F4A7C15ULL + n * 131 + align, align, n); + const Slice s(buf.data() + align, n); + const uint32_t lib = doris::snii::crc32c(s); + const uint32_t sw = detail::crc32c_slice8_extend(0, s); + const uint32_t hws = detail::crc32c_hw_serial_extend(0, s); + const uint32_t hw3 = detail::crc32c_hw3_extend(0, s); + EXPECT_EQ(hw3, sw) << "n=" << n << " align=" << align; + EXPECT_EQ(hw3, hws) << "n=" << n << " align=" << align; + EXPECT_EQ(hw3, lib) << "n=" << n << " align=" << align; + } + } +} + +// FV-2: degenerate + exact-divisible boundaries. Empty -> canonical crc32c("")==0 +// on every path; n=3072 is exactly 3*1024 (zero interleave tail). +TEST(SniiCrc32cTest, DegenerateAndExactDivisibleBoundaries) { + const Slice empty(nullptr, 0); + EXPECT_EQ(0U, doris::snii::crc32c(empty)); + EXPECT_EQ(0U, detail::crc32c_slice8_extend(0, empty)); + EXPECT_EQ(0U, detail::crc32c_hw_serial_extend(0, empty)); + EXPECT_EQ(0U, detail::crc32c_hw3_extend(0, empty)); + + for (size_t n : {size_t {1}, size_t {7}, size_t {8}, size_t {3072}}) { + const std::vector buf = make_padded_bytes(0xD1CE5EEDULL + n, 0, n); + const Slice s(buf.data(), n); + const uint32_t lib = doris::snii::crc32c(s); + EXPECT_EQ(detail::crc32c_slice8_extend(0, s), lib) << "n=" << n; + EXPECT_EQ(detail::crc32c_hw_serial_extend(0, s), lib) << "n=" << n; + EXPECT_EQ(detail::crc32c_hw3_extend(0, s), lib) << "n=" << n; + } +} + +// FV-3: seeded-extend chaining equals the one-shot over the whole, including a +// length that crosses the 1024 threshold (4097). Covers the seed chain + the hw3 +// GF(2) combine linearity (the property every framed section relies on). +TEST(SniiCrc32cTest, ExtendEqualsConcatenationAcrossThreshold) { + for (size_t total : {size_t {16}, size_t {1024}, size_t {4097}}) { + const std::vector buf = make_padded_bytes(0xABCDEF01ULL + total, 0, total); + const uint8_t* p = buf.data(); + const uint32_t whole = doris::snii::crc32c(Slice(p, total)); + for (size_t k : + {size_t {0}, size_t {1}, size_t {7}, size_t {8}, total / 2, total - 1, total}) { + const Slice a(p, k); + const Slice b(p + k, total - k); + // Public seeded-extend chaining (delegates to the bundled library). + EXPECT_EQ(doris::snii::crc32c_extend(doris::snii::crc32c(a), b), whole) + << "total=" << total << " k=" << k; + // The hw3 reference threads the same seed through lane A and the combine. + EXPECT_EQ(detail::crc32c_hw3_extend(doris::snii::crc32c(a), b), whole) + << "total=" << total << " k=" << k; + } + } +} + +// FV-4: canonical CRC32C vectors from an external authority. A path that altered +// the value -- and hence the on-disk format -- would break these. +TEST(SniiCrc32cTest, KnownVectorsMatchExternalAuthority) { + const char* digits = "123456789"; + const Slice ds(reinterpret_cast(digits), 9); + EXPECT_EQ(0xE3069283U, doris::snii::crc32c(ds)); + EXPECT_EQ(0xE3069283U, detail::crc32c_slice8_extend(0, ds)); + EXPECT_EQ(0xE3069283U, detail::crc32c_hw_serial_extend(0, ds)); + EXPECT_EQ(0xE3069283U, detail::crc32c_hw3_extend(0, ds)); + + EXPECT_EQ(0U, doris::snii::crc32c(Slice(nullptr, 0))); + + // 4096 x 'a' crosses the interleave threshold; cross-check every path against + // the independent byte-at-a-time reference defined above. + const std::vector aaa(4096, static_cast('a')); + const Slice as(aaa.data(), aaa.size()); + const uint32_t ref = crc32c_ref(aaa); + EXPECT_EQ(ref, doris::snii::crc32c(as)); + EXPECT_EQ(ref, detail::crc32c_slice8_extend(0, as)); + EXPECT_EQ(ref, detail::crc32c_hw_serial_extend(0, as)); + EXPECT_EQ(ref, detail::crc32c_hw3_extend(0, as)); +} + +// FV-5: single-bit-flip regression -- the checksum still distinguishes a one-bit +// change (verify capability not degraded) on both the production and hw3 paths. +TEST(SniiCrc32cTest, SingleBitFlipChangesChecksum) { + std::vector buf = make_padded_bytes(0x5A5A5A5AULL, 0, 2048); // crosses threshold + const Slice s(buf.data(), buf.size()); + const uint32_t base_lib = doris::snii::crc32c(s); + const uint32_t base_hw3 = detail::crc32c_hw3_extend(0, s); + for (size_t bit : {size_t {0}, size_t {1}, size_t {8 * 7 + 3}, size_t {8 * 1000 + 5}, + size_t {8 * 2047 + 7}}) { + const size_t byte = bit / 8; + const auto mask = static_cast(1U << (bit % 8)); + buf[byte] ^= mask; // flip + const Slice fs(buf.data(), buf.size()); + EXPECT_NE(base_lib, doris::snii::crc32c(fs)) << "bit=" << bit; + EXPECT_NE(base_hw3, detail::crc32c_hw3_extend(0, fs)) << "bit=" << bit; + buf[byte] ^= mask; // restore + } +} + +// FV-6 ([perf-deterministic]): the interleave optimization is engaged and correct. +// threshold > 0, and when hardware CRC is present the dispatcher-selected hardware +// paths equal the authoritative library value. +TEST(SniiCrc32cPerfTest, OptimizationEngagedThresholdAndHardwarePath) { + EXPECT_GT(detail::crc32c_interleave_threshold(), 0U); + + const std::vector buf = make_padded_bytes(0xC0FFEEULL, 0, 4096); // >= threshold + const Slice s(buf.data(), buf.size()); + const uint32_t lib = doris::snii::crc32c(s); + // Portable path is always available and correct. + EXPECT_EQ(lib, detail::crc32c_slice8_extend(0, s)); + if (detail::crc32c_has_hw()) { + EXPECT_EQ(lib, detail::crc32c_hw_serial_extend(0, s)); + EXPECT_EQ(lib, detail::crc32c_hw3_extend(0, s)); + } +} + +// [perf-report-only]: wall-clock throughput, never a CI gate. The only assertions +// are deterministic byte-equivalence; the GB/s figures are printed for reference. +TEST(SniiCrc32cPerfTest, Hw3ThroughputReportOnly) { + constexpr size_t kBytes = 1U << 16; // 64 KiB + const std::vector buf = make_padded_bytes(0x1234ULL, 0, kBytes); + const Slice s(buf.data(), buf.size()); + const uint32_t expected = detail::crc32c_slice8_extend(0, s); + ASSERT_EQ(expected, detail::crc32c_hw_serial_extend(0, s)); + ASSERT_EQ(expected, detail::crc32c_hw3_extend(0, s)); + + constexpr int kIters = 2000; + auto time_path = [&](auto fn) { + volatile uint32_t sink = 0; + const auto t0 = std::chrono::steady_clock::now(); + for (int i = 0; i < kIters; ++i) { + sink ^= fn(0, s); + } + const auto t1 = std::chrono::steady_clock::now(); + (void)sink; + return std::chrono::duration(t1 - t0).count(); + }; + const double hw3_s = time_path(detail::crc32c_hw3_extend); + const double hws_s = time_path(detail::crc32c_hw_serial_extend); + const double gigabytes = static_cast(kBytes) * kIters / 1e9; + std::cout << "[ REPORT ] crc32c 64KiB hw3=" << (hw3_s > 0 ? gigabytes / hw3_s : 0.0) + << " GB/s hw_serial=" << (hws_s > 0 ? gigabytes / hws_s : 0.0) + << " GB/s (report-only, not a gate)\n"; + SUCCEED(); +} + +// FV-7: reader black-box regression. build_reader stamps every dict-block / +// prx-window / frq-region crc via the production crc32c(); open_index and the +// queries re-verify them, so assert_ok fails on any Status::Corruption from the +// dict_block / prx_pod / frq_pod verify paths. Results must match the known-good +// sets (identical to SniiPhraseQueryTest), confirming the seam addition perturbs +// nothing and stored CRCs still verify byte-identically. +TEST(SniiCrc32cTest, ReaderPhraseAndTermQueriesVerifyCrc) { + using doris::snii::reader::LogicalIndexReader; + using doris::snii::reader::SniiSegmentReader; + using doris::snii::snii_test::assert_ok; + using doris::snii::snii_test::build_reader; + using doris::snii::snii_test::MemoryFile; + + MemoryFile file; + SniiSegmentReader segment_reader; + LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + // Term lookup drives the dict-block crc verify path. + bool found = false; + doris::snii::format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + assert_ok(index_reader.lookup("failed", &found, &entry, &frq_base, &prx_base)); + EXPECT_TRUE(found); + + // Phrase query drives the framed prx-window crc verify path. + std::vector phrase_docids; + assert_ok(doris::snii::query::phrase_query(index_reader, {"failed", "order"}, &phrase_docids)); + EXPECT_EQ(phrase_docids, (std::vector {5000, 7000, 8000})); + + // Phrase-prefix query drives term expansion + prx crc paths. + std::vector prefix_docids; + assert_ok(doris::snii::query::phrase_prefix_query(index_reader, {"failed", "ord"}, + &prefix_docids, 10)); + EXPECT_EQ(prefix_docids, (std::vector {5000, 6000, 7000, 8000})); +} diff --git a/be/test/storage/index/snii/encoding/pfor_test.cpp b/be/test/storage/index/snii/encoding/pfor_test.cpp new file mode 100644 index 00000000000000..b5685813fa2a63 --- /dev/null +++ b/be/test/storage/index/snii/encoding/pfor_test.cpp @@ -0,0 +1,488 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/pfor.h" + +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/byte_source.h" + +using namespace doris::snii; + +static void roundtrip(const std::vector& v) { + ByteSink sink; + pfor_encode(v.data(), v.size(), &sink); + ByteSource src(sink.view()); + std::vector out(v.size()); + ASSERT_TRUE(pfor_decode(&src, v.size(), out.data()).ok()); + EXPECT_EQ(out, v); +} + +TEST(SniiPfor, Uniform) { + roundtrip(std::vector(200, 5)); +} + +TEST(SniiPfor, Ramp) { + std::vector v; + for (uint32_t i = 0; i < 256; ++i) { + v.push_back(i); + } + roundtrip(v); +} + +TEST(SniiPfor, WithOutliers) { + std::vector v(128, 3); + v[10] = 1000000; + v[77] = 999; + roundtrip(v); +} + +TEST(SniiPfor, AllZero) { + roundtrip(std::vector(64, 0)); +} + +TEST(SniiPfor, MaxValues) { + std::vector v(32, 0xFFFFFFFFU); + v[0] = 0; + roundtrip(v); +} + +// Exhaustive: every bit width 0..32, capped random values, across a spread of n +// (including non-power-of-two and prime lengths) so the bit-unpacker's tail handling +// (partial trailing 64-bit word / partial byte) is exercised for each width. A +// deterministic LCG keeps it reproducible without Math.random. +TEST(SniiPfor, AllWidthsRandomLengths) { + uint64_t rng = 0x9E3779B97F4A7C15ULL; + auto next = [&rng]() { + rng = rng * 6364136223846793005ULL + 1442695040888963407ULL; + return static_cast(rng >> 32); + }; + const size_t lens[] = {1, 2, 3, 7, 8, 9, 31, 32, 33, 63, 64, 65, 127, 200, 257}; + for (int w = 0; w <= 32; ++w) { + const uint32_t cap = (w >= 32) ? 0xFFFFFFFFU : ((1U << w) - 1U); + for (size_t n : lens) { + std::vector v(n); + for (size_t i = 0; i < n; ++i) { + v[i] = (w == 0) ? 0U : (next() & cap); + } + roundtrip(v); // byte-identical decode required for every (w, n) + } + } +} + +// A few exceptions sprinkled into otherwise-narrow data, at boundary indices, to +// cover the exception-patch path alongside the fast unpack. +TEST(SniiPfor, ExceptionsAtBoundaries) { + std::vector v(130, 2); + v[0] = 0x00ABCDEF; // first + v[63] = 0x12345678; // around the 64-value word boundary + v[64] = 0x0BADF00D; + v[129] = 0xFFFFFFFF; // last + roundtrip(v); +} + +// --------------------------------------------------------------------------- +// T11: histogram + suffix-sum choose_width and single-pass pfor_encode. +// +// The oracle helpers below are a faithful transcription of the PRE-T11 encoder +// (O(maxw*n) bits_for width selection + std::vector low/exc split). The optimized +// pfor_encode must (a) select the same bit_width and (b) emit byte-identical +// output for arbitrary inputs -- the strongest guard that the on-disk format is +// unchanged. The deterministic op-count seam proves each value's bit-width is +// evaluated exactly once per run (vs the former ~(maxw+2) times). +// --------------------------------------------------------------------------- +namespace { + +uint8_t ref_bits_for(uint32_t v) { + uint8_t b = 0; + while (v) { + ++b; + v >>= 1; + } + return b; +} + +// Original O(maxw*n) width selection -- identical cost formula and ascending-w +// strict-'<' tie-break. +uint8_t ref_choose_width(const std::vector& v) { + const size_t n = v.size(); + uint8_t maxw = 0; + for (size_t i = 0; i < n; ++i) { + // NOLINTNEXTLINE(readability-use-std-min-max): naive reference mirror of the original + if (ref_bits_for(v[i]) > maxw) { + maxw = ref_bits_for(v[i]); + } + } + uint8_t best = maxw; + size_t best_cost = SIZE_MAX; + for (uint8_t w = 0; w <= maxw; ++w) { + size_t exc = 0; + for (size_t i = 0; i < n; ++i) { + if (ref_bits_for(v[i]) > w) { + ++exc; + } + } + const size_t cost = (static_cast(w) * n + 7) / 8 + exc * 6; + if (cost < best_cost) { + best_cost = cost; + best = w; + } + } + return best; +} + +uint32_t ref_low_mask(uint8_t w) { + return (w >= 32) ? 0xFFFFFFFFU : ((1U << w) - 1U); +} + +// Faithful transcription of the pre-T11 pfor_encode byte layout. +std::vector reference_pfor_encode(const std::vector& values) { + const size_t n = values.size(); + const uint8_t w = ref_choose_width(values); + ByteSink sink; + std::vector> exc; // (index, full value) + std::vector low(values.begin(), values.end()); + for (size_t i = 0; i < n; ++i) { + if (ref_bits_for(values[i]) > w) { + exc.emplace_back(static_cast(i), values[i]); + low[i] = 0; + } + } + sink.put_u8(w); + sink.put_varint32(static_cast(exc.size())); + if (w != 0) { + uint64_t acc = 0; + int filled = 0; + for (size_t i = 0; i < n; ++i) { + acc |= static_cast(low[i] & ref_low_mask(w)) << filled; + filled += w; + while (filled >= 8) { + sink.put_u8(static_cast(acc)); + acc >>= 8; + filled -= 8; + } + } + if (filled > 0) { + sink.put_u8(static_cast(acc)); + } + } + uint32_t prev = 0; + for (const auto& e : exc) { + sink.put_varint32(e.first - prev); + sink.put_varint32(e.second); + prev = e.first; + } + return sink.buffer(); +} + +// Deterministic LCG (same constants as AllWidthsRandomLengths above). +struct Lcg { + uint64_t state; + explicit Lcg(uint64_t seed) : state(seed) {} + uint32_t next() { + state = state * 6364136223846793005ULL + 1442695040888963407ULL; + return static_cast(state >> 32); + } +}; + +std::vector encode_to_bytes(const std::vector& v) { + ByteSink sink; + pfor_encode(v.data(), v.size(), &sink); + return sink.buffer(); +} + +// Representative inputs spanning the layout's branches: all-zero, all-one, +// monotonic delta, freq-like (mostly 1 with bursts), bit31-set large values, and +// narrow data with sparse wide exceptions. +std::vector> representative_inputs() { + std::vector> sets; + sets.emplace_back(256, 0U); // all zero -> w == 0 + sets.emplace_back(256, 1U); // all one -> w == 1 + { + std::vector ramp(256); + for (uint32_t i = 0; i < 256; ++i) { + ramp[i] = i; // monotonic delta + } + sets.push_back(std::move(ramp)); + } + { + std::vector freq(256, 1U); // freq-like: mostly 1, a few bursts + freq[3] = 5; + freq[100] = 9; + freq[255] = 2; + sets.push_back(std::move(freq)); + } + { + std::vector big(64, 7U); // some bit31-set values (width 32) + big[0] = 0x80000000U; + big[31] = 0xFFFFFFFFU; + big[63] = 0x80000001U; + sets.push_back(std::move(big)); + } + { + std::vector mixed(200, 4U); // narrow with sparse wide exceptions + mixed[5] = 70000; + mixed[6] = 70000; + mixed[199] = 1234567; + sets.push_back(std::move(mixed)); + } + return sets; +} + +} // namespace + +// FW-01: random data (mostly small, occasional large to force exceptions) round-trips +// exactly and consumes the whole stream. +TEST(SniiPforTest, RoundTripRandom) { + Lcg rng(0xD1B54A32D192ED03ULL); + for (int trial = 0; trial < 50; ++trial) { + std::vector v(256); + for (uint32_t& vi : v) { + const uint32_t r = rng.next(); + vi = (r % 16 == 0) ? r : (r & 0xFFU); // ~1/16 large, rest in [0,255] + } + ByteSink sink; + pfor_encode(v.data(), v.size(), &sink); + ByteSource src(sink.view()); + std::vector out(v.size()); + ASSERT_TRUE(pfor_decode(&src, v.size(), out.data()).ok()); + EXPECT_EQ(out, v); + EXPECT_TRUE(src.eof()); + } +} + +// FW-02: the chosen bit_width (encoded as byte 0) matches the original linear-scan +// selection for representative + random inputs. +TEST(SniiPforTest, WidthMatchesLinearScan) { + std::vector> sets = representative_inputs(); + Lcg rng(0x0123456789ABCDEFULL); + for (int t = 0; t < 60; ++t) { + const uint32_t cap_bits = rng.next() % 33; // target max width 0..32 + const uint32_t cap = (cap_bits >= 32) ? 0xFFFFFFFFU : ((1U << cap_bits) - 1U); + const size_t n = 1 + (rng.next() % 256); + std::vector v(n); + for (size_t i = 0; i < n; ++i) { + v[i] = (cap == 0) ? 0U : (rng.next() & cap); + } + sets.push_back(std::move(v)); + } + for (const auto& v : sets) { + const std::vector buf = encode_to_bytes(v); + ASSERT_FALSE(buf.empty()); + EXPECT_EQ(static_cast(buf[0]), static_cast(ref_choose_width(v))); + } +} + +// FW-03: exact, hand-verified golden bytes freeze the on-disk encoding for the +// clean cases (w==0, and clean bit-packing at w==1/4/8). Any drift in width +// choice or bit-packing is caught immediately. +TEST(SniiPforTest, GoldenByteIdentical) { + using B = std::vector; + EXPECT_EQ(encode_to_bytes(std::vector(8, 0U)), (B {0x00, 0x00})); + EXPECT_EQ(encode_to_bytes(std::vector(8, 1U)), (B {0x01, 0x00, 0xFF})); + EXPECT_EQ(encode_to_bytes(std::vector(4, 10U)), (B {0x04, 0x00, 0xAA, 0xAA})); + EXPECT_EQ(encode_to_bytes(std::vector(4, 200U)), + (B {0x08, 0x00, 0xC8, 0xC8, 0xC8, 0xC8})); +} + +// FW-03 (extended): the optimized encoder is byte-identical to the pre-T11 +// reference for representative inputs AND a large random sweep over mixed widths, +// lengths (including n==0 and n>256), and exception densities. +TEST(SniiPforTest, EncodeMatchesLegacyReference) { + for (const auto& v : representative_inputs()) { + EXPECT_EQ(encode_to_bytes(v), reference_pfor_encode(v)); + } + Lcg rng(0x0F0F0F0F0F0F0F0FULL); + for (int t = 0; t < 300; ++t) { + const uint32_t cap_bits = rng.next() % 33; + const uint32_t cap = (cap_bits >= 32) ? 0xFFFFFFFFU : ((1U << cap_bits) - 1U); + const size_t n = rng.next() % 300; // spans n==0 and the n>256 heap path + std::vector v(n); + for (size_t i = 0; i < n; ++i) { + // ~1/32 chance of a full-width value above the cap -> an exception. + uint32_t vi = 0U; + if (rng.next() % 32 == 0) { + vi = rng.next(); + } else if (cap != 0) { + vi = rng.next() & cap; + } + v[i] = vi; + } + EXPECT_EQ(encode_to_bytes(v), reference_pfor_encode(v)) + << "n=" << n << " cap_bits=" << cap_bits; + } +} + +// FW-04: all zeros -> width 0 (the clz(0) guard), no packed bytes, no exceptions. +TEST(SniiPforTest, AllZeros) { + std::vector v(256, 0U); + ByteSink sink; + pfor_encode(v.data(), v.size(), &sink); + const std::vector& buf = sink.buffer(); + ASSERT_EQ(buf.size(), 2U); // [w=0][n_exc=0], nothing else + EXPECT_EQ(buf[0], 0U); + EXPECT_EQ(buf[1], 0U); + ByteSource src(sink.view()); + std::vector out(v.size(), 0xDEADBEEFU); + ASSERT_TRUE(pfor_decode(&src, v.size(), out.data()).ok()); + EXPECT_EQ(out, v); + EXPECT_TRUE(src.eof()); +} + +// FW-05: single-element runs (small, zero, max). +TEST(SniiPforTest, SingleElement) { + roundtrip(std::vector {42}); + roundtrip(std::vector {0}); + roundtrip(std::vector {0xFFFFFFFFU}); +} + +// FW-06: empty run -> [w=0][n_exc=0]; decode of 0 values is a no-op and consumes +// the header. nullptr values with n==0 must not be dereferenced. +TEST(SniiPforTest, EmptyRun) { + ByteSink sink; + pfor_encode(nullptr, 0, &sink); + ASSERT_EQ(sink.buffer().size(), 2U); + EXPECT_EQ(sink.buffer()[0], 0U); + EXPECT_EQ(sink.buffer()[1], 0U); + ByteSource src(sink.view()); + std::vector out(1, 0xDEADBEEFU); + ASSERT_TRUE(pfor_decode(&src, 0, out.data()).ok()); + EXPECT_EQ(out[0], 0xDEADBEEFU); // nothing written + EXPECT_TRUE(src.eof()); +} + +// FW-07: values with bit31 set (width 32) round-trip, exercising the width-32 +// path (no `value >> w` UB; the exception split uses the cached width comparison). +TEST(SniiPforTest, TopBitSet) { + std::vector v(64, 5U); + v[0] = 0x80000000U; + v[30] = 0xFFFFFFFFU; + v[63] = 0xC0000000U; + roundtrip(v); + roundtrip(std::vector(40, 0x80000000U)); // chosen width == 32 +} + +// FW-08: bimodal data (mostly narrow, a large minority very wide) keeps the chosen +// width small, sending many values to the exception table. Byte-identical to the +// legacy split and round-trips. +TEST(SniiPforTest, ManyExceptions) { + std::vector v(256, 5U); // width 3 + Lcg rng(0x55AA55AA55AA55AAULL); + for (size_t i = 0; i < v.size(); ++i) { + if (i % 4 == 0) { + v[i] = 0x20000000U | (rng.next() & 0x1FFFFFFFU); // width 30 + } + } + const std::vector buf = encode_to_bytes(v); + EXPECT_EQ(buf, reference_pfor_encode(v)); + size_t exceptions = 0; + for (uint32_t x : v) { + if (ref_bits_for(x) > buf[0]) { + ++exceptions; + } + } + EXPECT_GT(exceptions, 50U); // exception-heavy path actually exercised + roundtrip(v); +} + +// FW-09: runs larger than the 256-element stack buffer exercise the heap fallback; +// byte-identical to the reference and round-trip exact. +TEST(SniiPforTest, LargeRunHeapFallback) { + for (size_t n : {257U, 300U, 512U}) { + Lcg rng(0xC0FFEEULL + n); + std::vector v(n); + for (size_t i = 0; i < n; ++i) { + v[i] = rng.next() & 0x3FFU; + } + v[n / 2] = 0xABCDEF01U; // an exception + EXPECT_EQ(encode_to_bytes(v), reference_pfor_encode(v)) << "n=" << n; + roundtrip(v); + } +} + +// FW-10: a stream whose exception index is out of range must return Corruption, +// not throw. (Decoder path, unchanged by T11, guarded here against regressions.) +TEST(SniiPforTest, CorruptExceptionIndexReturnsCorruption) { + ByteSink sink; + sink.put_u8(0); // w = 0 -> no packed region + sink.put_varint32(1); // n_exc = 1 + sink.put_varint32(10); // index_delta = 10 -> idx = 10 + sink.put_varint32(7); // exception value + ByteSource src(sink.view()); + std::vector out(4, 0U); // n = 4, so idx 10 is out of range + auto st = pfor_decode(&src, 4, out.data()); + EXPECT_FALSE(st.ok()); + EXPECT_TRUE(st.is()) << st.to_string(); +} + +// Perf (deterministic): exactly one bit-width evaluation per value per run -- the +// single histogram pass, with the encoder reusing the cached widths (no 3rd +// bits_for pass, no per-candidate O(maxw*n) rescan). RED on the un-optimized code +// (which evaluated ~(maxw+2)*n times). +TEST(SniiPforPerfTest, WidthEvalsEqualsNPerRun) { + Lcg rng(0x1357924680ULL); + for (size_t n : {0U, 1U, 7U, 8U, 9U, 64U, 255U, 256U, 257U, 300U}) { + std::vector v(n); + for (size_t i = 0; i < n; ++i) { + v[i] = rng.next() & 0xFFFFU; + } + doris::snii::testing::reset_pfor_width_evals(); + ByteSink sink; + pfor_encode(v.data(), v.size(), &sink); + EXPECT_EQ(doris::snii::testing::pfor_width_evals(), static_cast(n)) << "n=" << n; + } +} + +// Perf (deterministic): cumulative evaluations across multiple runs equal the total +// number of values encoded. +TEST(SniiPforPerfTest, WidthEvalsEqualsTotalValues) { + const std::vector runs = {256, 256, 100, 1, 256}; + Lcg rng(0x2468ACE0ULL); + doris::snii::testing::reset_pfor_width_evals(); + size_t total = 0; + for (size_t n : runs) { + std::vector v(n); + for (size_t i = 0; i < n; ++i) { + v[i] = rng.next() & 0xFFFFFFU; + } + ByteSink sink; + pfor_encode(v.data(), v.size(), &sink); + total += n; + } + EXPECT_EQ(doris::snii::testing::pfor_width_evals(), static_cast(total)); +} + +// Perf (deterministic): worst case for the old O(maxw*n) scan is maxw==32. The old +// encoder evaluated each value ~(maxw+2)==34 times; the histogram path evaluates +// each exactly once, so evals stay == n even when maxw is maximal. +TEST(SniiPforPerfTest, ChooseWidthHasNoInnerRescan) { + const size_t n = 256; + std::vector v(n, 1U); + v[0] = 0x80000000U; // forces maxw == 32 + doris::snii::testing::reset_pfor_width_evals(); + ByteSink sink; + pfor_encode(v.data(), v.size(), &sink); + EXPECT_EQ(doris::snii::testing::pfor_width_evals(), static_cast(n)); + EXPECT_LT(doris::snii::testing::pfor_width_evals(), static_cast(2 * n)); +} diff --git a/be/test/storage/index/snii/encoding/section_framer_test.cpp b/be/test/storage/index/snii/encoding/section_framer_test.cpp new file mode 100644 index 00000000000000..302c16d4f01b47 --- /dev/null +++ b/be/test/storage/index/snii/encoding/section_framer_test.cpp @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/section_framer.h" + +#include + +#include "common/status.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/byte_source.h" + +using namespace doris::snii; + +TEST(SniiSectionFramer, RoundTrip) { + ByteSink sink; + const uint8_t p[] = {9, 8, 7}; + SectionFramer::write(sink, 0x42, Slice(p, 3)); + ByteSource src(sink.view()); + FramedSection sec; + ASSERT_TRUE(SectionFramer::read(src, &sec).ok()); + EXPECT_EQ(sec.type, 0x42); + ASSERT_EQ(sec.payload.size(), 3U); + EXPECT_EQ(sec.payload[0], 9U); + EXPECT_TRUE(src.eof()); +} + +TEST(SniiSectionFramer, DetectsCorruption) { + ByteSink sink; + const uint8_t p[] = {1, 2, 3, 4}; + SectionFramer::write(sink, 1, Slice(p, 4)); + auto bytes = sink.buffer(); + bytes[3] ^= 0xFF; // flip one byte in the payload + Slice corrupted(bytes); + ByteSource src(corrupted); + FramedSection sec; + EXPECT_FALSE(SectionFramer::read(src, &sec).ok()); +} + +TEST(SniiSectionFramer, SkipMultiple) { + ByteSink sink; + const uint8_t a[] = {1}; + const uint8_t b[] = {2, 2}; + SectionFramer::write(sink, 10, Slice(a, 1)); + SectionFramer::write(sink, 11, Slice(b, 2)); + ByteSource src(sink.view()); + FramedSection s1, s2; + ASSERT_TRUE(SectionFramer::read(src, &s1).ok()); + ASSERT_TRUE(SectionFramer::read(src, &s2).ok()); + EXPECT_EQ(s1.type, 10); + EXPECT_EQ(s2.type, 11); + EXPECT_TRUE(src.eof()); +} + +TEST(SniiSectionFramer, EmptyPayload) { + ByteSink sink; + SectionFramer::write(sink, 7, Slice()); + ByteSource src(sink.view()); + FramedSection sec; + ASSERT_TRUE(SectionFramer::read(src, &sec).ok()); + EXPECT_EQ(sec.type, 7); + EXPECT_EQ(sec.payload.size(), 0U); +} diff --git a/be/test/storage/index/snii/encoding/varint_test.cpp b/be/test/storage/index/snii/encoding/varint_test.cpp new file mode 100644 index 00000000000000..2ae735c733c230 --- /dev/null +++ b/be/test/storage/index/snii/encoding/varint_test.cpp @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/varint.h" + +#include + +#include + +#include "common/status.h" + +using namespace doris::snii; + +TEST(SniiVarint, RoundTrip32) { + for (uint32_t v : {0U, 1U, 127U, 128U, 300U, 16384U, 0xFFFFFFFFU}) { + uint8_t buf[10]; + size_t n = encode_varint32(v, buf); + EXPECT_EQ(n, varint_len(v)); + uint32_t out; + const uint8_t* next; + ASSERT_TRUE(decode_varint32(buf, buf + n, &out, &next).ok()); + EXPECT_EQ(out, v); + EXPECT_EQ(next, buf + n); + } +} + +TEST(SniiVarint, RoundTrip64) { + for (uint64_t v : {0ULL, 1ULL, 127ULL, 128ULL, 1ULL << 35, 0xFFFFFFFFFFFFFFFFULL}) { + uint8_t buf[10]; + size_t n = encode_varint64(v, buf); + uint64_t out; + const uint8_t* next; + ASSERT_TRUE(decode_varint64(buf, buf + n, &out, &next).ok()); + EXPECT_EQ(out, v); + } +} + +TEST(SniiVarint, TruncatedFails) { + uint8_t buf[1] = {0x80}; // continuation bit set but no subsequent byte + uint32_t out; + const uint8_t* next; + EXPECT_FALSE(decode_varint32(buf, buf + 1, &out, &next).ok()); +} + +TEST(SniiVarint, Overflow32Fails) { + // encode a value > 2^32-1, decoding with decode_varint32 should fail + uint8_t buf[10]; + size_t n = encode_varint64((1ULL << 33), buf); + uint32_t out; + const uint8_t* next; + EXPECT_FALSE(decode_varint32(buf, buf + n, &out, &next).ok()); +} + +TEST(SniiVarint, ZigzagRoundTrip) { + const int64_t cases[] = {0, -1, 1, -1000, 1000, INT64_MIN, INT64_MAX}; + for (int64_t v : cases) { + EXPECT_EQ(zigzag_decode(zigzag_encode(v)), v); + } +} diff --git a/be/test/storage/index/snii/encoding/zstd_codec_test.cpp b/be/test/storage/index/snii/encoding/zstd_codec_test.cpp new file mode 100644 index 00000000000000..86b5a54a86a790 --- /dev/null +++ b/be/test/storage/index/snii/encoding/zstd_codec_test.cpp @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/encoding/zstd_codec.h" + +#include + +#include + +#include "common/status.h" + +using namespace doris::snii; + +TEST(SniiZstd, RoundTrip) { + std::vector in; + for (int i = 0; i < 10000; ++i) { + in.push_back(static_cast(i % 7)); + } + std::vector comp, decomp; + ASSERT_TRUE(zstd_compress(Slice(in), 3, &comp).ok()); + EXPECT_LT(comp.size(), in.size()); + ASSERT_TRUE(zstd_decompress(Slice(comp), in.size(), &decomp).ok()); + EXPECT_EQ(decomp, in); +} + +TEST(SniiZstd, WrongLenFails) { + std::vector in(100, 7), comp, decomp; + ASSERT_TRUE(zstd_compress(Slice(in), 3, &comp).ok()); + EXPECT_FALSE(zstd_decompress(Slice(comp), 99, &decomp).ok()); +} + +TEST(SniiZstd, EmptyInput) { + std::vector in, comp, decomp; + ASSERT_TRUE(zstd_compress(Slice(in), 3, &comp).ok()); + ASSERT_TRUE(zstd_decompress(Slice(comp), 0, &decomp).ok()); + EXPECT_TRUE(decomp.empty()); +} diff --git a/be/test/storage/index/snii/format/bootstrap_header_test.cpp b/be/test/storage/index/snii/format/bootstrap_header_test.cpp new file mode 100644 index 00000000000000..75c510c8b77ac1 --- /dev/null +++ b/be/test/storage/index/snii/format/bootstrap_header_test.cpp @@ -0,0 +1,184 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/bootstrap_header.h" + +#include + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/format_constants.h" + +using namespace doris::snii; +using namespace doris::snii::format; +using doris::Status; + +namespace { + +// Fixed on-disk size of the bootstrap header (all fields + trailing crc32c). +// u32 magic + u16 format_version + u16 min_reader_version + u32 flags +// + u32 header_length + u8 tail_pointer_size + u32 header_checksum +constexpr size_t kExpectedHeaderBytes = 4 + 2 + 2 + 4 + 4 + 1 + 4; + +} // namespace + +TEST(SniiBootstrapHeader, RoundTripDefaults) { + BootstrapHeader in {}; // defaults: magic/version/min_reader_version + in.flags = 0; + in.tail_pointer_size = 40; + + ByteSink sink; + ASSERT_TRUE(encode_bootstrap_header(in, &sink).ok()); + EXPECT_EQ(sink.size(), kExpectedHeaderBytes); + + BootstrapHeader out {}; + ASSERT_TRUE(decode_bootstrap_header(sink.view(), &out).ok()); + EXPECT_EQ(out.magic, in.magic); + EXPECT_EQ(out.format_version, in.format_version); + EXPECT_EQ(out.min_reader_version, in.min_reader_version); + EXPECT_EQ(out.flags, in.flags); + EXPECT_EQ(out.header_length, kExpectedHeaderBytes); + EXPECT_EQ(out.tail_pointer_size, in.tail_pointer_size); +} + +TEST(SniiBootstrapHeader, RoundTripNonDefaultFields) { + BootstrapHeader in {}; + in.flags = 0xDEADBEEFU; + in.tail_pointer_size = 255; + + ByteSink sink; + ASSERT_TRUE(encode_bootstrap_header(in, &sink).ok()); + + BootstrapHeader out {}; + ASSERT_TRUE(decode_bootstrap_header(sink.view(), &out).ok()); + EXPECT_EQ(out.flags, 0xDEADBEEFU); + EXPECT_EQ(out.tail_pointer_size, 255); + EXPECT_EQ(out.magic, kContainerMagic); + EXPECT_EQ(out.format_version, kFormatVersion); +} + +TEST(SniiBootstrapHeader, EncodeFillsHeaderLength) { + BootstrapHeader in {}; + ByteSink sink; + ASSERT_TRUE(encode_bootstrap_header(in, &sink).ok()); + // header_length is written into the buffer; first 4 bytes are magic LE. + const auto& bytes = sink.buffer(); + ASSERT_EQ(bytes.size(), kExpectedHeaderBytes); + uint32_t magic_le = static_cast(bytes[0]) | (static_cast(bytes[1]) << 8) | + (static_cast(bytes[2]) << 16) | + (static_cast(bytes[3]) << 24); + EXPECT_EQ(magic_le, kContainerMagic); +} + +TEST(SniiBootstrapHeader, WrongMagicRejected) { + BootstrapHeader in {}; + ByteSink sink; + ASSERT_TRUE(encode_bootstrap_header(in, &sink).ok()); + + auto bytes = sink.buffer(); + bytes[0] ^= 0xFF; // corrupt the magic; checksum would also fail, but magic + // check fires first + BootstrapHeader out {}; + Status s = decode_bootstrap_header(Slice(bytes), &out); + EXPECT_TRUE(s.is()); +} + +TEST(SniiBootstrapHeader, ChecksumMismatchRejected) { + BootstrapHeader in {}; + in.flags = 7; + ByteSink sink; + ASSERT_TRUE(encode_bootstrap_header(in, &sink).ok()); + + auto bytes = sink.buffer(); + // Flip a flags byte (offset 8 = after magic(4)+fmt(2)+min(2)); magic stays + // valid, so only the checksum can detect this. + bytes[8] ^= 0xFF; + BootstrapHeader out {}; + Status s = decode_bootstrap_header(Slice(bytes), &out); + EXPECT_TRUE(s.is()); +} + +TEST(SniiBootstrapHeader, MinReaderVersionTooNewUnsupported) { + BootstrapHeader in {}; + // A future writer that demands a reader >= kFormatVersion+1: current reader + // must refuse. + in.min_reader_version = static_cast(kFormatVersion + 1); + ByteSink sink; + ASSERT_TRUE(encode_bootstrap_header(in, &sink).ok()); + + BootstrapHeader out {}; + Status s = decode_bootstrap_header(sink.view(), &out); + EXPECT_TRUE(s.is()); +} + +TEST(SniiBootstrapHeader, OlderFormatVersionUnsupported) { + BootstrapHeader in {}; + in.format_version = static_cast(kFormatVersion - 1); + in.min_reader_version = static_cast(kFormatVersion - 1); + ByteSink sink; + ASSERT_TRUE(encode_bootstrap_header(in, &sink).ok()); + + BootstrapHeader out {}; + Status s = decode_bootstrap_header(sink.view(), &out); + EXPECT_TRUE(s.is()); +} + +TEST(SniiBootstrapHeader, FutureFormatVersionUnsupported) { + BootstrapHeader in {}; + in.format_version = static_cast(kFormatVersion + 1); + ByteSink sink; + ASSERT_TRUE(encode_bootstrap_header(in, &sink).ok()); + + BootstrapHeader out {}; + Status s = decode_bootstrap_header(sink.view(), &out); + EXPECT_TRUE(s.is()); +} + +TEST(SniiBootstrapHeader, TruncatedRejected) { + BootstrapHeader in {}; + ByteSink sink; + ASSERT_TRUE(encode_bootstrap_header(in, &sink).ok()); + + auto bytes = sink.buffer(); + bytes.pop_back(); // drop one checksum byte + BootstrapHeader out {}; + Status s = decode_bootstrap_header(Slice(bytes), &out); + EXPECT_TRUE(s.is()); +} + +TEST(SniiBootstrapHeader, EmptyInputRejected) { + BootstrapHeader out {}; + std::vector empty; + Status s = decode_bootstrap_header(Slice(empty), &out); + EXPECT_TRUE(s.is()); +} + +TEST(SniiBootstrapHeader, TrailingBytesAfterChecksumRejected) { + BootstrapHeader in {}; + ByteSink sink; + ASSERT_TRUE(encode_bootstrap_header(in, &sink).ok()); + + auto bytes = sink.buffer(); + bytes.push_back(0x00); // extra byte beyond the fixed header + BootstrapHeader out {}; + Status s = decode_bootstrap_header(Slice(bytes), &out); + EXPECT_TRUE(s.is()); +} diff --git a/be/test/storage/index/snii/format/bsbf_test.cpp b/be/test/storage/index/snii/format/bsbf_test.cpp new file mode 100644 index 00000000000000..8282c10822036d --- /dev/null +++ b/be/test/storage/index/snii/format/bsbf_test.cpp @@ -0,0 +1,211 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/bsbf.h" + +#include +#include + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/io/local_file.h" + +namespace doris::snii::format { +namespace { + +// Independent Parquet-canonical oracle (scalar): XXH64 seed 0 hash (via bsbf_hash), +// fastrange block index, SALT masks. Verifies BsbfBuilder (which may run AVX2) is +// algorithmically correct AND Parquet-canonical. +struct Oracle { + std::vector w; + uint32_t nblocks = 0; + void init(uint32_t ndv, double fpp) { + const uint32_t nb = bsbf_optimal_num_bytes(ndv, fpp); + nblocks = nb / kBsbfBytesPerBlock; + w.assign(nb / 4, 0U); + } + static void masks(uint32_t key, uint32_t m[8]) { + for (int i = 0; i < 8; ++i) { + m[i] = 1U << ((key * kBsbfSalt[i]) >> 27); + } + } + void insert(uint64_t h) { + const auto b = static_cast(((h >> 32) * nblocks) >> 32); + uint32_t m[8]; + masks(static_cast(h), m); + for (int i = 0; i < 8; ++i) { + w[b * 8 + i] |= m[i]; + } + } + bool find(uint64_t h) const { + const auto b = static_cast(((h >> 32) * nblocks) >> 32); + uint32_t m[8]; + masks(static_cast(h), m); + for (int i = 0; i < 8; ++i) { + if ((w[b * 8 + i] & m[i]) != m[i]) { + return false; + } + } + return true; + } +}; + +std::string tmp_path() { + return "/tmp/snii_bsbf_" + std::to_string(::getpid()) + "_" + std::to_string(std::rand()) + + ".bin"; +} + +TEST(SniiBsbf, BuildProbeMatchesParquetOracle) { + const uint32_t n = 100000; + const double fpp = 0.01; + BsbfBuilder b; + ASSERT_TRUE(BsbfBuilder::create(n, fpp, &b).ok()); + Oracle o; + o.init(n, fpp); + ASSERT_EQ(b.num_blocks(), o.nblocks); + for (uint32_t i = 0; i < n; ++i) { + const uint64_t h = bsbf_hash("t" + std::to_string(i)); + b.insert(h); + o.insert(h); + } + for (uint32_t i = 0; i < n; ++i) { + const uint64_t h = bsbf_hash("t" + std::to_string(i)); + EXPECT_TRUE(b.maybe_contains(h)) << i; // no false negatives + EXPECT_EQ(b.maybe_contains(h), o.find(h)); // == Parquet oracle (AVX2==scalar) + } + uint32_t fp = 0; + for (uint32_t i = 0; i < n; ++i) { + const uint64_t h = bsbf_hash("absent_" + std::to_string(i)); + EXPECT_EQ(b.maybe_contains(h), o.find(h)); + if (b.maybe_contains(h)) { + ++fp; + } + } + EXPECT_LT(fp, static_cast(n * 0.05)); // FPR << 5% (target 1%) +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiBsbf, SerializeParseOnDemandProbe) { + const uint32_t n = 50000; + BsbfBuilder b; + ASSERT_TRUE(BsbfBuilder::create(n, 0.01, &b).ok()); + std::vector keys; + for (uint32_t i = 0; i < n; ++i) { + const uint64_t h = bsbf_hash("k" + std::to_string(i)); + keys.push_back(h); + b.insert(h); + } + // Serialize at a non-zero section base (prefix) to exercise the constant offset. + ByteSink sink; + const uint64_t prefix = 777; + for (uint64_t i = 0; i < prefix; ++i) { + sink.put_u8(0xAB); + } + ASSERT_TRUE(b.serialize(&sink).ok()); + EXPECT_EQ(sink.size(), prefix + kBsbfHeaderSize + b.num_bytes()); // 28B header, exact + + const std::string path = tmp_path(); + { + io::LocalFileWriter w; + ASSERT_TRUE(w.open(path).ok()); + ASSERT_TRUE(w.append(sink.view()).ok()); + ASSERT_TRUE(w.finalize().ok()); + } + io::LocalFileReader r; + ASSERT_TRUE(r.open(path).ok()); + std::vector hdr; + ASSERT_TRUE(r.read_at(prefix, kBsbfHeaderSize, &hdr).ok()); + BsbfHeader h; + ASSERT_TRUE(BsbfHeader::parse(Slice(hdr), prefix, &h).ok()); + EXPECT_EQ(h.num_bytes, b.num_bytes()); + EXPECT_EQ(h.bitset_base, prefix + kBsbfHeaderSize); + + for (uint32_t i = 0; i < 2000; ++i) { + bool mp = false; + ASSERT_TRUE(bsbf_probe(&r, h, keys[i], &mp).ok()); + EXPECT_TRUE(mp) << i; // present -> maybe + EXPECT_EQ(mp, b.maybe_contains(keys[i])); // on-demand == in-memory + } + for (uint32_t i = 0; i < 2000; ++i) { + const uint64_t a = bsbf_hash("absent_" + std::to_string(i)); + bool mp = false; + ASSERT_TRUE(bsbf_probe(&r, h, a, &mp).ok()); + EXPECT_EQ(mp, b.maybe_contains(a)); // incl. any false positive, identical + } + std::remove(path.c_str()); +} + +TEST(SniiBsbf, HeaderValidation) { + BsbfBuilder b; + ASSERT_TRUE(BsbfBuilder::create(1000, 0.01, &b).ok()); + ByteSink sink; + ASSERT_TRUE(b.serialize(&sink).ok()); + const std::vector bytes(sink.buffer()); + BsbfHeader h; + EXPECT_TRUE(BsbfHeader::parse(Slice(bytes.data(), kBsbfHeaderSize), 0, &h).ok()); + + auto fails = [](std::vector c) { + BsbfHeader hh; + return !BsbfHeader::parse(Slice(c.data(), kBsbfHeaderSize), 0, &hh).ok(); + }; + { + auto c = bytes; + c[0] = 'X'; + EXPECT_TRUE(fails(c)); + } // magic + { + auto c = bytes; + c[4] = 9; + EXPECT_TRUE(fails(c)); + } // version + { + auto c = bytes; + c[5] = 1; + EXPECT_TRUE(fails(c)); + } // hash strategy + { + auto c = bytes; + c[8] ^= 0xFF; + EXPECT_TRUE(fails(c)); + } // field -> header crc + EXPECT_FALSE(BsbfHeader::parse(Slice(bytes.data(), 10), 0, &h).ok()); // short +} + +TEST(SniiBsbf, FastrangeNotMaskVariant) { + // The fastrange index must differ from Doris's `&(n-1)` for some hash, proving we + // use the Parquet-canonical multiply-shift (not the mask variant). + const uint32_t nblocks = 4096; // power of 2 so both formulas are defined + bool differ = false; + for (uint64_t s = 1; s < 200 && !differ; ++s) { + const uint64_t h = bsbf_hash("probe" + std::to_string(s)); + const uint32_t fastrange = bsbf_block_index(h, nblocks); + const uint32_t mask = static_cast(h >> 32) & (nblocks - 1); + if (fastrange != mask) { + differ = true; + } + } + EXPECT_TRUE(differ); +} + +} // namespace +} // namespace doris::snii::format diff --git a/be/test/storage/index/snii/format/dict_block_directory_test.cpp b/be/test/storage/index/snii/format/dict_block_directory_test.cpp new file mode 100644 index 00000000000000..fa9269888c8bd3 --- /dev/null +++ b/be/test/storage/index/snii/format/dict_block_directory_test.cpp @@ -0,0 +1,324 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/dict_block_directory.h" + +#include + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/section_framer.h" +#include "storage/index/snii/format/format_constants.h" + +using namespace doris::snii; +using namespace doris::snii::format; + +namespace { + +// Serialize a list of block_refs using the builder and return the framed section bytes. +std::vector Build(const std::vector& refs) { + DictBlockDirectoryBuilder builder; + for (const auto& r : refs) { + builder.add(r); + } + ByteSink sink; + builder.finish(&sink); + return sink.buffer(); +} + +// Assert that two BlockRef structs are equal field by field. +void ExpectRefEq(const BlockRef& a, const BlockRef& b) { + EXPECT_EQ(a.offset, b.offset); + EXPECT_EQ(a.length, b.length); + EXPECT_EQ(a.n_entries, b.n_entries); + EXPECT_EQ(a.flags, b.flags); + EXPECT_EQ(a.checksum, b.checksum); + EXPECT_EQ(a.uncomp_len, b.uncomp_len); +} + +} // namespace + +// A zstd-compressed block ref carries uncomp_len; a raw ref does not. Both +// round-trip exactly, and the raw ref's directory bytes stay v1-compact (no +// trailing uncomp_len varint when the kZstd flag is clear). +TEST(SniiDictBlockDirectory, ZstdRefCarriesUncompLen) { + std::vector refs = { + {.offset = 0, + .length = 40000, + .n_entries = 250, + .flags = block_ref_flags::kZstd, + .checksum = 0xABCDEF01U, + .uncomp_len = 65536}, // compressed + {.offset = 40000, + .length = 4096, + .n_entries = 64, + .flags = 0, + .checksum = 0x11223344U, + .uncomp_len = 0}, // raw + }; + auto bytes = Build(refs); + DictBlockDirectoryReader reader; + ASSERT_TRUE(DictBlockDirectoryReader::open(Slice(bytes), &reader).ok()); + ASSERT_EQ(reader.n_blocks(), 2U); + BlockRef z {}, r {}; + ASSERT_TRUE(reader.get(0, &z).ok()); + ASSERT_TRUE(reader.get(1, &r).ok()); + ExpectRefEq(z, refs[0]); + ExpectRefEq(r, refs[1]); + EXPECT_EQ(z.uncomp_len, 65536U); + EXPECT_TRUE((z.flags & block_ref_flags::kZstd) != 0); + EXPECT_EQ(r.uncomp_len, 0U); + EXPECT_FALSE((r.flags & block_ref_flags::kZstd) != 0); +} + +TEST(SniiDictBlockDirectory, RoundTripMultipleRefs) { + std::vector refs = { + {.offset = 0, .length = 4096, .n_entries = 120, .flags = 0x01, .checksum = 0xDEADBEEFU}, + {.offset = 4096, + .length = 8192, + .n_entries = 300, + .flags = 0x05, + .checksum = 0x12345678U}, + {.offset = 12288, + .length = 2048, + .n_entries = 64, + .flags = 0x00, + .checksum = 0xCAFEBABEU}, + }; + auto bytes = Build(refs); + + DictBlockDirectoryReader reader; + ASSERT_TRUE(DictBlockDirectoryReader::open(Slice(bytes), &reader).ok()); + ASSERT_EQ(reader.n_blocks(), 3U); + for (uint32_t i = 0; i < refs.size(); ++i) { + BlockRef out {}; + ASSERT_TRUE(reader.get(i, &out).ok()); + ExpectRefEq(out, refs[i]); + } +} + +TEST(SniiDictBlockDirectory, GetOrdinalCorrectMapping) { + std::vector refs; + for (uint32_t i = 0; i < 50; ++i) { + refs.push_back(BlockRef {.offset = static_cast(i) * 1000, + .length = 1000, + .n_entries = i + 1, + .flags = static_cast(i & 0xFF), + .checksum = i * 7U + 3U}); + } + auto bytes = Build(refs); + + DictBlockDirectoryReader reader; + ASSERT_TRUE(DictBlockDirectoryReader::open(Slice(bytes), &reader).ok()); + ASSERT_EQ(reader.n_blocks(), 50U); + // Sample several ordinals and verify the mapping produces no cross-slot errors. + for (uint32_t ord : {0U, 1U, 17U, 49U}) { + BlockRef out {}; + ASSERT_TRUE(reader.get(ord, &out).ok()); + ExpectRefEq(out, refs[ord]); + } +} + +TEST(SniiDictBlockDirectory, OutOfRangeOrdinalRejected) { + std::vector refs = { + {.offset = 0, .length = 100, .n_entries = 1, .flags = 0, .checksum = 0xAAU}}; + auto bytes = Build(refs); + + DictBlockDirectoryReader reader; + ASSERT_TRUE(DictBlockDirectoryReader::open(Slice(bytes), &reader).ok()); + ASSERT_EQ(reader.n_blocks(), 1U); + BlockRef out {}; + // ordinal == n_blocks is out of range + EXPECT_TRUE(reader.get(1, &out).is()); + // far beyond range + EXPECT_TRUE(reader.get(1000, &out).is()); +} + +TEST(SniiDictBlockDirectory, EmptyDirectory) { + std::vector refs; // 0 blocks + auto bytes = Build(refs); + + DictBlockDirectoryReader reader; + ASSERT_TRUE(DictBlockDirectoryReader::open(Slice(bytes), &reader).ok()); + EXPECT_EQ(reader.n_blocks(), 0U); + BlockRef out {}; + EXPECT_TRUE(reader.get(0, &out).is()); +} + +TEST(SniiDictBlockDirectory, LargeOffsetNear2Pow48) { + const uint64_t kBig = (1ULL << 48) - 1; // near 2^48 + std::vector refs = { + {.offset = kBig, + .length = kBig - 1, + .n_entries = 0xFFFFFFFFU, + .flags = 0xFF, + .checksum = 0xFFFFFFFFU}, + {.offset = kBig + 12345, .length = 1, .n_entries = 1, .flags = 0x02, .checksum = 0U}, + }; + auto bytes = Build(refs); + + DictBlockDirectoryReader reader; + ASSERT_TRUE(DictBlockDirectoryReader::open(Slice(bytes), &reader).ok()); + ASSERT_EQ(reader.n_blocks(), 2U); + BlockRef out0 {}; + ASSERT_TRUE(reader.get(0, &out0).ok()); + ExpectRefEq(out0, refs[0]); + BlockRef out1 {}; + ASSERT_TRUE(reader.get(1, &out1).ok()); + ExpectRefEq(out1, refs[1]); +} + +TEST(SniiDictBlockDirectory, FramedAsDictBlockDirectoryType) { + std::vector refs = { + {.offset = 0, .length = 10, .n_entries = 1, .flags = 0, .checksum = 0}}; + auto bytes = Build(refs); + ASSERT_GE(bytes.size(), 1U); + // The first byte is the SectionFramer type and must be kDictBlockDirectory. + EXPECT_EQ(bytes[0], static_cast(SectionType::kDictBlockDirectory)); +} + +TEST(SniiDictBlockDirectory, DetectsCorruption) { + std::vector refs = { + {.offset = 0, .length = 4096, .n_entries = 120, .flags = 0x01, .checksum = 0xDEADBEEFU}, + {.offset = 4096, + .length = 8192, + .n_entries = 300, + .flags = 0x05, + .checksum = 0x12345678U}, + }; + auto bytes = Build(refs); + ASSERT_GE(bytes.size(), 4U); + // Flip one byte in the payload region (skip the type+len prefix); the section CRC must detect the corruption. + bytes[3] ^= 0xFF; + DictBlockDirectoryReader reader; + EXPECT_TRUE(DictBlockDirectoryReader::open(Slice(bytes), &reader) + .is()); +} + +TEST(SniiDictBlockDirectory, DetectsTruncation) { + std::vector refs = {{.offset = 0, + .length = 4096, + .n_entries = 120, + .flags = 0x01, + .checksum = 0xDEADBEEFU}}; + auto bytes = Build(refs); + bytes.pop_back(); // truncate the last byte + DictBlockDirectoryReader reader; + EXPECT_FALSE(DictBlockDirectoryReader::open(Slice(bytes), &reader).ok()); +} + +TEST(SniiDictBlockDirectory, WrongSectionTypeRejected) { + // Write a section with a type other than kDictBlockDirectory via the framer; open must reject it. + ByteSink sink; + const uint8_t p[] = {0, 1, 2}; + SectionFramer::write(sink, static_cast(SectionType::kXFilter), Slice(p, 3)); + auto bytes = sink.buffer(); + DictBlockDirectoryReader reader; + EXPECT_TRUE(DictBlockDirectoryReader::open(Slice(bytes), &reader) + .is()); +} + +TEST(SniiDictBlockDirectory, TrailingBytesRejected) { + // Extra trailing bytes at the end of the payload should be detected (n_blocks does not match actual data). + ByteSink payload; + payload.put_varint32(1); // n_blocks = 1 + payload.put_varint64(0); // offset + payload.put_varint64(10); // length + payload.put_varint32(1); // n_entries + payload.put_u8(0); // flags + payload.put_fixed32(0); // checksum + payload.put_u8(0xEE); // extra trailing byte + ByteSink sink; + SectionFramer::write(sink, static_cast(SectionType::kDictBlockDirectory), + payload.view()); + DictBlockDirectoryReader reader; + EXPECT_TRUE(DictBlockDirectoryReader::open(Slice(sink.buffer()), &reader) + .is()); +} + +// An attacker-inflated n_blocks count must hit the reserve-bomb guard in +// decode_payload BEFORE the vector reserve, returning Corruption rather than +// attempting a multi-gigabyte allocation. The payload is framed through +// SectionFramer::write so the section crc is VALID over the crafted bytes: +// this proves the inner n_blocks-vs-capacity check fires, not an outer crc flip. +TEST(SniiDictBlockDirectory, InflatedNBlocksHitsReserveGuard) { + ByteSink payload; + payload.put_varint32(0xFFFFFFFFU); // n_blocks = ~4.29 billion (impossible) + // Only a few real bytes follow; capacity is nowhere near 4.29B refs. + payload.put_u8(0x00); + payload.put_u8(0x00); + payload.put_u8(0x00); + payload.put_u8(0x00); + ByteSink sink; + SectionFramer::write(sink, static_cast(SectionType::kDictBlockDirectory), + payload.view()); + DictBlockDirectoryReader reader; + // remaining()/kMinRefBytes == 4/8 == 0, so n_blocks > 0 trips the guard. + EXPECT_TRUE(DictBlockDirectoryReader::open(Slice(sink.buffer()), &reader) + .is()); +} + +// A kZstd ref whose trailing uncomp_len varint is missing (truncated) must be +// rejected as Corruption: decode_ref reads offset/length/n_entries/flags/checksum +// fine, sees the kZstd flag, then overruns when it tries to read uncomp_len. +// n_blocks == 1 with an ~8-byte ref body passes the reserve guard (1 > 8/8 is +// false), so the corruption surfaces from the inner uncomp_len read -- not the +// capacity cap and not the section crc (the frame is crc-valid by construction). +TEST(SniiDictBlockDirectory, ZstdRefTruncatedUncompLenRejected) { + ByteSink payload; + payload.put_varint32(1); // n_blocks = 1 + payload.put_varint64(0); // offset (1 byte) + payload.put_varint64(10); // length (1 byte) + payload.put_varint32(1); // n_entries (1 byte) + payload.put_u8(block_ref_flags::kZstd); // flags: kZstd set + payload.put_fixed32(0xDEADBEEFU); // checksum (4 bytes) + // INTENTIONALLY OMIT the trailing varint64 uncomp_len -> decode_ref overruns. + ByteSink sink; + SectionFramer::write(sink, static_cast(SectionType::kDictBlockDirectory), + payload.view()); + DictBlockDirectoryReader reader; + EXPECT_TRUE(DictBlockDirectoryReader::open(Slice(sink.buffer()), &reader) + .is()); +} + +// A kZstd ref round-trips through the builder/reader preserving uncomp_len +// exactly, including a large 64-bit value, proving the trailing varint is both +// written and read back on the kZstd path. +TEST(SniiDictBlockDirectory, ZstdRefRoundTripPreservesUncompLen) { + const uint64_t kUncomp = (1ULL << 40) + 1234567U; // large, multi-byte varint + std::vector refs = { + {.offset = 1024, + .length = 999, + .n_entries = 7, + .flags = block_ref_flags::kZstd, + .checksum = 0x0BADF00DU, + .uncomp_len = kUncomp}, + }; + auto bytes = Build(refs); + DictBlockDirectoryReader reader; + ASSERT_TRUE(DictBlockDirectoryReader::open(Slice(bytes), &reader).ok()); + ASSERT_EQ(reader.n_blocks(), 1U); + BlockRef out {}; + ASSERT_TRUE(reader.get(0, &out).ok()); + ExpectRefEq(out, refs[0]); + EXPECT_EQ(out.uncomp_len, kUncomp); + EXPECT_TRUE((out.flags & block_ref_flags::kZstd) != 0); +} diff --git a/be/test/storage/index/snii/format/dict_block_test.cpp b/be/test/storage/index/snii/format/dict_block_test.cpp new file mode 100644 index 00000000000000..99c17f07709b1c --- /dev/null +++ b/be/test/storage/index/snii/format/dict_block_test.cpp @@ -0,0 +1,564 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/dict_block.h" + +#include + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" + +using namespace doris::snii; // NOLINT +using namespace doris::snii::format; // NOLINT +using doris::Status; + +namespace { + +// Construct a pod_ref slim dict entry. frq/prx off_delta is already relative to the block base. +DictEntry MakePodRef(std::string term, uint32_t df, uint64_t frq_off, uint64_t prx_off = 0) { + DictEntry e; + e.term = std::move(term); + e.kind = DictEntryKind::kPodRef; + e.enc = DictEntryEnc::kSlim; + e.df = df; + e.ttf_delta = df * 2; // written only when tier>=T2 + e.max_freq = 9; // written only when tier>=T2 + e.frq_off_delta = frq_off; + e.frq_len = 128; + e.prx_off_delta = prx_off; // written only when positions are enabled + e.prx_len = 64; // written only when positions are enabled + return e; +} + +DictEntry MakeInline(std::string term, uint32_t df) { + DictEntry e; + e.term = std::move(term); + e.kind = DictEntryKind::kInline; + e.enc = DictEntryEnc::kSlim; + e.df = df; + e.frq_bytes = {0x10, 0x20, 0x30}; + return e; +} + +void ExpectCommon(const DictEntry& a, const DictEntry& b) { + EXPECT_EQ(a.term, b.term); + EXPECT_EQ(a.kind, b.kind); + EXPECT_EQ(a.enc, b.enc); + EXPECT_EQ(a.df, b.df); +} + +// Serialize a set of entries into one block using the builder; return the byte buffer. +std::vector BuildBlock(const std::vector& entries, IndexTier tier, + bool has_positions, uint64_t frq_base, uint64_t prx_base, + uint32_t anchor_interval) { + DictBlockBuilder builder(tier, has_positions, frq_base, prx_base, anchor_interval); + for (const auto& e : entries) { + builder.add_entry(e); + } + ByteSink sink; + builder.finish(&sink); + return sink.buffer(); +} + +} // namespace + +// Empty block: n_entries=0, open should still succeed, find_term on any target returns not-found. +TEST(SniiDictBlock, EmptyBlock) { + std::vector bytes = + BuildBlock({}, IndexTier::kT1, /*has_positions=*/false, 1000, 0, 16); + DictBlockReader reader; + ASSERT_TRUE(DictBlockReader::open(Slice(bytes), IndexTier::kT1, + /*has_positions=*/false, &reader) + .ok()); + EXPECT_EQ(reader.n_entries(), 0U); + EXPECT_EQ(reader.frq_base(), 1000U); + + bool found = true; + DictEntry out; + ASSERT_TRUE(reader.find_term("anything", &found, &out).ok()); + EXPECT_FALSE(found); +} + +// Single-entry round-trip. +TEST(SniiDictBlock, SingleEntryRoundTrip) { + DictEntry e = MakePodRef("solo", 7, 0); + std::vector bytes = BuildBlock({e}, IndexTier::kT1, false, 4096, 0, 16); + DictBlockReader reader; + ASSERT_TRUE(DictBlockReader::open(Slice(bytes), IndexTier::kT1, false, &reader).ok()); + EXPECT_EQ(reader.n_entries(), 1U); + EXPECT_EQ(reader.frq_base(), 4096U); + + bool found = false; + DictEntry out; + ASSERT_TRUE(reader.find_term("solo", &found, &out).ok()); + ASSERT_TRUE(found); + ExpectCommon(e, out); + EXPECT_EQ(out.frq_off_delta, e.frq_off_delta); + EXPECT_EQ(out.frq_len, e.frq_len); +} + +// Multi-entry round-trip: all terms found, fields preserved. +TEST(SniiDictBlock, MultiEntryRoundTrip) { + std::vector entries = {MakePodRef("alpha", 3, 0), MakePodRef("beta", 5, 100), + MakeInline("gamma", 2), MakePodRef("delta", 9, 300), + MakePodRef("epsilon", 11, 500)}; + // delta < gamma lexicographically, so reorder entries to be sorted. + entries = {MakePodRef("alpha", 3, 0), MakePodRef("beta", 5, 100), MakePodRef("delta", 9, 300), + MakePodRef("epsilon", 11, 500), MakeInline("gamma", 2)}; + std::vector bytes = BuildBlock(entries, IndexTier::kT2, true, 8192, 16384, 16); + DictBlockReader reader; + ASSERT_TRUE(DictBlockReader::open(Slice(bytes), IndexTier::kT2, true, &reader).ok()); + EXPECT_EQ(reader.n_entries(), entries.size()); + + for (const auto& e : entries) { + bool found = false; + DictEntry out; + ASSERT_TRUE(reader.find_term(e.term, &found, &out).ok()) << e.term; + ASSERT_TRUE(found) << e.term; + ExpectCommon(e, out); + } +} + +TEST(SniiDictBlock, DecodeAll) { + std::vector entries = {MakePodRef("alpha", 3, 0), MakePodRef("beta", 5, 100), + MakePodRef("delta", 9, 300), MakePodRef("epsilon", 11, 500), + MakeInline("gamma", 2)}; + std::vector bytes = BuildBlock(entries, IndexTier::kT2, true, 8192, 16384, 16); + DictBlockReader reader; + ASSERT_TRUE(DictBlockReader::open(Slice(bytes), IndexTier::kT2, true, &reader).ok()); + + // Null output is rejected, not dereferenced. + EXPECT_FALSE(reader.decode_all(nullptr).ok()); + + // decode_all returns every entry, in ascending order, matching n_entries. + std::vector all; + ASSERT_TRUE(reader.decode_all(&all).ok()); + ASSERT_EQ(all.size(), entries.size()); + for (size_t i = 0; i < entries.size(); ++i) { + EXPECT_EQ(all[i].term, entries[i].term) << i; + ExpectCommon(entries[i], all[i]); + } +} + +// Front-coding across anchors: with a small anchor_interval, terms spanning anchor boundaries are decoded correctly. +TEST(SniiDictBlock, PrefixCompressionAcrossAnchors) { + std::vector entries; + // 21 sorted terms sharing a long common prefix, anchor_interval=4 -> multiple anchors. + std::vector terms = {"interest", "interested", "interesting", "interestingly", + "interests", "internal", "internally", "international", + "internet", "internets", "interplay", "interpose", + "interpret", "interpreted", "interval", "intervene", + "interview", "interviewed", "intestine", "intimate", + "intricate"}; + uint64_t off = 0; + for (const auto& t : terms) { + entries.push_back(MakePodRef(t, 4, off)); + off += 50; + } + std::vector bytes = + BuildBlock(entries, IndexTier::kT1, false, 0, 0, /*anchor_interval=*/4); + DictBlockReader reader; + ASSERT_TRUE(DictBlockReader::open(Slice(bytes), IndexTier::kT1, false, &reader).ok()); + EXPECT_EQ(reader.n_entries(), terms.size()); + + for (size_t i = 0; i < terms.size(); ++i) { + bool found = false; + DictEntry out; + ASSERT_TRUE(reader.find_term(terms[i], &found, &out).ok()) << terms[i]; + ASSERT_TRUE(found) << terms[i]; + EXPECT_EQ(out.term, terms[i]); + EXPECT_EQ(out.frq_off_delta, static_cast(i * 50)); + } +} + +// find_term boundaries: less than first term, greater than last term, exactly an anchor term. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiDictBlock, FindTermBoundaries) { + std::vector terms; + for (int i = 0; i < 40; ++i) { + char buf[8]; + snprintf(buf, sizeof(buf), "t%02d", i); + terms.emplace_back(buf); + } + std::vector entries; + for (size_t i = 0; i < terms.size(); ++i) { + entries.push_back(MakePodRef(terms[i], 1, i * 10)); + } + std::vector bytes = BuildBlock(entries, IndexTier::kT1, false, 0, 0, 8); + DictBlockReader reader; + ASSERT_TRUE(DictBlockReader::open(Slice(bytes), IndexTier::kT1, false, &reader).ok()); + + // Less than the first term. + { + bool found = true; + DictEntry out; + ASSERT_TRUE(reader.find_term("\x01", &found, &out).ok()); + EXPECT_FALSE(found); + } + // Greater than the last term. + { + bool found = true; + DictEntry out; + ASSERT_TRUE(reader.find_term("zzzz", &found, &out).ok()); + EXPECT_FALSE(found); + } + // Hit terms that are exactly at anchor positions (anchor_interval=8 -> t08, t16, t24 ...). + for (const char* t : {"t00", "t08", "t16", "t24", "t32"}) { + bool found = false; + DictEntry out; + ASSERT_TRUE(reader.find_term(t, &found, &out).ok()) << t; + ASSERT_TRUE(found) << t; + EXPECT_EQ(out.term, t); + } + // Hit terms that are NOT at anchor positions. + for (const char* t : {"t03", "t11", "t27"}) { + bool found = false; + DictEntry out; + ASSERT_TRUE(reader.find_term(t, &found, &out).ok()) << t; + ASSERT_TRUE(found) << t; + EXPECT_EQ(out.term, t); + } + // A key within the term range that does not exist (gap). + { + bool found = true; + DictEntry out; + ASSERT_TRUE(reader.find_term("t05x", &found, &out).ok()); + EXPECT_FALSE(found); + } +} + +// CRC corruption detection: flipping any byte must cause open() to report Corruption. +TEST(SniiDictBlock, CrcCorruptionDetected) { + std::vector entries = {MakePodRef("aaa", 1, 0), MakePodRef("bbb", 2, 50), + MakePodRef("ccc", 3, 100)}; + std::vector bytes = BuildBlock(entries, IndexTier::kT2, true, 100, 200, 16); + // Flip one byte in the middle of the entries region. + bytes[bytes.size() / 2] ^= 0xFF; + DictBlockReader reader; + Status s = DictBlockReader::open(Slice(bytes), IndexTier::kT2, true, &reader); + EXPECT_FALSE(s.ok()); + EXPECT_FALSE(s.ok()); +} + +// A truncated block (shorter than the CRC footer) must report Corruption rather than crash with an out-of-range access. +TEST(SniiDictBlock, TruncatedBlockDetected) { + std::vector entries = {MakePodRef("xxx", 1, 0)}; + std::vector bytes = BuildBlock(entries, IndexTier::kT1, false, 0, 0, 16); + bytes.resize(2); // Only a few bytes of the header remain. + DictBlockReader reader; + Status s = DictBlockReader::open(Slice(bytes), IndexTier::kT1, false, &reader); + EXPECT_FALSE(s.ok()); +} + +// Positions mode: prx_base and prx fields are preserved correctly. +TEST(SniiDictBlock, PositionsPrxRoundTrip) { + std::vector entries = {MakePodRef("phrase", 5, 0, 0), + MakePodRef("query", 6, 200, 80)}; + std::vector bytes = BuildBlock(entries, IndexTier::kT2, true, 7000, 9000, 16); + DictBlockReader reader; + ASSERT_TRUE(DictBlockReader::open(Slice(bytes), IndexTier::kT2, true, &reader).ok()); + EXPECT_EQ(reader.prx_base(), 9000U); + + bool found = false; + DictEntry out; + ASSERT_TRUE(reader.find_term("query", &found, &out).ok()); + ASSERT_TRUE(found); + EXPECT_EQ(out.prx_off_delta, 80U); + EXPECT_EQ(out.prx_len, 64U); + EXPECT_EQ(out.ttf_delta, 12U); +} + +// estimated_bytes is monotonically non-decreasing: grows after each add_entry, and the actual byte +// count after finish() does not exceed the estimate (the estimate is an upper bound for block-splitting decisions). +TEST(SniiDictBlock, EstimatedBytesMonotonic) { + DictBlockBuilder builder(IndexTier::kT1, false, 0, 0, 16); + size_t prev = builder.estimated_bytes(); + std::vector terms = {"aa", "bb", "cc", "dd"}; + for (const auto& t : terms) { + builder.add_entry(MakePodRef(t, 1, 0)); + size_t now = builder.estimated_bytes(); + EXPECT_GE(now, prev); + prev = now; + } + ByteSink sink; + builder.finish(&sink); + EXPECT_LE(sink.size(), builder.estimated_bytes()); +} + +// Security regression: anchor offsets must be strictly increasing with the first +// anchor at the entries start. A tampered anchor table (offsets swapped) whose crc +// is re-stamped must be rejected by open(); otherwise scan_from_anchor would +// underflow seg_end-seg_begin and read out of bounds (found via ASAN/UBSAN review). +TEST(SniiDictBlock, RejectsNonMonotonicAnchorOffsets) { + std::vector entries = {MakePodRef("aaa", 1, 0), MakePodRef("bbb", 2, 10), + MakePodRef("ccc", 3, 20)}; + // anchor_interval = 1 -> every entry is an anchor (3 anchors). + std::vector bytes = BuildBlock(entries, IndexTier::kT1, false, 100, 0, 1); + const size_t n = bytes.size(); + ASSERT_GT(n, 20U); + // Tail layout: [...][anchor_offsets: 3 * u32][n_anchors u32][crc32c u32]. + const size_t off1 = n - 4 /*crc*/ - 4 /*n_anchors*/ - 4 /*offset[2]*/ - 4 /*offset[1]*/; + const size_t off2 = off1 + 4; + for (int k = 0; k < 4; ++k) { + uint8_t tmp = bytes[off1 + k]; + bytes[off1 + k] = bytes[off2 + k]; + bytes[off2 + k] = tmp; + } + // Re-stamp crc32c over the covered region [0, n-4) so corruption is structural. + uint32_t crc = doris::snii::crc32c(Slice(bytes.data(), n - 4)); + for (int k = 0; k < 4; ++k) { + bytes[n - 4 + k] = static_cast(crc >> (8 * k)); + } + + DictBlockReader reader; + Status s = DictBlockReader::open(Slice(bytes), IndexTier::kT1, false, &reader); + EXPECT_FALSE(s.ok()); +} + +// =========================================================================== +// T16: DictBlockBuilder move-entry overload + dead prev_term_ removal. +// The move overload must be byte-for-byte equivalent to the const-ref (copy) +// overload, must actually transfer (not copy) the entry, and removing the dead +// prev_term_ member must leave finish() output unchanged. These live in the +// SniiDictBlockTest suite per the T16 plan. +// =========================================================================== +namespace { + +// Inline entry carrying BOTH non-empty frq_bytes and prx_bytes, so the move path +// has real heap-allocated vectors (and a heap term) to transfer -- exercised by +// the byte-equivalence and moved-from tests below. +DictEntry MakeInlineWithPrx(std::string term, uint32_t df) { + DictEntry e; + e.term = std::move(term); + e.kind = DictEntryKind::kInline; + e.enc = DictEntryEnc::kSlim; + e.df = df; + e.ttf_delta = df * 3; // written only when tier>=T2 + e.max_freq = 7; // written only when tier>=T2 + e.frq_bytes = {0x01, 0x02, 0x03, 0x04, 0x05}; + e.prx_bytes = {0xAA, 0xBB, 0xCC}; // written only when positions are enabled + return e; +} + +// Build a block by MOVING each entry into the builder. The caller's vector stays +// intact because each entry is copied once and only the copy is moved in (so the +// returned bytes can be compared against the const-ref BuildBlock path). +std::vector BuildBlockMoved(const std::vector& entries, IndexTier tier, + bool has_positions, uint64_t frq_base, uint64_t prx_base, + uint32_t anchor_interval) { + DictBlockBuilder builder(tier, has_positions, frq_base, prx_base, anchor_interval); + for (const auto& e : entries) { + DictEntry tmp = e; // copy, then move the copy into the builder + builder.add_entry(std::move(tmp)); + } + ByteSink sink; + builder.finish(&sink); + return sink.buffer(); +} + +// A mixed, sorted entry set that crosses the default anchor_interval (16): 18 +// entries, inline-with-prx at a few indices, pod_ref elsewhere. Zero-padded term +// names keep lexicographic order == construction order. +std::vector MakeMixedCrossAnchorEntries() { + std::vector entries; + for (int i = 0; i < 18; ++i) { + char buf[16]; + snprintf(buf, sizeof(buf), "term%02d", i); + if (i == 3 || i == 8 || i == 14) { + entries.push_back(MakeInlineWithPrx(buf, static_cast(i + 1))); + } else { + entries.push_back(MakePodRef(buf, static_cast(i + 1), + static_cast(i) * 64, + static_cast(i) * 32)); + } + } + return entries; +} + +} // namespace + +// FC-1 / RED-1: the move overload produces byte-identical block output to the +// const-ref (copy) overload, and the size estimate matches -- the latter only +// holds if estimate_entry_bytes runs BEFORE the move (estimating a moved-from +// entry would undercount entries_est_ and diverge here). +TEST(SniiDictBlockTest, MoveAddProducesByteIdenticalOutput) { + const std::vector entries = MakeMixedCrossAnchorEntries(); + + DictBlockBuilder builder_a(IndexTier::kT2, /*has_positions=*/true, 8192, 16384, 16); + DictBlockBuilder builder_b(IndexTier::kT2, /*has_positions=*/true, 8192, 16384, 16); + for (const auto& e : entries) { + builder_a.add_entry(e); // const-ref copy path + DictEntry tmp = e; + builder_b.add_entry(std::move(tmp)); // move path + } + + // estimate-before-move guard: undercounting on the move path would show here. + EXPECT_EQ(builder_a.estimated_bytes(), builder_b.estimated_bytes()); + EXPECT_EQ(builder_a.n_entries(), builder_b.n_entries()); + + ByteSink sink_a; + ByteSink sink_b; + builder_a.finish(&sink_a); + builder_b.finish(&sink_b); + EXPECT_EQ(sink_a.buffer(), sink_b.buffer()); // on-disk bytes identical +} + +// FC-2 / RED-2: add_entry(DictEntry&&) actually moves -- the source entry's term +// and both byte vectors are left empty (moved-from). A heap-sized term (beyond +// SSO) makes "the buffer was stolen" observable regardless of small-string opt. +TEST(SniiDictBlockTest, MoveAddLeavesSourceMovedFrom) { + DictEntry e = MakeInlineWithPrx("a-deliberately-long-term-well-beyond-sso", 5); + ASSERT_FALSE(e.term.empty()); + ASSERT_FALSE(e.frq_bytes.empty()); + ASSERT_FALSE(e.prx_bytes.empty()); + + DictBlockBuilder builder(IndexTier::kT2, /*has_positions=*/true, 0, 0, 16); + builder.add_entry(std::move(e)); + + EXPECT_TRUE( + e.term.empty()); // NOLINT(bugprone-use-after-move): intentionally inspects moved-from state + EXPECT_TRUE(e.frq_bytes.empty()); // NOLINT(bugprone-use-after-move) + EXPECT_TRUE(e.prx_bytes.empty()); // NOLINT(bugprone-use-after-move) + EXPECT_EQ(builder.n_entries(), 1U); +} + +// FC-3: single-entry block built via the move overload round-trips through +// open()/find_term() (open succeeding also verifies the block CRC). +TEST(SniiDictBlockTest, MoveAddSingleEntryRoundTrip) { + DictEntry e = MakePodRef("solo", 7, 0); + const DictEntry expected = e; // pristine copy for comparison + + DictBlockBuilder builder(IndexTier::kT1, /*has_positions=*/false, 4096, 0, 16); + builder.add_entry(std::move(e)); + ByteSink sink; + builder.finish(&sink); + + DictBlockReader reader; + ASSERT_TRUE(DictBlockReader::open(Slice(sink.buffer()), IndexTier::kT1, + /*has_positions=*/false, &reader) + .ok()); + EXPECT_EQ(reader.n_entries(), 1U); + EXPECT_EQ(reader.frq_base(), 4096U); + + bool found = false; + DictEntry out; + ASSERT_TRUE(reader.find_term("solo", &found, &out).ok()); + ASSERT_TRUE(found); + ExpectCommon(expected, out); + EXPECT_EQ(out.frq_off_delta, expected.frq_off_delta); + EXPECT_EQ(out.frq_len, expected.frq_len); +} + +// FC-4: empty-block boundary still serializes and opens cleanly (the move +// overload is irrelevant when nothing is added, but the boundary must hold). +TEST(SniiDictBlockTest, MoveAddEmptyBlock) { + DictBlockBuilder builder(IndexTier::kT1, /*has_positions=*/false, 1000, 0, 16); + ByteSink sink; + builder.finish(&sink); + + DictBlockReader reader; + ASSERT_TRUE(DictBlockReader::open(Slice(sink.buffer()), IndexTier::kT1, + /*has_positions=*/false, &reader) + .ok()); + EXPECT_EQ(reader.n_entries(), 0U); + + bool found = true; + DictEntry out; + ASSERT_TRUE(reader.find_term("anything", &found, &out).ok()); + EXPECT_FALSE(found); +} + +// FC-5: anchor-boundary readback. anchor_interval default = 16, so 17 move-added +// entries force a second anchor (index 16); decode_all must reset the +// front-coding base at each anchor segment and return every term in order. +TEST(SniiDictBlockTest, MoveAddAnchorBoundaryDecodeAll) { + std::vector terms; + std::vector entries; + for (int i = 0; i < 17; ++i) { + char buf[16]; + snprintf(buf, sizeof(buf), "key%02d", i); + terms.emplace_back(buf); + entries.push_back( + MakePodRef(buf, static_cast(i + 1), static_cast(i) * 50)); + } + + DictBlockBuilder builder(IndexTier::kT1, /*has_positions=*/false, 0, 0, /*anchor_interval=*/16); + for (const auto& e : entries) { + DictEntry tmp = e; + builder.add_entry(std::move(tmp)); + } + ByteSink sink; + builder.finish(&sink); + + DictBlockReader reader; + ASSERT_TRUE(DictBlockReader::open(Slice(sink.buffer()), IndexTier::kT1, + /*has_positions=*/false, &reader) + .ok()); + EXPECT_EQ(reader.n_entries(), 17U); + + std::vector all; + ASSERT_TRUE(reader.decode_all(&all).ok()); + ASSERT_EQ(all.size(), entries.size()); + for (size_t i = 0; i < entries.size(); ++i) { + EXPECT_EQ(all[i].term, terms[i]) << i; + ExpectCommon(entries[i], all[i]); + EXPECT_EQ(all[i].frq_off_delta, static_cast(i) * 50) << i; + } +} + +// RED-3: removing the dead prev_term_ member must not change finish() output. +// A multi-anchor, long-shared-prefix dataset stresses front coding across anchor +// segments -- exactly what finish() rebuilds from its local `prev` (never from +// the removed member). The const-ref path is the pristine golden; the move path +// must reproduce it byte-for-byte, and the block must decode back to the inputs. +TEST(SniiDictBlockTest, FinishUnaffectedByDeadPrevTermRemoval) { + const std::vector terms = { + "interest", "interested", "interesting", "interestingly", "interests", + "internal", "internally", "international", "internet", "internets", + "interplay", "interpose", "interpret", "interpreted", "interval", + "intervene", "interview", "interviewed", "intestine", "intimate", + "intricate"}; + std::vector entries; + uint64_t off = 0; + for (const auto& t : terms) { + entries.push_back(MakePodRef(t, 4, off)); + off += 50; + } + + const std::vector golden = BuildBlock(entries, IndexTier::kT1, /*has_positions=*/false, + 0, 0, /*anchor_interval=*/4); + const std::vector moved = + BuildBlockMoved(entries, IndexTier::kT1, /*has_positions=*/false, 0, 0, 4); + EXPECT_EQ(golden, moved); // dead-field removal + move path leave bytes identical + + DictBlockReader reader; + ASSERT_TRUE(DictBlockReader::open(Slice(golden), IndexTier::kT1, false, &reader).ok()); + std::vector all; + ASSERT_TRUE(reader.decode_all(&all).ok()); + ASSERT_EQ(all.size(), terms.size()); + for (size_t i = 0; i < terms.size(); ++i) { + EXPECT_EQ(all[i].term, terms[i]) << i; // front-coding base rebuilt without prev_term_ + } +} diff --git a/be/test/storage/index/snii/format/dict_entry_test.cpp b/be/test/storage/index/snii/format/dict_entry_test.cpp new file mode 100644 index 00000000000000..33005415f473a1 --- /dev/null +++ b/be/test/storage/index/snii/format/dict_entry_test.cpp @@ -0,0 +1,370 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/dict_entry.h" + +#include + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/format_constants.h" + +using namespace doris::snii; // NOLINT +using namespace doris::snii::format; // NOLINT +using doris::Status; + +namespace { + +// Construct a pod_ref slim dict entry (tier determines which optional fields +// are present). +DictEntry MakePodRefSlim(std::string term, uint32_t df) { + DictEntry e; + e.term = std::move(term); + e.kind = DictEntryKind::kPodRef; + e.enc = DictEntryEnc::kSlim; + e.has_sb = false; + e.df = df; + e.ttf_delta = df * 3; // written only when tier>=T2 + e.max_freq = 7; // written only when tier>=T2 + e.frq_off_delta = 4096; + e.frq_len = 333; + e.frq_docs_len = 200; // slim pod_ref: docs-only prefix (<= frq_len) + e.prx_off_delta = 8192; // written only when positions are stored + e.prx_len = 512; // written only when positions are stored + return e; +} + +DictEntry MakePodRefWindowed(std::string term, uint32_t df) { + DictEntry e = MakePodRefSlim(std::move(term), df); + e.enc = DictEntryEnc::kWindowed; + e.has_sb = true; + e.prelude_len = 64; // written only for windowed encoding + e.frq_docs_len = 240; // windowed docs-only prefix: [prelude][dd-block] + return e; +} + +DictEntry MakeInline(std::string term, uint32_t df) { + DictEntry e; + e.term = std::move(term); + e.kind = DictEntryKind::kInline; + e.enc = DictEntryEnc::kSlim; + e.df = df; + e.ttf_delta = df * 2; + e.max_freq = 5; + e.frq_bytes = {0x01, 0x02, 0x03, 0x04}; + e.prx_bytes = {0xAA, 0xBB}; // written only when positions are stored + return e; +} + +// round-trip: encode then decode, assert that fields retained by the given tier +// match. +DictEntry RoundTrip(const DictEntry& in, std::string_view prev, IndexTier tier) { + ByteSink sink; + EXPECT_TRUE(encode_dict_entry(in, prev, tier, &sink).ok()); + ByteSource src(sink.view()); + DictEntry out; + Status s = decode_dict_entry(&src, prev, tier, &out); + EXPECT_TRUE(s.ok()) << s.to_string(); + EXPECT_EQ(src.remaining(), 0U) << "decode did not consume the entire entry"; + return out; +} + +void ExpectCommon(const DictEntry& a, const DictEntry& b) { + EXPECT_EQ(a.term, b.term); + EXPECT_EQ(a.kind, b.kind); + EXPECT_EQ(a.enc, b.enc); + EXPECT_EQ(a.has_sb, b.has_sb); + EXPECT_EQ(a.df, b.df); +} + +} // namespace + +TEST(SniiDictEntry, PodRefSlimTier1RoundTrip) { + DictEntry in = MakePodRefSlim("apple", 42); + DictEntry out = RoundTrip(in, "", IndexTier::kT1); + ExpectCommon(in, out); + EXPECT_EQ(out.frq_off_delta, in.frq_off_delta); + EXPECT_EQ(out.frq_len, in.frq_len); + EXPECT_EQ(out.frq_docs_len, in.frq_docs_len); // slim docs-only prefix preserved + // tier1: ttf/max_freq/prx are not written; decode restores them to default 0. + EXPECT_EQ(out.ttf_delta, 0U); + EXPECT_EQ(out.max_freq, 0U); + EXPECT_EQ(out.prx_len, 0U); +} + +TEST(SniiDictEntry, PodRefSlimTier2RoundTrip) { + DictEntry in = MakePodRefSlim("banana", 100); + DictEntry out = RoundTrip(in, "", IndexTier::kT2); + ExpectCommon(in, out); + EXPECT_EQ(out.frq_docs_len, in.frq_docs_len); + EXPECT_EQ(out.ttf_delta, in.ttf_delta); + EXPECT_EQ(out.max_freq, in.max_freq); + EXPECT_EQ(out.prx_off_delta, in.prx_off_delta); + EXPECT_EQ(out.prx_len, in.prx_len); +} + +// A slim pod_ref whose frq_docs_len exceeds frq_len is rejected on decode. +TEST(SniiDictEntry, PodRefSlimFrqDocsLenExceedsFrqLenRejected) { + DictEntry in = MakePodRefSlim("kiwi", 50); + in.frq_docs_len = in.frq_len + 1; // impossible prefix + ByteSink sink; + ASSERT_TRUE(encode_dict_entry(in, "", IndexTier::kT2, &sink).ok()); + ByteSource src(sink.view()); + DictEntry out; + Status s = decode_dict_entry(&src, "", IndexTier::kT2, &out); + EXPECT_TRUE(s.is()); +} + +TEST(SniiDictEntry, PodRefSlimTier3RoundTrip) { + DictEntry in = MakePodRefSlim("cherry", 500); + DictEntry out = RoundTrip(in, "", IndexTier::kT3); + ExpectCommon(in, out); + EXPECT_EQ(out.ttf_delta, in.ttf_delta); + EXPECT_EQ(out.max_freq, in.max_freq); + EXPECT_EQ(out.prx_off_delta, in.prx_off_delta); + EXPECT_EQ(out.prx_len, in.prx_len); +} + +TEST(SniiDictEntry, PodRefWindowedTier2RoundTrip) { + DictEntry in = MakePodRefWindowed("durian", 2000); + DictEntry out = RoundTrip(in, "", IndexTier::kT2); + ExpectCommon(in, out); + EXPECT_EQ(out.enc, DictEntryEnc::kWindowed); + EXPECT_EQ(out.has_sb, true); + EXPECT_EQ(out.prelude_len, in.prelude_len); + EXPECT_EQ(out.frq_docs_len, in.frq_docs_len); + EXPECT_EQ(out.prx_len, in.prx_len); +} + +TEST(SniiDictEntry, PodRefWindowedTier1RoundTrip) { + DictEntry in = MakePodRefWindowed("elderberry", 1500); + DictEntry out = RoundTrip(in, "", IndexTier::kT1); + ExpectCommon(in, out); + EXPECT_EQ(out.prelude_len, in.prelude_len); + EXPECT_EQ(out.frq_docs_len, in.frq_docs_len); + EXPECT_EQ(out.prx_len, 0U); // tier1 has no prx +} + +TEST(SniiDictEntry, PodRefWindowedDocsPrefixBoundsRejected) { + DictEntry in = MakePodRefWindowed("jackfruit", 1500); + in.frq_docs_len = in.prelude_len - 1; + ByteSink sink; + ASSERT_TRUE(encode_dict_entry(in, "", IndexTier::kT2, &sink).ok()); + ByteSource src(sink.view()); + DictEntry out; + Status s = decode_dict_entry(&src, "", IndexTier::kT2, &out); + EXPECT_TRUE(s.is()); + + in.frq_docs_len = in.frq_len + 1; + ByteSink sink2; + ASSERT_TRUE(encode_dict_entry(in, "", IndexTier::kT2, &sink2).ok()); + ByteSource src2(sink2.view()); + s = decode_dict_entry(&src2, "", IndexTier::kT2, &out); + EXPECT_TRUE(s.is()); +} + +TEST(SniiDictEntry, InlineTier1RoundTrip) { + DictEntry in = MakeInline("fig", 3); + DictEntry out = RoundTrip(in, "", IndexTier::kT1); + ExpectCommon(in, out); + EXPECT_EQ(out.frq_bytes, in.frq_bytes); + EXPECT_TRUE(out.prx_bytes.empty()); // tier1 has no prx +} + +TEST(SniiDictEntry, InlineTier2RoundTrip) { + DictEntry in = MakeInline("grape", 8); + DictEntry out = RoundTrip(in, "", IndexTier::kT2); + ExpectCommon(in, out); + EXPECT_EQ(out.frq_bytes, in.frq_bytes); + EXPECT_EQ(out.prx_bytes, in.prx_bytes); + EXPECT_EQ(out.ttf_delta, in.ttf_delta); + EXPECT_EQ(out.max_freq, in.max_freq); +} + +TEST(SniiDictEntry, InlineTier3RoundTrip) { + DictEntry in = MakeInline("honeydew", 12); + DictEntry out = RoundTrip(in, "", IndexTier::kT3); + ExpectCommon(in, out); + EXPECT_EQ(out.frq_bytes, in.frq_bytes); + EXPECT_EQ(out.prx_bytes, in.prx_bytes); +} + +// Format v2: an INLINE entry omits the redundant per-region crc32c bytes +// (dd_crc + freq_crc) -- its region bytes are covered by the dict block crc32c. +// Decoded inline metas carry verify_crc=false so region decode skips the check. +// A slim POD-ref entry KEEPS its crc (regions live in the separately-fetched +// .frq POD), so it is exactly 8 bytes larger for the same df. +TEST(SniiDictEntry, InlineOmitsRedundantRegionCrc) { + DictEntry inl = MakeInline("melon", 8); + inl.dd_meta.crc = 0xDEADBEEF; // would be written by v1; must be omitted now + inl.freq_meta.crc = 0xCAFEF00D; // ditto + ByteSink inl_sink; + ASSERT_TRUE(encode_dict_entry(inl, "", IndexTier::kT2, &inl_sink).ok()); + + // Same payload but encoded as a slim POD-ref keeps both crcs (dd + freq = + // 8B). + DictEntry ref = inl; + ref.kind = DictEntryKind::kPodRef; + ref.frq_len = inl.frq_bytes.size(); + ref.frq_docs_len = inl.inline_dd_disk_len; + ByteSink ref_sink; + ASSERT_TRUE(encode_dict_entry(ref, "", IndexTier::kT2, &ref_sink).ok()); + // The POD-ref locator differs structurally; assert the inline encoding does + // not carry the 8 crc bytes by decoding and checking verify_crc is OFF. + ByteSource src(inl_sink.view()); + DictEntry out; + ASSERT_TRUE(decode_dict_entry(&src, "", IndexTier::kT2, &out).ok()); + EXPECT_FALSE(out.dd_meta.verify_crc); + EXPECT_FALSE(out.freq_meta.verify_crc); + EXPECT_EQ(out.dd_meta.crc, 0U); // crc not stored -> decoded as 0 + EXPECT_EQ(out.freq_meta.crc, 0U); +} + +// A POD-ref slim entry decodes with verify_crc=true (its regions carry a crc). +TEST(SniiDictEntry, PodRefSlimKeepsRegionCrc) { + DictEntry in = MakePodRefSlim("nectarine", 50); + in.dd_meta.crc = 0x12345678; + in.freq_meta.crc = 0x9ABCDEF0; + DictEntry out = RoundTrip(in, "", IndexTier::kT2); + EXPECT_TRUE(out.dd_meta.verify_crc); + EXPECT_TRUE(out.freq_meta.verify_crc); + EXPECT_EQ(out.dd_meta.crc, 0x12345678U); + EXPECT_EQ(out.freq_meta.crc, 0x9ABCDEF0U); +} + +// Front coding: consecutive encode/decode of sorted terms; suffix stores only +// the differing part. +TEST(SniiDictEntry, PrefixCompressionSharedPrefix) { + // "interest" and "interesting" share the prefix "interest". + DictEntry a = MakePodRefSlim("interest", 10); + DictEntry b = MakePodRefSlim("interesting", 11); + + ByteSink sink; + ASSERT_TRUE(encode_dict_entry(a, "", IndexTier::kT2, &sink).ok()); + size_t a_end = sink.size(); + ASSERT_TRUE(encode_dict_entry(b, a.term, IndexTier::kT2, &sink).ok()); + + ByteSource src(sink.view()); + DictEntry oa; + ASSERT_TRUE(decode_dict_entry(&src, "", IndexTier::kT2, &oa).ok()); + EXPECT_EQ(src.position(), a_end); + EXPECT_EQ(oa.term, "interest"); + + DictEntry ob; + ASSERT_TRUE(decode_dict_entry(&src, oa.term, IndexTier::kT2, &ob).ok()); + EXPECT_EQ(ob.term, "interesting"); + EXPECT_TRUE(src.eof()); +} + +// Encode/decode three sorted terms consecutively to verify the prefix chain. +TEST(SniiDictEntry, PrefixCompressionChain) { + std::vector terms = {"app", "apple", "application"}; + ByteSink sink; + std::string prev; + for (const auto& t : terms) { + DictEntry e = MakePodRefSlim(t, 5); + ASSERT_TRUE(encode_dict_entry(e, prev, IndexTier::kT1, &sink).ok()); + prev = t; + } + ByteSource src(sink.view()); + prev.clear(); + for (const auto& t : terms) { + DictEntry out; + ASSERT_TRUE(decode_dict_entry(&src, prev, IndexTier::kT1, &out).ok()); + EXPECT_EQ(out.term, t); + prev = out.term; + } + EXPECT_TRUE(src.eof()); +} + +// entry_len must allow a reader to skip the entire entry without parsing its +// internal fields. +TEST(SniiDictEntry, EntryLenAllowsSkip) { + DictEntry a = MakePodRefSlim("first", 9); + DictEntry b = MakeInline("second", 4); + ByteSink sink; + ASSERT_TRUE(encode_dict_entry(a, "", IndexTier::kT2, &sink).ok()); + ASSERT_TRUE(encode_dict_entry(b, a.term, IndexTier::kT2, &sink).ok()); + + ByteSource src(sink.view()); + // Skip the first entry using only entry_len. + ASSERT_TRUE(skip_dict_entry(&src).ok()); + DictEntry out; + ASSERT_TRUE(decode_dict_entry(&src, a.term, IndexTier::kT2, &out).ok()); + EXPECT_EQ(out.term, "second"); + EXPECT_TRUE(src.eof()); +} + +// Edge case: empty suffix (term is identical to prev). +TEST(SniiDictEntry, EmptySuffixEqualsPrev) { + DictEntry in = MakePodRefSlim("same", 1); + DictEntry out = RoundTrip(in, "same", IndexTier::kT1); + EXPECT_EQ(out.term, "same"); +} + +// Edge case: df = 0. +TEST(SniiDictEntry, ZeroDf) { + DictEntry in = MakePodRefSlim("zero", 0); + DictEntry out = RoundTrip(in, "", IndexTier::kT2); + EXPECT_EQ(out.df, 0U); +} + +// Edge case: empty term (first entry, prev is empty, suffix is also empty). +TEST(SniiDictEntry, EmptyTerm) { + DictEntry in = MakePodRefSlim("", 1); + DictEntry out = RoundTrip(in, "", IndexTier::kT1); + EXPECT_EQ(out.term, ""); +} + +// Structural integrity: no CRC at the entry level (CRC is at the DICT block +// level), but entry_len and the actual body length must match; tampering with +// entry_len to make it smaller than the real body -> decode must report +// Corruption. +TEST(SniiDictEntry, EntryLenMismatchDetected) { + DictEntry in = MakePodRefSlim("payload", 99); + ByteSink sink; + ASSERT_TRUE(encode_dict_entry(in, "", IndexTier::kT2, &sink).ok()); + std::vector bytes = sink.buffer(); + ASSERT_GT(bytes[0], 1U); // single-byte entry_len varint + bytes[0] -= 1; // claim the body is 1 byte shorter than it actually is + + Slice corrupt(bytes); + ByteSource src(corrupt); + DictEntry out; + Status s = decode_dict_entry(&src, "", IndexTier::kT2, &out); + EXPECT_FALSE(s.ok()); + EXPECT_TRUE(s.is()); +} + +// prefix_len exceeding the length of prev must be rejected (guard against +// malformed input). +TEST(SniiDictEntry, RejectPrefixLongerThanPrev) { + DictEntry in = MakePodRefSlim("abcdef", 1); + ByteSink sink; + ASSERT_TRUE(encode_dict_entry(in, "abc", IndexTier::kT1, &sink).ok()); + // Decode with a shorter prev: prefix_len(=3) > prev.size()(=2) -> Corruption. + ByteSource src(sink.view()); + DictEntry out; + Status s = decode_dict_entry(&src, "ab", IndexTier::kT1, &out); + EXPECT_FALSE(s.ok()); +} diff --git a/be/test/storage/index/snii/format/format_constants_test.cpp b/be/test/storage/index/snii/format/format_constants_test.cpp new file mode 100644 index 00000000000000..9a23fce614538e --- /dev/null +++ b/be/test/storage/index/snii/format/format_constants_test.cpp @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/format_constants.h" + +#include + +#include "common/status.h" + +using namespace doris::snii::format; + +// Lock down on-disk contract values to prevent accidental changes from breaking +// readability of already-written files. +TEST(SniiFormatConstants, MagicAndVersionStable) { + EXPECT_EQ(kContainerMagic, 0x49494E53U); + EXPECT_EQ(kTailMagic, 0x4C494154U); + EXPECT_EQ(kFormatVersion, 2); + EXPECT_EQ(kMinReaderVersion, 2); +} + +TEST(SniiFormatConstants, ConfigToTierMapping) { + EXPECT_EQ(tier_of(IndexConfig::kDocsOnly), IndexTier::kT1); + EXPECT_EQ(tier_of(IndexConfig::kDocsPositions), IndexTier::kT2); + EXPECT_EQ(tier_of(IndexConfig::kDocsPositionsScoring), IndexTier::kT3); +} + +TEST(SniiFormatConstants, CapabilityPredicates) { + EXPECT_FALSE(has_positions(IndexConfig::kDocsOnly)); + EXPECT_TRUE(has_positions(IndexConfig::kDocsPositions)); + EXPECT_TRUE(has_scoring(IndexConfig::kDocsPositionsScoring)); + EXPECT_FALSE(has_scoring(IndexConfig::kDocsPositions)); +} + +TEST(SniiFormatConstants, DictFlagBitsDistinct) { + EXPECT_EQ(dict_flags::kKind, 0x01); + EXPECT_EQ(dict_flags::kEnc, 0x02); + EXPECT_EQ(dict_flags::kHasSb, 0x04); +} diff --git a/be/test/storage/index/snii/format/frq_pod_test.cpp b/be/test/storage/index/snii/format/frq_pod_test.cpp new file mode 100644 index 00000000000000..095c18b2678a08 --- /dev/null +++ b/be/test/storage/index/snii/format/frq_pod_test.cpp @@ -0,0 +1,527 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/frq_pod.h" + +#include + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/encoding/pfor.h" +#include "storage/index/snii/encoding/zstd_codec.h" +#include "storage/index/snii/format/format_constants.h" + +using doris::snii::ByteSink; +using doris::snii::Slice; +using doris::Status; +using doris::snii::format::build_dd_region; +using doris::snii::format::build_freq_region; +using doris::snii::format::decode_dd_region; +using doris::snii::format::decode_freq_region; +using doris::snii::format::FrqRegionMeta; + +namespace { + +using U32Vec = std::vector; + +// Round-trips a window's separately-encoded dd + freq regions: build each +// region, then decode the dd region (docs-only) and the freq region, comparing +// originals. +Status RoundTrip(const U32Vec& docs, const U32Vec& freqs, uint64_t win_base, bool has_freq, + int level, U32Vec* out_docs, U32Vec* out_freqs) { + ByteSink dd_sink; + FrqRegionMeta dd_meta; + RETURN_IF_ERROR(build_dd_region(docs, win_base, level, &dd_sink, &dd_meta)); + RETURN_IF_ERROR(decode_dd_region(Slice(dd_sink.buffer()), dd_meta, win_base, out_docs)); + if (!has_freq) { + out_freqs->clear(); + return Status::OK(); + } + ByteSink freq_sink; + FrqRegionMeta freq_meta; + RETURN_IF_ERROR(build_freq_region(freqs, level, &freq_sink, &freq_meta)); + return decode_freq_region(Slice(freq_sink.buffer()), freq_meta, out_docs->size(), out_freqs); +} + +} // namespace + +// Basic round-trip: first window win_base=0, both dd and freq are restored. +TEST(SniiFrqPod, BasicDocFreqRoundTrip) { + U32Vec docs = {0, 3, 5, 10, 11, 50, 200}; + U32Vec freqs = {1, 2, 1, 7, 3, 1, 9}; + U32Vec out_docs, out_freqs; + ASSERT_TRUE(RoundTrip(docs, freqs, /*win_base=*/0, /*has_freq=*/true, -1, &out_docs, &out_freqs) + .ok()); + EXPECT_EQ(out_docs, docs); + EXPECT_EQ(out_freqs, freqs); +} + +// Non-first window: win_base != 0, dd[0]=first-win_base, cross-window delta +// rebuild. +TEST(SniiFrqPod, NonFirstWindowDeltaRebuild) { + uint64_t win_base = 1000; + U32Vec docs = {1001, 1005, 1006, 2000, 2001}; + U32Vec freqs = {3, 1, 1, 2, 8}; + U32Vec out_docs, out_freqs; + ASSERT_TRUE( + RoundTrip(docs, freqs, win_base, /*has_freq=*/true, -1, &out_docs, &out_freqs).ok()); + EXPECT_EQ(out_docs, docs); + EXPECT_EQ(out_freqs, freqs); +} + +// dd region decodes WITHOUT the freq region present (docs-only path). +TEST(SniiFrqPod, DdRegionDecodesWithoutFreq) { + uint64_t win_base = 500; + U32Vec docs = {600, 700, 701, 900}; + ByteSink dd_sink; + FrqRegionMeta dd_meta; + ASSERT_TRUE(build_dd_region(docs, win_base, -1, &dd_sink, &dd_meta).ok()); + U32Vec out_docs; + ASSERT_TRUE(decode_dd_region(Slice(dd_sink.buffer()), dd_meta, win_base, &out_docs).ok()); + EXPECT_EQ(out_docs, docs); +} + +// Single-doc window. +TEST(SniiFrqPod, SingleDocWindow) { + U32Vec docs = {42}; + U32Vec freqs = {7}; + U32Vec out_docs, out_freqs; + ASSERT_TRUE(RoundTrip(docs, freqs, /*win_base=*/0, /*has_freq=*/true, -1, &out_docs, &out_freqs) + .ok()); + EXPECT_EQ(out_docs, docs); + EXPECT_EQ(out_freqs, freqs); +} + +// Large window (2048 docs): auto mode triggers zstd on each region and is +// lossless. +TEST(SniiFrqPod, LargeWindowAutoZstdRoundTrip) { + U32Vec docs, freqs; + docs.reserve(2048); + freqs.reserve(2048); + uint32_t cur = 0; + for (int i = 0; i < 2048; ++i) { + cur += 1 + (i % 4); + docs.push_back(cur); + freqs.push_back(1 + (i % 3)); + } + ByteSink dd_sink; + FrqRegionMeta dd_meta; + ASSERT_TRUE(build_dd_region(docs, 0, -1, &dd_sink, &dd_meta).ok()); + EXPECT_TRUE(dd_meta.zstd); // dd region large enough for zstd + + U32Vec out_docs, out_freqs; + ASSERT_TRUE(RoundTrip(docs, freqs, /*win_base=*/0, /*has_freq=*/true, -1, &out_docs, &out_freqs) + .ok()); + EXPECT_EQ(out_docs, docs); + EXPECT_EQ(out_freqs, freqs); +} + +// Small window level<0 auto -> raw (dd region not compressed). +TEST(SniiFrqPod, SmallWindowUsesRaw) { + U32Vec docs = {0, 1, 2}; + ByteSink dd_sink; + FrqRegionMeta dd_meta; + ASSERT_TRUE(build_dd_region(docs, 0, -1, &dd_sink, &dd_meta).ok()); + EXPECT_FALSE(dd_meta.zstd); + EXPECT_EQ(dd_meta.disk_len, dd_meta.uncomp_len); // raw => disk == uncomp +} + +// Explicit level=0 forces raw on both regions; large window still lossless. +TEST(SniiFrqPod, ExplicitRawLargeWindowRoundTrip) { + U32Vec docs, freqs; + uint32_t cur = 0; + for (int i = 0; i < 600; ++i) { + cur += 1 + (i % 7); + docs.push_back(cur); + freqs.push_back(1 + (i % 5)); + } + ByteSink dd_sink, freq_sink; + FrqRegionMeta dd_meta, freq_meta; + ASSERT_TRUE(build_dd_region(docs, 0, /*level=*/0, &dd_sink, &dd_meta).ok()); + ASSERT_TRUE(build_freq_region(freqs, /*level=*/0, &freq_sink, &freq_meta).ok()); + EXPECT_FALSE(dd_meta.zstd); + EXPECT_FALSE(freq_meta.zstd); + + U32Vec out_docs, out_freqs; + ASSERT_TRUE(decode_dd_region(Slice(dd_sink.buffer()), dd_meta, 0, &out_docs).ok()); + ASSERT_TRUE( + decode_freq_region(Slice(freq_sink.buffer()), freq_meta, out_docs.size(), &out_freqs) + .ok()); + EXPECT_EQ(out_docs, docs); + EXPECT_EQ(out_freqs, freqs); +} + +// Explicit level>0 forces zstd, still lossless. +TEST(SniiFrqPod, ExplicitZstdRoundTrip) { + U32Vec docs, freqs; + for (uint32_t i = 0; i < 300; ++i) { + docs.push_back(i * 2); + freqs.push_back(2); + } + U32Vec out_docs, out_freqs; + ASSERT_TRUE(RoundTrip(docs, freqs, /*win_base=*/0, /*has_freq=*/true, + /*level=*/5, &out_docs, &out_freqs) + .ok()); + EXPECT_EQ(out_docs, docs); + EXPECT_EQ(out_freqs, freqs); +} + +// Non-ascending docids must be rejected (InvalidArgument). +TEST(SniiFrqPod, NonAscendingDocsRejected) { + U32Vec docs = {0, 5, 3}; + ByteSink sink; + FrqRegionMeta meta; + Status s = build_dd_region(docs, 0, -1, &sink, &meta); + EXPECT_TRUE(s.is()); +} + +// first_docid < win_base (dd[0] would underflow) -> InvalidArgument. +TEST(SniiFrqPod, FirstDocBelowWinBaseRejected) { + U32Vec docs = {100, 200}; + ByteSink sink; + FrqRegionMeta meta; + Status s = build_dd_region(docs, 500, -1, &sink, &meta); + EXPECT_TRUE(s.is()); +} + +// CRC corruption inside the dd region is caught by crc_dd on decode. +TEST(SniiFrqPod, DdRegionCrcCorruptionDetected) { + U32Vec docs = {0, 1, 2, 3, 4, 5}; + ByteSink sink; + FrqRegionMeta meta; + ASSERT_TRUE(build_dd_region(docs, 0, -1, &sink, &meta).ok()); + std::vector bytes = sink.buffer(); + ASSERT_GT(bytes.size(), 1U); + bytes.back() ^= 0xFF; + U32Vec out_docs; + Status s = decode_dd_region(Slice(bytes), meta, 0, &out_docs); + EXPECT_TRUE(s.is()); +} + +// CRC corruption inside the freq region is caught by crc_freq on decode. +TEST(SniiFrqPod, FreqRegionCrcCorruptionDetected) { + U32Vec freqs = {1, 2, 3, 4, 5, 6}; + ByteSink sink; + FrqRegionMeta meta; + ASSERT_TRUE(build_freq_region(freqs, -1, &sink, &meta).ok()); + std::vector bytes = sink.buffer(); + ASSERT_GT(bytes.size(), 1U); + bytes.back() ^= 0xFF; + U32Vec out_freqs; + Status s = decode_freq_region(Slice(bytes), meta, freqs.size(), &out_freqs); + EXPECT_TRUE(s.is()); +} + +// A region slice whose length disagrees with meta.disk_len is rejected +// (anti-DoS). +TEST(SniiFrqPod, RegionSliceLengthMismatchRejected) { + U32Vec docs = {0, 1, 2, 3}; + ByteSink sink; + FrqRegionMeta meta; + ASSERT_TRUE(build_dd_region(docs, 0, /*level=*/0, &sink, &meta).ok()); + std::vector bytes = sink.buffer(); + bytes.pop_back(); // slice now shorter than disk_len + U32Vec out_docs; + Status s = decode_dd_region(Slice(bytes), meta, 0, &out_docs); + EXPECT_TRUE(s.is()); +} + +// Empty window (0 docs) round-trip: dd region has n=0, freq region is empty. +TEST(SniiFrqPod, EmptyWindowRoundTrip) { + U32Vec docs, freqs; + U32Vec out_docs, out_freqs; + ASSERT_TRUE(RoundTrip(docs, freqs, /*win_base=*/0, /*has_freq=*/true, -1, &out_docs, &out_freqs) + .ok()); + EXPECT_TRUE(out_docs.empty()); + EXPECT_TRUE(out_freqs.empty()); +} + +// dd region on-disk length is strictly smaller than dd+freq combined (freq +// carries real bytes), proving the docs-only prefix saving at the region level. +TEST(SniiFrqPod, DdRegionSmallerThanCombined) { + U32Vec docs = {0, 3, 5, 10, 11, 50, 200}; + U32Vec freqs = {1, 2, 1, 7, 3, 1, 9}; + ByteSink dd_sink, freq_sink; + FrqRegionMeta dd_meta, freq_meta; + ASSERT_TRUE(build_dd_region(docs, 0, -1, &dd_sink, &dd_meta).ok()); + ASSERT_TRUE(build_freq_region(freqs, -1, &freq_sink, &freq_meta).ok()); + EXPECT_GT(dd_meta.disk_len, 0U); + EXPECT_GT(freq_meta.disk_len, 0U); + EXPECT_LT(dd_meta.disk_len, dd_meta.disk_len + freq_meta.disk_len); +} + +// Null argument guard. +TEST(SniiFrqPod, NullArgsRejected) { + U32Vec docs = {0, 1}; + FrqRegionMeta meta; + EXPECT_TRUE( + build_dd_region(docs, 0, -1, nullptr, &meta).is()); +} + +// N=0 empty-term prelude round-trips: the dd region encodes VInt n=0 (no PFOR +// runs) and the freq region is zero-length; both decode back to empty vectors. +TEST(SniiFrqPod, EmptyTermZeroDocsRoundTrip) { + U32Vec docs; // N = 0 (empty term) + U32Vec freqs; // no freqs + ByteSink dd_sink, freq_sink; + FrqRegionMeta dd_meta, freq_meta; + ASSERT_TRUE(build_dd_region(docs, /*win_base=*/0, /*level=*/0, &dd_sink, &dd_meta).ok()); + ASSERT_TRUE(build_freq_region(freqs, /*level=*/0, &freq_sink, &freq_meta).ok()); + EXPECT_EQ(freq_meta.uncomp_len, 0U); // empty freq region carries no bytes + + U32Vec out_docs, out_freqs; + ASSERT_TRUE(decode_dd_region(Slice(dd_sink.buffer()), dd_meta, 0, &out_docs).ok()); + EXPECT_TRUE(out_docs.empty()); + ASSERT_TRUE( + decode_freq_region(Slice(freq_sink.buffer()), freq_meta, out_docs.size(), &out_freqs) + .ok()); + EXPECT_TRUE(out_freqs.empty()); +} + +// A truncated freq region (slice shorter than meta.disk_len) is rejected: the +// length-mismatch guard in open_region fires before any payload decode. +TEST(SniiFrqPod, TruncatedFreqRegionRejected) { + U32Vec freqs = {1, 2, 3, 4, 5, 6, 7, 8}; + ByteSink sink; + FrqRegionMeta meta; + ASSERT_TRUE(build_freq_region(freqs, /*level=*/0, &sink, &meta).ok()); + std::vector bytes = sink.buffer(); + ASSERT_GT(bytes.size(), 1U); + bytes.pop_back(); // slice now shorter than meta.disk_len + U32Vec out_freqs; + Status s = decode_freq_region(Slice(bytes), meta, freqs.size(), &out_freqs); + EXPECT_TRUE(s.is()); +} + +// A dd region with extra trailing bytes (crc + length kept VALID for the longer +// slice) must be rejected by the post-decode eof() check, not the crc/length +// guards. We craft a raw region, append a garbage byte, then fix disk_len, +// uncomp_len, and crc so every earlier guard passes and only the inner +// "trailing bytes" assertion can fire. +TEST(SniiFrqPod, DdRegionTrailingBytesRejected) { + U32Vec docs = {0, 1, 2, 3, 4, 5}; + ByteSink sink; + FrqRegionMeta meta; + ASSERT_TRUE(build_dd_region(docs, 0, /*level=*/0, &sink, &meta).ok()); // raw + ASSERT_FALSE(meta.zstd); + std::vector bytes = sink.buffer(); + bytes.push_back(0x00); // garbage trailing byte appended to the plaintext + // Keep raw invariant uncomp_len == disk_len and a matching crc so open_region + // accepts the slice; only the payload-level eof() check can reject it. + meta.disk_len = bytes.size(); + meta.uncomp_len = bytes.size(); + meta.crc = doris::snii::crc32c(Slice(bytes)); + U32Vec out_docs; + Status s = decode_dd_region(Slice(bytes), meta, 0, &out_docs); + EXPECT_TRUE(s.is()); +} + +// A freq region with extra trailing bytes (crc + length kept VALID) must be +// rejected by the post-decode eof() check. Same crafting strategy as above. +TEST(SniiFrqPod, FreqRegionTrailingBytesRejected) { + U32Vec freqs = {3, 1, 4, 1, 5, 9}; + ByteSink sink; + FrqRegionMeta meta; + ASSERT_TRUE(build_freq_region(freqs, /*level=*/0, &sink, &meta).ok()); // raw + ASSERT_FALSE(meta.zstd); + std::vector bytes = sink.buffer(); + bytes.push_back(0x00); // garbage trailing byte appended to the plaintext + meta.disk_len = bytes.size(); + meta.uncomp_len = bytes.size(); + meta.crc = doris::snii::crc32c(Slice(bytes)); + U32Vec out_freqs; + Status s = decode_freq_region(Slice(bytes), meta, freqs.size(), &out_freqs); + EXPECT_TRUE(s.is()); +} + +// The uncomp_len sanity cap (kMaxRegionUncompBytes = 256 MiB) guards against a +// corrupted length that would inflate to a giant allocation. We keep the +// on-disk slice + crc valid (so the length/crc guards pass) and only override +// uncomp_len to an absurd value, proving the cap branch is what rejects it. +TEST(SniiFrqPod, UncompLenCapRejected) { + U32Vec docs = {0, 1, 2, 3, 4, 5}; + ByteSink sink; + FrqRegionMeta meta; + ASSERT_TRUE(build_dd_region(docs, 0, /*level=*/0, &sink, &meta).ok()); + std::vector bytes = sink.buffer(); + // kMaxRegionUncompBytes is an internal (anonymous-namespace) constant; use + // its literal value (256 MiB) + 1 to step just past the cap. + meta.uncomp_len = static_cast(256U * 1024 * 1024) + 1; + U32Vec out_docs; + Status s = decode_dd_region(Slice(bytes), meta, 0, &out_docs); + EXPECT_TRUE(s.is()); +} + +// =========================================================================== +// T22 -- emit_region single-copy (raw branch writes the plaintext view straight +// to `out` with no temp `disk` vector). The raw region's on-disk bytes must equal +// the plaintext EXACTLY (regions carry no header / no trailing crc), and the meta +// (disk_len / crc / zstd / uncomp_len) must be unchanged. The zstd branch keeps +// its own buffer and stays byte-identical. All bytes MUST match the pre-refactor +// output. +// =========================================================================== + +namespace { + +// PFOR_runs(values) wire form: fixed runs of kFrqBaseUnit, run count derived from +// n (mirrors the in-source encode_pfor_runs so a test can rebuild region bytes). +void AppendPforRuns(const U32Vec& values, ByteSink* out) { + const size_t n = values.size(); + const size_t unit = doris::snii::format::kFrqBaseUnit; + for (size_t off = 0; off < n; off += unit) { + const size_t run = (n - off < unit) ? (n - off) : unit; + doris::snii::pfor_encode(values.data() + off, run, out); + } +} + +// dd_region plaintext (VInt n ++ PFOR_runs(doc_delta)) -- the exact bytes +// emit_region writes for a raw dd region. +ByteSink DdPlaintext(const U32Vec& docs, uint64_t win_base) { + U32Vec dd(docs.size()); + uint64_t prev = win_base; + for (size_t i = 0; i < docs.size(); ++i) { + dd[i] = static_cast(static_cast(docs[i]) - prev); + prev = docs[i]; + } + ByteSink plain; + plain.put_varint32(static_cast(docs.size())); + AppendPforRuns(dd, &plain); + return plain; +} + +} // namespace + +// [perf-deterministic] FRQ-BYTE-DD: a raw (level=0) dd region's on-disk bytes +// equal the plaintext exactly, and the meta invariants hold: disk_len == +// plain.size(), uncomp_len == plain.size(), crc == crc32c(plain), zstd == false. +TEST(SniiFrqPodTest, DdRegionRawProducesByteIdenticalOutputAndMeta) { + U32Vec docs = {0, 3, 5, 10, 11, 50, 200}; + const uint64_t win_base = 0; + ByteSink out; + FrqRegionMeta meta; + ASSERT_TRUE(build_dd_region(docs, win_base, /*level=*/0, &out, &meta).ok()); + ASSERT_FALSE(meta.zstd); + + const ByteSink plain = DdPlaintext(docs, win_base); + EXPECT_EQ(out.buffer(), plain.buffer()); // raw region == plaintext, byte-for-byte + EXPECT_EQ(meta.disk_len, plain.view().size()); + EXPECT_EQ(meta.uncomp_len, plain.view().size()); + EXPECT_EQ(meta.crc, doris::snii::crc32c(plain.view())); + EXPECT_EQ(meta.crc, doris::snii::crc32c(Slice(out.buffer()))); + + U32Vec got; + ASSERT_TRUE(decode_dd_region(Slice(out.buffer()), meta, win_base, &got).ok()); + EXPECT_EQ(got, docs); +} + +// [perf-deterministic] FRQ-BYTE-FREQ: a raw (level=0) freq region's on-disk bytes +// equal PFOR_runs(freqs) exactly (no count prefix), with matching meta. +TEST(SniiFrqPodTest, FreqRegionRawProducesByteIdenticalOutputAndMeta) { + U32Vec freqs = {1, 2, 1, 7, 3, 1, 9}; + ByteSink out; + FrqRegionMeta meta; + ASSERT_TRUE(build_freq_region(freqs, /*level=*/0, &out, &meta).ok()); + ASSERT_FALSE(meta.zstd); + + ByteSink plain; // freq_region plaintext == PFOR_runs(freqs) + AppendPforRuns(freqs, &plain); + EXPECT_EQ(out.buffer(), plain.buffer()); + EXPECT_EQ(meta.disk_len, plain.view().size()); + EXPECT_EQ(meta.uncomp_len, plain.view().size()); + EXPECT_EQ(meta.crc, doris::snii::crc32c(plain.view())); + + U32Vec got; + ASSERT_TRUE(decode_freq_region(Slice(out.buffer()), meta, freqs.size(), &got).ok()); + EXPECT_EQ(got, freqs); +} + +// [perf-deterministic] FRQ-ZSTD-PATH: the zstd branch (level>0) is preserved +// byte-identical -- the on-disk bytes equal an independent zstd(plaintext) at the +// same level, with meta.zstd == true, disk_len == comp.size(), uncomp_len == +// plaintext size, and a lossless round-trip. +TEST(SniiFrqPodTest, DdRegionZstdBranchPreservedByteIdentical) { + U32Vec docs; + uint32_t cur = 0; + for (uint32_t i = 0; i < 2048; ++i) { + cur += 1 + (i % 4); + docs.push_back(cur); + } + const uint64_t win_base = 0; + ByteSink out; + FrqRegionMeta meta; + ASSERT_TRUE(build_dd_region(docs, win_base, /*level=*/5, &out, &meta).ok()); + ASSERT_TRUE(meta.zstd); + + const ByteSink plain = DdPlaintext(docs, win_base); + std::vector comp; + ASSERT_TRUE(doris::snii::zstd_compress(plain.view(), /*level=*/5, &comp).ok()); + EXPECT_EQ(out.buffer(), comp); + EXPECT_EQ(meta.disk_len, comp.size()); + EXPECT_EQ(meta.uncomp_len, plain.view().size()); + EXPECT_EQ(meta.crc, doris::snii::crc32c(Slice(comp))); + + U32Vec got; + ASSERT_TRUE(decode_dd_region(Slice(out.buffer()), meta, win_base, &got).ok()); + EXPECT_EQ(got, docs); +} + +// [functional] FRQ-RT: a raw dd + freq round-trip with the meta raw invariant +// (disk_len == uncomp_len) checked on both regions -- the single-copy raw path +// must keep uncomp_len == disk_len so open_region accepts the region. +TEST(SniiFrqPodTest, RawRegionRoundTripKeepsDiskEqualsUncomp) { + U32Vec docs = {100, 103, 104, 200, 201}; + U32Vec freqs = {1, 4, 2, 1, 9}; + const uint64_t win_base = 100; + ByteSink dd_sink, freq_sink; + FrqRegionMeta dd_meta, freq_meta; + ASSERT_TRUE(build_dd_region(docs, win_base, /*level=*/0, &dd_sink, &dd_meta).ok()); + ASSERT_TRUE(build_freq_region(freqs, /*level=*/0, &freq_sink, &freq_meta).ok()); + EXPECT_FALSE(dd_meta.zstd); + EXPECT_FALSE(freq_meta.zstd); + EXPECT_EQ(dd_meta.disk_len, dd_meta.uncomp_len); + EXPECT_EQ(freq_meta.disk_len, freq_meta.uncomp_len); + + U32Vec out_docs, out_freqs; + ASSERT_TRUE(decode_dd_region(Slice(dd_sink.buffer()), dd_meta, win_base, &out_docs).ok()); + ASSERT_TRUE( + decode_freq_region(Slice(freq_sink.buffer()), freq_meta, out_docs.size(), &out_freqs) + .ok()); + EXPECT_EQ(out_docs, docs); + EXPECT_EQ(out_freqs, freqs); +} + +// [functional/boundary] FRQ-DOCCOUNT0: an empty freq region (0 freqs) yields a +// zero-length region with zero-valued meta and decodes to an empty vector. +TEST(SniiFrqPodTest, FreqRegionZeroDocCountDecodesEmpty) { + U32Vec freqs; // empty term + ByteSink out; + FrqRegionMeta meta; + ASSERT_TRUE(build_freq_region(freqs, /*level=*/0, &out, &meta).ok()); + EXPECT_EQ(meta.uncomp_len, 0U); + EXPECT_EQ(meta.disk_len, 0U); + EXPECT_FALSE(meta.zstd); + EXPECT_TRUE(out.buffer().empty()); + + U32Vec got = {123U}; // sentinel proves decode clears it + ASSERT_TRUE(decode_freq_region(Slice(out.buffer()), meta, /*doc_count=*/0, &got).ok()); + EXPECT_TRUE(got.empty()); +} diff --git a/be/test/storage/index/snii/format/frq_prelude_test.cpp b/be/test/storage/index/snii/format/frq_prelude_test.cpp new file mode 100644 index 00000000000000..c088c0d4c0c7da --- /dev/null +++ b/be/test/storage/index/snii/format/frq_prelude_test.cpp @@ -0,0 +1,428 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/frq_prelude.h" + +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/crc32c.h" + +using doris::snii::ByteSink; +using doris::snii::Slice; +using doris::snii::format::build_frq_prelude; +using doris::snii::format::FrqPreludeColumns; +using doris::snii::format::FrqPreludeReader; +using doris::snii::format::WindowMeta; +using doris::Status; + +namespace { + +// Builds N windows that tile a docid space in fixed strides, with deterministic +// per-window metadata. doc_count is `stride`; absolute last docid of window w is +// (w+1)*stride - 1 so windows are contiguous and strictly ascending. dd and freq +// region offsets are CONTIGUOUS prefix sums (the prelude validates this). +FrqPreludeColumns MakeColumns(uint32_t n, uint32_t group_size, uint32_t stride, bool has_prx) { + FrqPreludeColumns cols; + cols.has_freq = true; + cols.has_prx = has_prx; + cols.group_size = group_size; + uint64_t dd_running = 0, freq_running = 0, prx_running = 0; + for (uint32_t w = 0; w < n; ++w) { + WindowMeta m; + m.last_docid = (w + 1) * stride - 1; + m.doc_count = stride; + m.dd_zstd = (w % 2) == 0; + m.dd_off = dd_running; + m.dd_disk_len = 8 + w; + // T18: a raw region's uncomp_len == disk_len (the row stores uncomp_len only + // for zstd regions; the reader derives it otherwise). Keep the test columns + // on that invariant so the round-trip's derived uncomp_len matches. + m.dd_uncomp_len = m.dd_zstd ? m.dd_disk_len + 2 : m.dd_disk_len; + m.crc_dd = 0xDD000000U + w; + dd_running += m.dd_disk_len; + m.freq_zstd = (w % 3) == 0; + m.freq_off = freq_running; + m.freq_disk_len = 5 + (w % 4); + m.freq_uncomp_len = m.freq_zstd ? m.freq_disk_len + 1 : m.freq_disk_len; + m.crc_freq = 0xEE000000U + w; + freq_running += m.freq_disk_len; + if (has_prx) { + m.prx_off = prx_running; + m.prx_len = 7 + (w % 5); + prx_running += m.prx_len; + } + m.max_freq = w * 3 + 1; + m.max_norm = static_cast((w * 13 + 3) & 0xFF); + cols.windows.push_back(m); + } + return cols; +} + +ByteSink MakeSingleWindowPrelude(uint64_t last_docid_delta, uint64_t doc_count, uint64_t max_freq) { + ByteSink block; + block.put_varint64(last_docid_delta); + block.put_varint64(doc_count); + block.put_u8(0); // win_mode: raw dd/freq (no zstd bit => no uncomp_len fields) + block.put_varint64(4); // dd_disk_len (dd_uncomp_len omitted: raw region) + block.put_fixed32(0xDDU); // crc_dd + block.put_varint64(0); // freq_disk_len (freq_uncomp_len omitted: raw region) + block.put_fixed32(0xEEU); // crc_freq + block.put_varint64(max_freq); + block.put_u8(0); // max_norm + + ByteSink dir; + dir.put_varint64(last_docid_delta); + dir.put_varint64(0); // first block starts at the window region + dir.put_varint64(block.size()); + + ByteSink covered; + covered.put_u8(doris::snii::format::frq_prelude_flags::kHasFreq); + covered.put_varint64(1); // N + covered.put_varint64(1); // G + covered.put_varint64(1); // n_super + covered.put_varint64(dir.size()); + covered.put_bytes(dir.view()); + + ByteSink frame; + frame.put_bytes(covered.view()); + frame.put_fixed32(doris::snii::crc32c(covered.view())); + frame.put_bytes(block.view()); + return frame; +} + +// Round-trips columns through build + open, asserting every window's absolute +// fields match (and win_base chains correctly). +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +void ExpectRoundTrip(const FrqPreludeColumns& cols) { + ByteSink sink; + ASSERT_TRUE(build_frq_prelude(cols, &sink).ok()); + + FrqPreludeReader reader; + ASSERT_TRUE(FrqPreludeReader::open(sink.view(), &reader).ok()); + + ASSERT_EQ(reader.n_windows(), cols.windows.size()); + EXPECT_EQ(reader.has_freq(), cols.has_freq); + EXPECT_EQ(reader.has_prx(), cols.has_prx); + + uint64_t expect_win_base = 0; + for (uint32_t w = 0; w < reader.n_windows(); ++w) { + WindowMeta got; + ASSERT_TRUE(reader.window(w, &got).ok()) << "window w=" << w; + const WindowMeta& exp = cols.windows[w]; + EXPECT_EQ(got.last_docid, exp.last_docid) << "ldd w=" << w; + EXPECT_EQ(got.win_base, expect_win_base) << "win_base w=" << w; + EXPECT_EQ(got.doc_count, exp.doc_count) << "doc_count w=" << w; + EXPECT_EQ(got.dd_zstd, exp.dd_zstd) << "dd_zstd w=" << w; + EXPECT_EQ(got.dd_off, exp.dd_off) << "dd_off w=" << w; + EXPECT_EQ(got.dd_disk_len, exp.dd_disk_len) << "dd_disk_len w=" << w; + EXPECT_EQ(got.dd_uncomp_len, exp.dd_uncomp_len) << "dd_uncomp_len w=" << w; + EXPECT_EQ(got.crc_dd, exp.crc_dd) << "crc_dd w=" << w; + EXPECT_EQ(got.freq_zstd, exp.freq_zstd) << "freq_zstd w=" << w; + EXPECT_EQ(got.freq_off, exp.freq_off) << "freq_off w=" << w; + EXPECT_EQ(got.freq_disk_len, exp.freq_disk_len) << "freq_disk_len w=" << w; + EXPECT_EQ(got.freq_uncomp_len, exp.freq_uncomp_len) << "freq_uncomp_len w=" << w; + EXPECT_EQ(got.crc_freq, exp.crc_freq) << "crc_freq w=" << w; + EXPECT_EQ(got.max_freq, exp.max_freq) << "max_freq w=" << w; + EXPECT_EQ(got.max_norm, exp.max_norm) << "max_norm w=" << w; + if (cols.has_prx) { + EXPECT_EQ(got.prx_off, exp.prx_off) << "prx_off w=" << w; + EXPECT_EQ(got.prx_len, exp.prx_len) << "prx_len w=" << w; + } + expect_win_base = exp.last_docid; + } +} + +} // namespace + +// Many windows across multiple super-blocks, no prx. +TEST(SniiFrqPrelude, RoundTripManyWindowsNoPrx) { + ExpectRoundTrip(MakeColumns(/*n=*/20, /*group_size=*/4, /*stride=*/256, false)); +} + +// Many windows across multiple super-blocks, with prx. +TEST(SniiFrqPrelude, RoundTripManyWindowsWithPrx) { + ExpectRoundTrip(MakeColumns(/*n=*/17, /*group_size=*/4, /*stride=*/256, true)); +} + +// Single window: N=1 collapses to a single super-block. +TEST(SniiFrqPrelude, SingleWindow) { + FrqPreludeColumns cols = MakeColumns(/*n=*/1, /*group_size=*/64, /*stride=*/300, true); + ByteSink sink; + ASSERT_TRUE(build_frq_prelude(cols, &sink).ok()); + FrqPreludeReader reader; + ASSERT_TRUE(FrqPreludeReader::open(sink.view(), &reader).ok()); + EXPECT_EQ(reader.n_windows(), 1U); + EXPECT_EQ(reader.n_super_blocks(), 1U); + WindowMeta m; + ASSERT_TRUE(reader.window(0, &m).ok()); + EXPECT_EQ(m.win_base, 0U); + EXPECT_EQ(m.last_docid, 299U); +} + +// N exactly == group_size -> exactly one super-block. +TEST(SniiFrqPrelude, ExactlyOneSuperBlock) { + FrqPreludeColumns cols = MakeColumns(/*n=*/4, /*group_size=*/4, /*stride=*/256, true); + ByteSink sink; + ASSERT_TRUE(build_frq_prelude(cols, &sink).ok()); + FrqPreludeReader reader; + ASSERT_TRUE(FrqPreludeReader::open(sink.view(), &reader).ok()); + EXPECT_EQ(reader.n_super_blocks(), 1U); +} + +// Large N exercises many super-blocks and varint streams. +TEST(SniiFrqPrelude, LargeN) { + ExpectRoundTrip(MakeColumns(/*n=*/1000, /*group_size=*/64, /*stride=*/256, true)); +} + +// locate_window finds the covering window at boundaries, inside, and past-end. +TEST(SniiFrqPrelude, LocateWindowBoundaries) { + // 10 windows, stride 100 -> window w covers docids [w*100, w*100+99]. + FrqPreludeColumns cols = MakeColumns(/*n=*/10, /*group_size=*/3, /*stride=*/100, true); + ByteSink sink; + ASSERT_TRUE(build_frq_prelude(cols, &sink).ok()); + FrqPreludeReader reader; + ASSERT_TRUE(FrqPreludeReader::open(sink.view(), &reader).ok()); + + struct Case { + uint32_t docid; + bool found; + uint32_t w; + }; + const Case cases[] = { + {.docid = 0, .found = true, .w = 0}, // first docid of window 0 + {.docid = 99, .found = true, .w = 0}, // last docid of window 0 + {.docid = 100, .found = true, .w = 1}, // first docid of window 1 (boundary) + {.docid = 250, .found = true, .w = 2}, // inside window 2 + {.docid = 299, .found = true, .w = 2}, // last docid of window 2 + {.docid = 300, .found = true, .w = 3}, // boundary into window 3 + {.docid = 999, .found = true, .w = 9}, // last docid overall (window 9 covers [900,999]) + {.docid = 1000, .found = false, .w = 0}, // past the term's last docid + {.docid = 5000, .found = false, .w = 0}, // far past end + }; + for (const auto& c : cases) { + bool found = false; + uint32_t w = 999; + ASSERT_TRUE(reader.locate_window(c.docid, &found, &w).ok()) << "docid=" << c.docid; + EXPECT_EQ(found, c.found) << "docid=" << c.docid; + if (c.found) { + EXPECT_EQ(w, c.w) << "docid=" << c.docid; + } + } +} + +// locate_window must agree with a linear scan over windows for every docid in a +// gappy docid layout (windows not contiguous: gaps between windows). +TEST(SniiFrqPrelude, LocateWindowAgainstLinearScan) { + FrqPreludeColumns cols; + cols.has_prx = true; + cols.group_size = 4; + // Non-contiguous absolute last docids with gaps: 50, 130, 131, 400, 900. + const uint32_t lasts[] = {50, 130, 131, 400, 900}; + uint64_t dd_running = 0, freq_running = 0, prx_running = 0; + for (uint32_t v : lasts) { + WindowMeta m; + m.last_docid = v; + m.doc_count = 1; + m.dd_off = dd_running; + m.dd_disk_len = 12; + m.dd_uncomp_len = 12; + m.crc_dd = v; + dd_running += 12; + m.freq_off = freq_running; + m.freq_disk_len = 6; + m.freq_uncomp_len = 6; + m.crc_freq = v + 1; + freq_running += 6; + m.prx_off = prx_running; + m.prx_len = 6; + prx_running += 6; + cols.windows.push_back(m); + } + ByteSink sink; + ASSERT_TRUE(build_frq_prelude(cols, &sink).ok()); + FrqPreludeReader reader; + ASSERT_TRUE(FrqPreludeReader::open(sink.view(), &reader).ok()); + + for (uint32_t docid = 0; docid <= 950; ++docid) { + // Linear oracle: first window whose absolute last_docid >= docid. + bool exp_found = false; + uint32_t exp_w = 0; + for (uint32_t w = 0; w < std::size(lasts); ++w) { + if (docid <= lasts[w]) { + exp_found = true; + exp_w = w; + break; + } + } + bool found = false; + uint32_t w = 999; + ASSERT_TRUE(reader.locate_window(docid, &found, &w).ok()); + EXPECT_EQ(found, exp_found) << "docid=" << docid; + if (exp_found) { + EXPECT_EQ(w, exp_w) << "docid=" << docid; + } + } +} + +// Builder rejects a null sink. +TEST(SniiFrqPrelude, BuildNullSinkRejected) { + FrqPreludeColumns cols = MakeColumns(3, 4, 100, false); + EXPECT_TRUE(build_frq_prelude(cols, nullptr).is()); +} + +// Builder rejects group_size == 0. +TEST(SniiFrqPrelude, BuildZeroGroupSizeRejected) { + FrqPreludeColumns cols = MakeColumns(3, 4, 100, false); + cols.group_size = 0; + ByteSink sink; + EXPECT_TRUE(build_frq_prelude(cols, &sink).is()); +} + +// Builder rejects non-monotonic last_docid ordering across windows. +TEST(SniiFrqPrelude, BuildNonMonotonicRejected) { + FrqPreludeColumns cols = MakeColumns(3, 4, 100, false); + cols.windows[2].last_docid = cols.windows[1].last_docid - 1; // goes backwards + ByteSink sink; + EXPECT_TRUE(build_frq_prelude(cols, &sink).is()); +} + +// T18: dd_off is DERIVED (running prefix sum), not serialized. Tampering a +// column's dd_off no longer changes the on-disk bytes, and the reader +// reconstructs the contiguous offset regardless -- so the offset is always +// contiguous by construction (pre-T18 this stored + cross-checked an explicit +// dd_off, and a gap was rejected on open). +TEST(SniiFrqPrelude, DdOffsetIsDerivedNotStored) { + FrqPreludeColumns cols = MakeColumns(/*n=*/5, /*group_size=*/4, /*stride=*/256, false); + const uint64_t expect_dd_off_2 = + cols.windows[0].dd_disk_len + cols.windows[1].dd_disk_len; // running sum + cols.windows[2].dd_off += 100; // in-memory tamper has no on-disk effect now + ByteSink sink; + ASSERT_TRUE(build_frq_prelude(cols, &sink).ok()); + FrqPreludeReader reader; + ASSERT_TRUE(FrqPreludeReader::open(sink.view(), &reader).ok()); + WindowMeta m; + ASSERT_TRUE(reader.window(2, &m).ok()); + EXPECT_EQ(m.dd_off, expect_dd_off_2); // derived, not the tampered column value +} + +TEST(SniiFrqPrelude, RejectsDocCountBeyondWindowWidth) { + FrqPreludeColumns cols = MakeColumns(/*n=*/1, /*group_size=*/1, + /*stride=*/10, false); + cols.windows[0].doc_count = 11; + ByteSink sink; + ASSERT_TRUE(build_frq_prelude(cols, &sink).ok()); + FrqPreludeReader reader; + Status s = FrqPreludeReader::open(sink.view(), &reader); + EXPECT_TRUE(s.is()); +} + +TEST(SniiFrqPrelude, RejectsWindowDocCountThatDoesNotFitU32) { + ByteSink sink = MakeSingleWindowPrelude( + /*last_docid_delta=*/0, static_cast(std::numeric_limits::max()) + 1, + /*max_freq=*/1); + FrqPreludeReader reader; + Status s = FrqPreludeReader::open(sink.view(), &reader); + EXPECT_TRUE(s.is()); +} + +TEST(SniiFrqPrelude, RejectsWindowMaxFreqThatDoesNotFitU32) { + ByteSink sink = MakeSingleWindowPrelude( + /*last_docid_delta=*/0, /*doc_count=*/1, + static_cast(std::numeric_limits::max()) + 1); + FrqPreludeReader reader; + Status s = FrqPreludeReader::open(sink.view(), &reader); + EXPECT_TRUE(s.is()); +} + +// The reader reports dd_block_len / freq_block_len as the sums of per-window region +// disk lengths (the contiguous block sizes a reader range-fetches). +TEST(SniiFrqPrelude, BlockLengthsAreRegionSums) { + FrqPreludeColumns cols = MakeColumns(/*n=*/6, /*group_size=*/4, /*stride=*/256, true); + ByteSink sink; + ASSERT_TRUE(build_frq_prelude(cols, &sink).ok()); + FrqPreludeReader reader; + ASSERT_TRUE(FrqPreludeReader::open(sink.view(), &reader).ok()); + uint64_t dd_sum = 0, freq_sum = 0; + for (const auto& m : cols.windows) { + dd_sum += m.dd_disk_len; + freq_sum += m.freq_disk_len; + } + EXPECT_EQ(reader.dd_block_len(), dd_sum); + EXPECT_EQ(reader.freq_block_len(), freq_sum); +} + +// CRC corruption inside the header / super_block_dir is detected by open(). +TEST(SniiFrqPrelude, CrcCorruptionDetected) { + FrqPreludeColumns cols = MakeColumns(8, 4, 256, true); + ByteSink sink; + ASSERT_TRUE(build_frq_prelude(cols, &sink).ok()); + std::vector bytes = sink.buffer(); + ASSERT_GE(bytes.size(), 4U); + bytes[1] ^= 0xFF; // flip a flags/header byte + FrqPreludeReader reader; + Status s = FrqPreludeReader::open(Slice(bytes), &reader); + EXPECT_TRUE(s.is()); +} + +// Truncated buffer (chopping into a window block) is rejected. +TEST(SniiFrqPrelude, TruncatedRejected) { + FrqPreludeColumns cols = MakeColumns(8, 4, 256, true); + ByteSink sink; + ASSERT_TRUE(build_frq_prelude(cols, &sink).ok()); + std::vector bytes = sink.buffer(); + bytes.resize(bytes.size() - 5); // drop tail bytes of the last window block + FrqPreludeReader reader; + Status s = FrqPreludeReader::open(Slice(bytes), &reader); + EXPECT_FALSE(s.ok()); +} + +// Out-of-range window() returns InvalidArgument (no OOB read). +TEST(SniiFrqPrelude, WindowOutOfRangeRejected) { + FrqPreludeColumns cols = MakeColumns(3, 4, 100, true); + ByteSink sink; + ASSERT_TRUE(build_frq_prelude(cols, &sink).ok()); + FrqPreludeReader reader; + ASSERT_TRUE(FrqPreludeReader::open(sink.view(), &reader).ok()); + WindowMeta m; + EXPECT_TRUE(reader.window(99, &m).is()); +} + +// Anti-DoS: a crc-valid header declaring an absurd window count N must be +// rejected before any reserve(N). We craft header + super_block_dir bytes by +// hand with a matching crc, so verify_crc passes but the count is rejected. +TEST(SniiFrqPrelude, RejectsOversizedWindowCount) { + ByteSink covered; + covered.put_u8(doris::snii::format::frq_prelude_flags::kHasFreq); + covered.put_varint64(0xFFFFFFFFULL); // N: absurd window count + covered.put_varint64(64); // G + covered.put_varint64(1); // n_super (bogus but small) + covered.put_varint64(0); // sbdir_len = 0 + ByteSink frame; + frame.put_bytes(covered.view()); + frame.put_fixed32(doris::snii::crc32c(covered.view())); + FrqPreludeReader reader; + Status s = FrqPreludeReader::open(frame.view(), &reader); + EXPECT_TRUE(s.is()); +} diff --git a/be/test/storage/index/snii/format/logical_index_directory_test.cpp b/be/test/storage/index/snii/format/logical_index_directory_test.cpp new file mode 100644 index 00000000000000..9b339b85d01046 --- /dev/null +++ b/be/test/storage/index/snii/format/logical_index_directory_test.cpp @@ -0,0 +1,313 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/logical_index_directory.h" + +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/section_framer.h" +#include "storage/index/snii/format/format_constants.h" + +using namespace doris::snii; +using namespace doris::snii::format; + +namespace { + +// Serialize a list of refs via the builder and return the framed section bytes. +std::vector Build(const std::vector& refs) { + LogicalIndexDirectoryBuilder builder; + for (const auto& r : refs) { + builder.add(r); + } + ByteSink sink; + builder.finish(&sink); + return sink.buffer(); +} + +// Assert that two LogicalIndexRef structs are equal field by field. +void ExpectRefEq(const LogicalIndexRef& a, const LogicalIndexRef& b) { + EXPECT_EQ(a.index_id, b.index_id); + EXPECT_EQ(a.index_suffix, b.index_suffix); + EXPECT_EQ(a.meta_off, b.meta_off); + EXPECT_EQ(a.meta_len, b.meta_len); +} + +} // namespace + +TEST(SniiLogicalIndexDirectory, RoundTripMultipleEntries) { + std::vector refs = { + {.index_id = 1, .index_suffix = "", .meta_off = 0, .meta_len = 4096}, + {.index_id = 2, .index_suffix = "fulltext", .meta_off = 4096, .meta_len = 8192}, + {.index_id = 7, .index_suffix = "phrase_v2", .meta_off = 12288, .meta_len = 2048}, + }; + auto bytes = Build(refs); + + LogicalIndexDirectoryReader reader; + ASSERT_TRUE(LogicalIndexDirectoryReader::open(Slice(bytes), &reader).ok()); + ASSERT_EQ(reader.size(), 3U); + for (uint32_t i = 0; i < refs.size(); ++i) { + LogicalIndexRef out {}; + ASSERT_TRUE(reader.get(i, &out).ok()); + ExpectRefEq(out, refs[i]); + } +} + +TEST(SniiLogicalIndexDirectory, GetOrdinalOutOfRangeRejected) { + std::vector refs = { + {.index_id = 1, .index_suffix = "a", .meta_off = 0, .meta_len = 100}}; + auto bytes = Build(refs); + + LogicalIndexDirectoryReader reader; + ASSERT_TRUE(LogicalIndexDirectoryReader::open(Slice(bytes), &reader).ok()); + ASSERT_EQ(reader.size(), 1U); + LogicalIndexRef out {}; + // ordinal == size is out of range + EXPECT_TRUE(reader.get(1, &out).is()); + EXPECT_TRUE(reader.get(1000, &out).is()); +} + +TEST(SniiLogicalIndexDirectory, FindHit) { + std::vector refs = { + {.index_id = 1, .index_suffix = "", .meta_off = 0, .meta_len = 4096}, + {.index_id = 2, .index_suffix = "fulltext", .meta_off = 4096, .meta_len = 8192}, + {.index_id = 7, .index_suffix = "phrase_v2", .meta_off = 12288, .meta_len = 2048}, + }; + auto bytes = Build(refs); + + LogicalIndexDirectoryReader reader; + ASSERT_TRUE(LogicalIndexDirectoryReader::open(Slice(bytes), &reader).ok()); + + bool found = false; + LogicalIndexRef out {}; + ASSERT_TRUE(reader.find(2, "fulltext", &found, &out).ok()); + EXPECT_TRUE(found); + ExpectRefEq(out, refs[1]); + + // hit on the empty-suffix entry + found = false; + ASSERT_TRUE(reader.find(1, "", &found, &out).ok()); + EXPECT_TRUE(found); + ExpectRefEq(out, refs[0]); +} + +TEST(SniiLogicalIndexDirectory, FindMissByIdAndBySuffix) { + std::vector refs = { + {.index_id = 1, .index_suffix = "", .meta_off = 0, .meta_len = 4096}, + {.index_id = 2, .index_suffix = "fulltext", .meta_off = 4096, .meta_len = 8192}, + }; + auto bytes = Build(refs); + + LogicalIndexDirectoryReader reader; + ASSERT_TRUE(LogicalIndexDirectoryReader::open(Slice(bytes), &reader).ok()); + + bool found = true; + LogicalIndexRef out {}; + // unknown index_id + ASSERT_TRUE(reader.find(99, "fulltext", &found, &out).ok()); + EXPECT_FALSE(found); + + // known index_id but wrong suffix + found = true; + ASSERT_TRUE(reader.find(2, "wrong", &found, &out).ok()); + EXPECT_FALSE(found); + + // known index_id but suffix mismatch (empty vs non-empty) + found = true; + ASSERT_TRUE(reader.find(1, "fulltext", &found, &out).ok()); + EXPECT_FALSE(found); +} + +TEST(SniiLogicalIndexDirectory, EmptyVersusNonEmptySuffixSameId) { + // Same index_id, different suffix: both must be addressable distinctly. + std::vector refs = { + {.index_id = 5, .index_suffix = "", .meta_off = 0, .meta_len = 100}, + {.index_id = 5, .index_suffix = "tokenized", .meta_off = 100, .meta_len = 200}, + }; + auto bytes = Build(refs); + + LogicalIndexDirectoryReader reader; + ASSERT_TRUE(LogicalIndexDirectoryReader::open(Slice(bytes), &reader).ok()); + ASSERT_EQ(reader.size(), 2U); + + bool found = false; + LogicalIndexRef out {}; + ASSERT_TRUE(reader.find(5, "", &found, &out).ok()); + EXPECT_TRUE(found); + ExpectRefEq(out, refs[0]); + + found = false; + ASSERT_TRUE(reader.find(5, "tokenized", &found, &out).ok()); + EXPECT_TRUE(found); + ExpectRefEq(out, refs[1]); +} + +TEST(SniiLogicalIndexDirectory, DuplicateIdDifferentSuffix) { + // Multiple entries share index_id with distinct suffixes (Doris sub-column indexes). + std::vector refs = { + {.index_id = 10, .index_suffix = "a", .meta_off = 0, .meta_len = 10}, + {.index_id = 10, .index_suffix = "b", .meta_off = 10, .meta_len = 20}, + {.index_id = 10, .index_suffix = "c", .meta_off = 30, .meta_len = 30}, + }; + auto bytes = Build(refs); + + LogicalIndexDirectoryReader reader; + ASSERT_TRUE(LogicalIndexDirectoryReader::open(Slice(bytes), &reader).ok()); + ASSERT_EQ(reader.size(), 3U); + + for (const auto& want : refs) { + bool found = false; + LogicalIndexRef out {}; + ASSERT_TRUE(reader.find(10, want.index_suffix, &found, &out).ok()); + EXPECT_TRUE(found); + ExpectRefEq(out, want); + } +} + +TEST(SniiLogicalIndexDirectory, EmptyDirectory) { + std::vector refs; // 0 entries + auto bytes = Build(refs); + + LogicalIndexDirectoryReader reader; + ASSERT_TRUE(LogicalIndexDirectoryReader::open(Slice(bytes), &reader).ok()); + EXPECT_EQ(reader.size(), 0U); + + LogicalIndexRef out {}; + EXPECT_TRUE(reader.get(0, &out).is()); + + bool found = true; + ASSERT_TRUE(reader.find(1, "x", &found, &out).ok()); + EXPECT_FALSE(found); +} + +TEST(SniiLogicalIndexDirectory, LargeOffsetsRoundTrip) { + const uint64_t kBig = (1ULL << 48) - 1; // near 2^48 + std::vector refs = { + {.index_id = 0xFFFFFFFFFFFFFFFFULL, + .index_suffix = "edge", + .meta_off = kBig, + .meta_len = kBig - 1}, + {.index_id = 12345, .index_suffix = "", .meta_off = kBig + 99, .meta_len = 1}, + }; + auto bytes = Build(refs); + + LogicalIndexDirectoryReader reader; + ASSERT_TRUE(LogicalIndexDirectoryReader::open(Slice(bytes), &reader).ok()); + ASSERT_EQ(reader.size(), 2U); + LogicalIndexRef out0 {}; + ASSERT_TRUE(reader.get(0, &out0).ok()); + ExpectRefEq(out0, refs[0]); + LogicalIndexRef out1 {}; + ASSERT_TRUE(reader.get(1, &out1).ok()); + ExpectRefEq(out1, refs[1]); +} + +TEST(SniiLogicalIndexDirectory, FramedAsLogicalIndexDirectoryType) { + std::vector refs = { + {.index_id = 1, .index_suffix = "x", .meta_off = 0, .meta_len = 10}}; + auto bytes = Build(refs); + ASSERT_GE(bytes.size(), 1U); + // The first byte is the SectionFramer type and must be kLogicalIndexDirectory. + EXPECT_EQ(bytes[0], static_cast(SectionType::kLogicalIndexDirectory)); +} + +TEST(SniiLogicalIndexDirectory, DetectsCorruption) { + std::vector refs = { + {.index_id = 1, .index_suffix = "fulltext", .meta_off = 0, .meta_len = 4096}, + {.index_id = 2, .index_suffix = "phrase", .meta_off = 4096, .meta_len = 8192}, + }; + auto bytes = Build(refs); + ASSERT_GE(bytes.size(), 4U); + // Flip one byte in the payload region; the section CRC must detect the corruption. + bytes[3] ^= 0xFF; + LogicalIndexDirectoryReader reader; + EXPECT_TRUE(LogicalIndexDirectoryReader::open(Slice(bytes), &reader) + .is()); +} + +TEST(SniiLogicalIndexDirectory, DetectsTruncation) { + std::vector refs = { + {.index_id = 1, .index_suffix = "fulltext", .meta_off = 0, .meta_len = 4096}}; + auto bytes = Build(refs); + bytes.pop_back(); // truncate the last byte + LogicalIndexDirectoryReader reader; + EXPECT_FALSE(LogicalIndexDirectoryReader::open(Slice(bytes), &reader).ok()); +} + +TEST(SniiLogicalIndexDirectory, WrongSectionTypeRejected) { + // A section with a type other than kLogicalIndexDirectory must be rejected. + ByteSink sink; + const uint8_t p[] = {0, 1, 2}; + SectionFramer::write(sink, static_cast(SectionType::kXFilter), Slice(p, 3)); + auto bytes = sink.buffer(); + LogicalIndexDirectoryReader reader; + EXPECT_TRUE(LogicalIndexDirectoryReader::open(Slice(bytes), &reader) + .is()); +} + +TEST(SniiLogicalIndexDirectory, TrailingBytesRejected) { + // Extra trailing bytes after the declared entries must be detected. + ByteSink payload; + payload.put_varint32(1); // n_entries = 1 + payload.put_varint64(1); // index_id + payload.put_varint32(1); // suffix_len + payload.put_u8('a'); // suffix bytes + payload.put_varint64(0); // meta_off + payload.put_varint64(10); // meta_len + payload.put_u8(0xEE); // extra trailing byte + ByteSink sink; + SectionFramer::write(sink, static_cast(SectionType::kLogicalIndexDirectory), + payload.view()); + LogicalIndexDirectoryReader reader; + EXPECT_TRUE(LogicalIndexDirectoryReader::open(Slice(sink.buffer()), &reader) + .is()); +} + +TEST(SniiLogicalIndexDirectory, OversizedNEntriesRejected) { + // n_entries declares far more entries than the remaining payload can hold. + ByteSink payload; + payload.put_varint32(0xFFFFFFFFU); // absurd n_entries + payload.put_varint64(1); // a tiny bit of real data + ByteSink sink; + SectionFramer::write(sink, static_cast(SectionType::kLogicalIndexDirectory), + payload.view()); + LogicalIndexDirectoryReader reader; + EXPECT_TRUE(LogicalIndexDirectoryReader::open(Slice(sink.buffer()), &reader) + .is()); +} + +TEST(SniiLogicalIndexDirectory, OversizedSuffixLenRejected) { + // suffix_len declares more bytes than the payload contains. + ByteSink payload; + payload.put_varint32(1); // n_entries = 1 + payload.put_varint64(1); // index_id + payload.put_varint32(0xFFFFFFFFU); // absurd suffix_len + payload.put_u8('a'); // only 1 byte present + ByteSink sink; + SectionFramer::write(sink, static_cast(SectionType::kLogicalIndexDirectory), + payload.view()); + LogicalIndexDirectoryReader reader; + EXPECT_TRUE(LogicalIndexDirectoryReader::open(Slice(sink.buffer()), &reader) + .is()); +} diff --git a/be/test/storage/index/snii/format/norms_pod_test.cpp b/be/test/storage/index/snii/format/norms_pod_test.cpp new file mode 100644 index 00000000000000..1de3cfeb33cddd --- /dev/null +++ b/be/test/storage/index/snii/format/norms_pod_test.cpp @@ -0,0 +1,170 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/norms_pod.h" + +#include + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/section_framer.h" +#include "storage/index/snii/format/format_constants.h" + +using namespace doris::snii; +using doris::Status; // RETURN_IF_ERROR expands to bare Status +using doris::snii::format::NormsPodReader; +using doris::snii::format::NormsPodWriter; + +namespace { + +// Use writer to encode a sequence of encoded_norms into a framed payload and return the buffer. +std::vector BuildPod(const std::vector& norms) { + NormsPodWriter writer; + for (uint8_t n : norms) { + writer.add(n); + } + ByteSink sink; + writer.finish(&sink); + return sink.buffer(); +} + +} // namespace + +// After writing N norms, read them back per-doc and verify they match. +TEST(SniiNormsPod, RoundTripValues) { + std::vector norms = {0, 1, 7, 42, 128, 200, 255}; + NormsPodWriter writer; + for (uint8_t n : norms) { + writer.add(n); + } + EXPECT_EQ(writer.count(), norms.size()); + + ByteSink sink; + writer.finish(&sink); + + NormsPodReader reader; + ASSERT_TRUE(NormsPodReader::open(sink.view(), &reader).ok()); + ASSERT_EQ(reader.doc_count(), norms.size()); + for (uint32_t docid = 0; docid < norms.size(); ++docid) { + EXPECT_EQ(reader.encoded_norm(docid), norms[docid]) << "docid=" << docid; + } +} + +// Large-scale round-trip covering the multi-byte varint doc_count path. +TEST(SniiNormsPod, RoundTripLarge) { + std::vector norms; + norms.reserve(5000); + for (uint32_t i = 0; i < 5000; ++i) { + norms.push_back(static_cast((i * 31 + 7) & 0xFF)); + } + auto buf = BuildPod(norms); + + NormsPodReader reader; + ASSERT_TRUE(NormsPodReader::open(Slice(buf), &reader).ok()); + ASSERT_EQ(reader.doc_count(), 5000U); + for (uint32_t docid = 0; docid < 5000; ++docid) { + EXPECT_EQ(reader.encoded_norm(docid), norms[docid]) << "docid=" << docid; + } +} + +// Empty POD: count = 0 is valid and open should succeed. +TEST(SniiNormsPod, EmptyPod) { + NormsPodWriter writer; + EXPECT_EQ(writer.count(), 0U); + + ByteSink sink; + writer.finish(&sink); + + NormsPodReader reader; + ASSERT_TRUE(NormsPodReader::open(sink.view(), &reader).ok()); + EXPECT_EQ(reader.doc_count(), 0U); +} + +// CRC corruption is detectable: flipping a byte in the payload causes open to fail. +// The integrated reader frames via SectionFramer, which reports a CRC mismatch as +// INVERTED_INDEX_FILE_CORRUPTED (the standalone test's generic Corruption code was +// replaced during integration with this inverted-index-specific code). +TEST(SniiNormsPod, DetectsCorruption) { + std::vector norms = {10, 20, 30, 40, 50}; + auto buf = BuildPod(norms); + // Flip a byte near the end so the framer CRC no longer matches the payload. + buf[buf.size() - 3] ^= 0xFF; + + NormsPodReader reader; + Status s = NormsPodReader::open(Slice(buf), &reader); + EXPECT_TRUE(s.is()) << s.to_string(); +} + +// Truncated input should return an error rather than crash. +TEST(SniiNormsPod, DetectsTruncation) { + std::vector norms = {1, 2, 3, 4, 5, 6, 7, 8}; + auto buf = BuildPod(norms); + buf.resize(buf.size() - 4); // Chop off the trailing CRC region. + + NormsPodReader reader; + Status s = NormsPodReader::open(Slice(buf), &reader); + EXPECT_FALSE(s.ok()); +} + +// A mismatch between the declared doc_count and the actual payload byte count should be detected. +// The integrated reader reports this as INVERTED_INDEX_FILE_CORRUPTED. +TEST(SniiNormsPod, DetectsLengthMismatch) { + // Manually construct: framer payload = [varint doc_count=4][only 2 norm bytes]. + ByteSink payload; + payload.put_varint64(4); + payload.put_u8(11); + payload.put_u8(22); + + ByteSink sink; + // Reuse the framer to ensure a self-consistent CRC, specifically to trigger the length-mismatch branch. + doris::snii::SectionFramer::write( + sink, static_cast(doris::snii::format::SectionType::kStatsBlock), + payload.view()); + + NormsPodReader reader; + Status s = NormsPodReader::open(sink.view(), &reader); + EXPECT_TRUE(s.is()) << s.to_string(); +} + +#ifndef NDEBUG +// In a debug build, an out-of-range docid triggers an assertion (death test). +TEST(SniiNormsPodDeathTest, OutOfRangeDocidAsserts) { + std::vector norms = {3, 6, 9}; + auto buf = BuildPod(norms); + NormsPodReader reader; + ASSERT_TRUE(NormsPodReader::open(Slice(buf), &reader).ok()); + EXPECT_DEATH({ (void)reader.encoded_norm(3); }, ""); +} +#endif + +// Checked access: a valid docid returns the value, an out-of-range docid returns +// InvalidArgument (also effective in Release builds). +TEST(SniiNormsPod, TryEncodedNormChecksBounds) { + std::vector norms = {3, 6, 9}; + auto buf = BuildPod(norms); + NormsPodReader reader; + ASSERT_TRUE(NormsPodReader::open(Slice(buf), &reader).ok()); + uint8_t v = 0; + ASSERT_TRUE(reader.try_encoded_norm(1, &v).ok()); + EXPECT_EQ(v, 6U); + Status s = reader.try_encoded_norm(3, &v); + EXPECT_TRUE(s.is()) << s.to_string(); +} diff --git a/be/test/storage/index/snii/format/null_bitmap_test.cpp b/be/test/storage/index/snii/format/null_bitmap_test.cpp new file mode 100644 index 00000000000000..25cfe9a283d581 --- /dev/null +++ b/be/test/storage/index/snii/format/null_bitmap_test.cpp @@ -0,0 +1,202 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/null_bitmap.h" + +#include + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/section_framer.h" + +using namespace doris::snii; +using doris::Status; +using doris::snii::format::NullBitmapReader; +using doris::snii::format::NullBitmapWriter; +using doris::snii::format::kNullBitmapSectionType; + +namespace { + +// Encode a set of null docids into a framed buffer using the writer. +std::vector BuildBitmap(const std::vector& nulls, uint32_t doc_count) { + NullBitmapWriter writer; + for (uint32_t d : nulls) { + writer.add_null(d); + } + ByteSink sink; + writer.finish(doc_count, &sink); + return sink.buffer(); +} + +} // namespace + +// After adding nulls, is_null(docid) must match the input set for every doc. +TEST(SniiNullBitmap, RoundTripPerDoc) { + std::vector nulls = {0, 3, 7, 11, 100, 4000}; + uint32_t doc_count = 5000; + auto buf = BuildBitmap(nulls, doc_count); + + NullBitmapReader reader; + ASSERT_TRUE(NullBitmapReader::open(Slice(buf), &reader).ok()); + EXPECT_EQ(reader.doc_count(), doc_count); + EXPECT_EQ(reader.null_count(), nulls.size()); + + std::vector expected(doc_count, false); + for (uint32_t d : nulls) { + expected[d] = true; + } + for (uint32_t docid = 0; docid < doc_count; ++docid) { + EXPECT_EQ(reader.is_null(docid), expected[docid]) << "docid=" << docid; + } +} + +// Writer null_count reflects the number of distinct null docids added. +TEST(SniiNullBitmap, WriterNullCount) { + NullBitmapWriter writer; + EXPECT_EQ(writer.null_count(), 0U); + writer.add_null(5); + writer.add_null(9); + writer.add_null(5); // duplicate is idempotent in a set + EXPECT_EQ(writer.null_count(), 2U); +} + +// Empty bitmap: no nulls. open succeeds, null_count == 0, nothing is null. +TEST(SniiNullBitmap, EmptyNoNulls) { + auto buf = BuildBitmap({}, 1000); + + NullBitmapReader reader; + ASSERT_TRUE(NullBitmapReader::open(Slice(buf), &reader).ok()); + EXPECT_EQ(reader.doc_count(), 1000U); + EXPECT_EQ(reader.null_count(), 0U); + EXPECT_FALSE(reader.is_null(0)); + EXPECT_FALSE(reader.is_null(999)); +} + +// All-null bitmap: every doc in [0, doc_count) is null. +TEST(SniiNullBitmap, AllNull) { + uint32_t doc_count = 256; + std::vector nulls; + for (uint32_t d = 0; d < doc_count; ++d) { + nulls.push_back(d); + } + auto buf = BuildBitmap(nulls, doc_count); + + NullBitmapReader reader; + ASSERT_TRUE(NullBitmapReader::open(Slice(buf), &reader).ok()); + EXPECT_EQ(reader.null_count(), doc_count); + for (uint32_t docid = 0; docid < doc_count; ++docid) { + EXPECT_TRUE(reader.is_null(docid)) << "docid=" << docid; + } +} + +// doc_count round-trips even when there are no nulls and a large doc_count. +TEST(SniiNullBitmap, DocCountRoundTrips) { + auto buf = BuildBitmap({42}, 1234567); + + NullBitmapReader reader; + ASSERT_TRUE(NullBitmapReader::open(Slice(buf), &reader).ok()); + EXPECT_EQ(reader.doc_count(), 1234567U); + EXPECT_TRUE(reader.is_null(42)); +} + +// is_null beyond doc_count is false (docid not in the null set). +TEST(SniiNullBitmap, IsNullOutsideRangeIsFalse) { + auto buf = BuildBitmap({1, 2, 3}, 10); + + NullBitmapReader reader; + ASSERT_TRUE(NullBitmapReader::open(Slice(buf), &reader).ok()); + EXPECT_FALSE(reader.is_null(10)); + EXPECT_FALSE(reader.is_null(1000000)); +} + +// CRC corruption is detectable: flipping a payload byte fails open with a +// corruption error (SectionFramer stamps the crc over type+len+payload, so a +// flipped payload byte makes the recomputed crc disagree with the stored one). +TEST(SniiNullBitmap, DetectsCorruption) { + std::vector nulls = {2, 4, 6, 8, 10, 12, 14}; + auto buf = BuildBitmap(nulls, 100); + // Flip a byte inside the roaring payload region (near the end, before the trailing CRC). + buf[buf.size() - 5] ^= 0xFF; + + NullBitmapReader reader; + Status s = NullBitmapReader::open(Slice(buf), &reader); + EXPECT_TRUE(s.is()); +} + +// Truncated input returns an error rather than crashing. +TEST(SniiNullBitmap, DetectsTruncation) { + auto buf = BuildBitmap({1, 2, 3, 4, 5}, 100); + buf.resize(buf.size() - 4); // chop trailing CRC region + + NullBitmapReader reader; + Status s = NullBitmapReader::open(Slice(buf), &reader); + EXPECT_FALSE(s.ok()); +} + +// An oversized declared roaring_size (larger than the remaining payload bytes) is rejected (anti-DoS). +TEST(SniiNullBitmap, RejectsOversizedRoaringSize) { + // Manually construct a self-consistent frame whose declared roaring_size + // exceeds the bytes actually present, to drive the guard branch. + ByteSink payload; + payload.put_varint64(100); // doc_count + payload.put_varint64(0xFFFFFFFFULL); // roaring_size: absurdly large + payload.put_u8(0x00); // only 1 byte of roaring data present + + ByteSink sink; + SectionFramer::write(sink, kNullBitmapSectionType, payload.view()); + + NullBitmapReader reader; + Status s = NullBitmapReader::open(sink.view(), &reader); + EXPECT_TRUE(s.is()); +} + +// doc_count overflowing uint32 is rejected. +TEST(SniiNullBitmap, RejectsDocCountOverflow) { + ByteSink payload; + payload.put_varint64(0x1'0000'0000ULL); // doc_count > uint32 max + payload.put_varint64(0); // roaring_size + + ByteSink sink; + SectionFramer::write(sink, kNullBitmapSectionType, payload.view()); + + NullBitmapReader reader; + Status s = NullBitmapReader::open(sink.view(), &reader); + EXPECT_TRUE(s.is()); +} + +// A CRC-valid frame carrying malformed roaring container bytes must be rejected +// gracefully (corruption error) without throwing or aborting. The roaring bytes are +// not a valid portable serialization; SectionFramer stamps a correct crc so the framer +// check passes and the roaring pre-validation (deserialize_size probe) must catch it. +TEST(SniiNullBitmap, RejectsMalformedRoaringContainer) { + ByteSink payload; + payload.put_varint64(10); // doc_count + payload.put_varint64(4); // roaring_size + const uint8_t garbage[] = {0xFF, 0xFF, 0xFF, 0xFF}; // invalid roaring cookie + payload.put_bytes(Slice(garbage, 4)); + + ByteSink sink; + SectionFramer::write(sink, kNullBitmapSectionType, payload.view()); + + NullBitmapReader reader; + Status s = NullBitmapReader::open(sink.view(), &reader); + EXPECT_TRUE(s.is()); +} diff --git a/be/test/storage/index/snii/format/per_index_meta_test.cpp b/be/test/storage/index/snii/format/per_index_meta_test.cpp new file mode 100644 index 00000000000000..1e6ad6409db5d0 --- /dev/null +++ b/be/test/storage/index/snii/format/per_index_meta_test.cpp @@ -0,0 +1,738 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/per_index_meta.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/section_framer.h" +#include "storage/index/snii/encoding/zstd_codec.h" +#include "storage/index/snii/format/dict_block_directory.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/sampled_term_index.h" +#include "storage/index/snii/format/stats_block.h" +#include "storage/index/snii_query_test_util.h" + +using namespace doris::snii; +using namespace doris::snii::format; +using doris::snii::snii_test::ScopedEnv; + +// Env value that disables G13 meta-section compression (threshold == SIZE_MAX). +static const char* const kCompressOff = "18446744073709551615"; + +namespace { + +// Produces the raw framed bytes of a SampledTermIndex from a list of first_terms. +std::vector BuildSampled(const std::vector& terms) { + SampledTermIndexBuilder b; + for (const auto& t : terms) { + b.add_block_first_term(t); + } + ByteSink sink; + b.finish(&sink); + return sink.buffer(); +} + +// Produces the raw framed bytes of a DICT block directory from a list of refs. +std::vector BuildDict(const std::vector& refs) { + DictBlockDirectoryBuilder b; + for (const auto& r : refs) { + b.add(r); + } + ByteSink sink; + b.finish(&sink); + return sink.buffer(); +} + +StatsBlock SampleStats() { + StatsBlock sb; + sb.doc_count = 1000; + sb.indexed_doc_count = 950; + sb.term_count = 4242; + sb.sum_total_term_freq = 1234567; + sb.null_count = 50; + return sb; +} + +SectionRefs SampleRefs() { + SectionRefs r; + r.dict_region = {.offset = 4096, .length = 65536}; + r.posting_region = {.offset = 70000, .length = 24576}; // single interleaved [prx][frq] region + r.norms = {.offset = 100000, .length = 4096}; + r.null_bitmap = {.offset = 0, .length = 0}; // absent + r.bsbf = {.offset = 120000, .length = 32768}; + return r; +} + +void ExpectRegionEq(const RegionRef& a, const RegionRef& b) { + EXPECT_EQ(a.offset, b.offset); + EXPECT_EQ(a.length, b.length); +} + +void ExpectRefsEq(const SectionRefs& a, const SectionRefs& b) { + ExpectRegionEq(a.dict_region, b.dict_region); + ExpectRegionEq(a.posting_region, b.posting_region); + ExpectRegionEq(a.norms, b.norms); + ExpectRegionEq(a.null_bitmap, b.null_bitmap); + ExpectRegionEq(a.bsbf, b.bsbf); +} + +void ExpectStatsEq(const StatsBlock& a, const StatsBlock& b) { + EXPECT_EQ(a.doc_count, b.doc_count); + EXPECT_EQ(a.indexed_doc_count, b.indexed_doc_count); + EXPECT_EQ(a.term_count, b.term_count); + EXPECT_EQ(a.sum_total_term_freq, b.sum_total_term_freq); + EXPECT_EQ(a.null_count, b.null_count); +} + +// Builds a full per-index meta block (SectionRefs carry the bsbf section ref). +std::vector BuildMeta(uint64_t index_id, std::string suffix, + const std::vector& sample_terms, + const std::vector& dict_refs) { + PerIndexMetaBuilder builder(index_id, std::move(suffix), PerIndexMetaBuilder::kHasBsbf); + builder.set_stats(SampleStats()); + builder.set_sampled_term_index(Slice(BuildSampled(sample_terms))); + builder.set_dict_block_directory(Slice(BuildDict(dict_refs))); + builder.set_section_refs(SampleRefs()); + ByteSink sink; + static_cast(builder.finish(&sink)); + return sink.buffer(); +} + +} // namespace + +TEST(SniiPerIndexMeta, RoundTrip) { + std::vector sample_terms = {"alpha", "kappa", "zeta"}; + std::vector dict_refs = { + {.offset = 4096, .length = 1024, .n_entries = 10, .flags = 0, .checksum = 0x11111111U}, + {.offset = 5120, .length = 2048, .n_entries = 20, .flags = 1, .checksum = 0x22222222U}, + {.offset = 7168, .length = 512, .n_entries = 5, .flags = 0, .checksum = 0x33333333U}, + }; + auto bytes = BuildMeta(7, "title", sample_terms, dict_refs); + + PerIndexMetaReader reader; + ASSERT_TRUE(PerIndexMetaReader::open(Slice(bytes), &reader).ok()); + + EXPECT_EQ(reader.index_id(), 7U); + EXPECT_EQ(reader.index_suffix(), "title"); + // The bsbf XFilter is a physical section ref, not embedded in the meta. + EXPECT_TRUE(reader.has_bsbf()); + EXPECT_NE(reader.flags() & PerIndexMetaBuilder::kHasBsbf, 0U); + + ExpectStatsEq(reader.stats(), SampleStats()); + ExpectRefsEq(reader.section_refs(), SampleRefs()); + + // The materialized sub-section frames must be openable by their own readers. + // Small sections stay raw, so the frames alias the block and leave the + // scratch untouched. + std::vector sti_scratch; + Slice sti_frame; + ASSERT_TRUE(reader.sampled_term_index_frame(&sti_scratch, &sti_frame).ok()); + EXPECT_TRUE(sti_scratch.empty()); + SampledTermIndexReader sti; + ASSERT_TRUE(SampledTermIndexReader::open(sti_frame, &sti).ok()); + EXPECT_EQ(sti.n_blocks(), 3U); + bool maybe = false; + uint32_t ord = 0; + ASSERT_TRUE(sti.locate("kappa", &maybe, &ord).ok()); + EXPECT_TRUE(maybe); + EXPECT_EQ(ord, 1U); + + std::vector dbd_scratch; + Slice dbd_frame; + ASSERT_TRUE(reader.dict_block_directory_frame(&dbd_scratch, &dbd_frame).ok()); + EXPECT_TRUE(dbd_scratch.empty()); + DictBlockDirectoryReader dbd; + ASSERT_TRUE(DictBlockDirectoryReader::open(dbd_frame, &dbd).ok()); + ASSERT_EQ(dbd.n_blocks(), 3U); + BlockRef got {}; + ASSERT_TRUE(dbd.get(1, &got).ok()); + EXPECT_EQ(got.offset, 5120U); + EXPECT_EQ(got.checksum, 0x22222222U); +} + +TEST(SniiPerIndexMeta, HasPositionsFlagRoundTrips) { + // The kHasPositions header bit must survive a build/open round-trip and be read + // back via the convenience accessor (capability lives in the flag, not a length). + PerIndexMetaBuilder builder(9, "body", + PerIndexMetaBuilder::kHasPositions | PerIndexMetaBuilder::kHasBsbf); + builder.set_stats(SampleStats()); + builder.set_sampled_term_index(Slice(BuildSampled({"a"}))); + builder.set_dict_block_directory(Slice( + BuildDict({{.offset = 0, .length = 1, .n_entries = 1, .flags = 0, .checksum = 0}}))); + builder.set_section_refs(SampleRefs()); + ByteSink sink; + ASSERT_TRUE(builder.finish(&sink).ok()); + + PerIndexMetaReader reader; + ASSERT_TRUE(PerIndexMetaReader::open(Slice(sink.buffer()), &reader).ok()); + EXPECT_TRUE(reader.has_positions()); + EXPECT_TRUE(reader.has_bsbf()); + EXPECT_NE(reader.flags() & PerIndexMetaBuilder::kHasPositions, 0U); + + // A docs-only meta (flag clear) reports has_positions() == false even though its + // posting_region length is non-zero -- capability never comes from a region length. + PerIndexMetaBuilder docs_only(10, "tag", 0U); + docs_only.set_stats(SampleStats()); + docs_only.set_sampled_term_index(Slice(BuildSampled({"a"}))); + docs_only.set_dict_block_directory(Slice( + BuildDict({{.offset = 0, .length = 1, .n_entries = 1, .flags = 0, .checksum = 0}}))); + docs_only.set_section_refs(SampleRefs()); // posting_region.length > 0 + ByteSink docs_sink; + ASSERT_TRUE(docs_only.finish(&docs_sink).ok()); + PerIndexMetaReader docs_reader; + ASSERT_TRUE(PerIndexMetaReader::open(Slice(docs_sink.buffer()), &docs_reader).ok()); + EXPECT_FALSE(docs_reader.has_positions()); + EXPECT_GT(docs_reader.section_refs().posting_region.length, 0U); +} + +TEST(SniiPerIndexMeta, PhraseBigramsDeferredFlagRoundTrips) { + PerIndexMetaBuilder builder( + 11, "body", + PerIndexMetaBuilder::kHasPositions | PerIndexMetaBuilder::kPhraseBigramsDeferred); + builder.set_stats(SampleStats()); + builder.set_sampled_term_index(Slice(BuildSampled({"a"}))); + builder.set_dict_block_directory(Slice( + BuildDict({{.offset = 0, .length = 1, .n_entries = 1, .flags = 0, .checksum = 0}}))); + builder.set_section_refs(SampleRefs()); + ByteSink sink; + ASSERT_TRUE(builder.finish(&sink).ok()); + + PerIndexMetaReader reader; + ASSERT_TRUE(PerIndexMetaReader::open(Slice(sink.buffer()), &reader).ok()); + EXPECT_TRUE(reader.has_positions()); + EXPECT_TRUE(reader.phrase_bigrams_deferred()); + EXPECT_NE(reader.flags() & PerIndexMetaBuilder::kPhraseBigramsDeferred, 0U); + + PerIndexMetaBuilder legacy(12, "body", PerIndexMetaBuilder::kHasPositions); + legacy.set_stats(SampleStats()); + legacy.set_sampled_term_index(Slice(BuildSampled({"a"}))); + legacy.set_dict_block_directory(Slice( + BuildDict({{.offset = 0, .length = 1, .n_entries = 1, .flags = 0, .checksum = 0}}))); + legacy.set_section_refs(SampleRefs()); + ByteSink legacy_sink; + ASSERT_TRUE(legacy.finish(&legacy_sink).ok()); + PerIndexMetaReader legacy_reader; + ASSERT_TRUE(PerIndexMetaReader::open(Slice(legacy_sink.buffer()), &legacy_reader).ok()); + EXPECT_FALSE(legacy_reader.phrase_bigrams_deferred()); +} + +TEST(SniiPerIndexMeta, EmptySuffix) { + auto bytes = BuildMeta(1, "", {"a"}, + {{.offset = 0, .length = 1, .n_entries = 1, .flags = 0, .checksum = 0}}); + PerIndexMetaReader reader; + ASSERT_TRUE(PerIndexMetaReader::open(Slice(bytes), &reader).ok()); + EXPECT_EQ(reader.index_id(), 1U); + EXPECT_TRUE(reader.index_suffix().empty()); +} + +TEST(SniiPerIndexMeta, BigramPruneMinDfRoundTripsAndDefaultsToZero) { + // G01: absent kBigramPruneInfo section (the builder default AND every legacy + // segment) reads back as 0 == "not pruned, legacy semantics". + auto legacy_bytes = + BuildMeta(1, "body", {"a"}, + {{.offset = 0, .length = 1, .n_entries = 1, .flags = 0, .checksum = 0}}); + PerIndexMetaReader legacy; + ASSERT_TRUE(PerIndexMetaReader::open(Slice(legacy_bytes), &legacy).ok()); + EXPECT_EQ(legacy.bigram_prune_min_df(), 0U); + EXPECT_EQ(legacy.bigram_prune_max_df(), 0U); // absent section: BOTH gates inactive + + // A non-zero threshold emits the optional framed section and round-trips + // exactly, without disturbing the required sub-sections. + PerIndexMetaBuilder builder(2, "body", PerIndexMetaBuilder::kHasPositions); + builder.set_stats(SampleStats()); + builder.set_sampled_term_index(Slice(BuildSampled({"a"}))); + builder.set_dict_block_directory(Slice( + BuildDict({{.offset = 0, .length = 1, .n_entries = 1, .flags = 0, .checksum = 0}}))); + builder.set_section_refs(SampleRefs()); + builder.set_bigram_prune_min_df(640); + ByteSink sink; + ASSERT_TRUE(builder.finish(&sink).ok()); + + PerIndexMetaReader pruned; + ASSERT_TRUE(PerIndexMetaReader::open(Slice(sink.buffer()), &pruned).ok()); + EXPECT_EQ(pruned.bigram_prune_min_df(), 640U); + // (min>0, max=0): a post-G15 writer with the upper gate disabled encodes an + // explicit 0 max varint; it must decode back as "max gate inactive". + EXPECT_EQ(pruned.bigram_prune_max_df(), 0U); + EXPECT_TRUE(pruned.has_positions()); + ExpectStatsEq(pruned.stats(), SampleStats()); + ExpectRefsEq(pruned.section_refs(), SampleRefs()); +} + +TEST(SniiPerIndexMeta, BigramPruneMaxDfRoundTripsAndPreG15PayloadDefaultsToZero) { + // G15: both thresholds round-trip through the (single) kBigramPruneInfo + // section; a max-only declaration must still emit the section (the reader's + // dict-miss fallback keys on EITHER gate being non-zero). + PerIndexMetaBuilder builder(2, "body", PerIndexMetaBuilder::kHasPositions); + builder.set_stats(SampleStats()); + builder.set_sampled_term_index(Slice(BuildSampled({"a"}))); + builder.set_dict_block_directory(Slice( + BuildDict({{.offset = 0, .length = 1, .n_entries = 1, .flags = 0, .checksum = 0}}))); + builder.set_section_refs(SampleRefs()); + builder.set_bigram_prune_min_df(640); + builder.set_bigram_prune_max_df(1800); + ByteSink sink; + ASSERT_TRUE(builder.finish(&sink).ok()); + PerIndexMetaReader both; + ASSERT_TRUE(PerIndexMetaReader::open(Slice(sink.buffer()), &both).ok()); + EXPECT_EQ(both.bigram_prune_min_df(), 640U); + EXPECT_EQ(both.bigram_prune_max_df(), 1800U); + + PerIndexMetaBuilder max_only_builder(3, "body", PerIndexMetaBuilder::kHasPositions); + max_only_builder.set_stats(SampleStats()); + max_only_builder.set_sampled_term_index(Slice(BuildSampled({"a"}))); + max_only_builder.set_dict_block_directory(Slice( + BuildDict({{.offset = 0, .length = 1, .n_entries = 1, .flags = 0, .checksum = 0}}))); + max_only_builder.set_section_refs(SampleRefs()); + max_only_builder.set_bigram_prune_max_df(1800); + ByteSink max_only_sink; + ASSERT_TRUE(max_only_builder.finish(&max_only_sink).ok()); + PerIndexMetaReader max_only; + ASSERT_TRUE(PerIndexMetaReader::open(Slice(max_only_sink.buffer()), &max_only).ok()); + EXPECT_EQ(max_only.bigram_prune_min_df(), 0U); + EXPECT_EQ(max_only.bigram_prune_max_df(), 1800U); + + // Back-compat: a pre-G15 (G01..G13) writer framed ONLY the min varint. + // Splice such a section in verbatim via add_raw_section; decode must read + // the min and default the absent max to 0 instead of erroring. + PerIndexMetaBuilder legacy_builder(4, "body", PerIndexMetaBuilder::kHasPositions); + legacy_builder.set_stats(SampleStats()); + legacy_builder.set_sampled_term_index(Slice(BuildSampled({"a"}))); + legacy_builder.set_dict_block_directory(Slice( + BuildDict({{.offset = 0, .length = 1, .n_entries = 1, .flags = 0, .checksum = 0}}))); + legacy_builder.set_section_refs(SampleRefs()); + ByteSink old_payload; + old_payload.put_varint64(640); + ByteSink old_frame; + SectionFramer::write(old_frame, static_cast(SectionType::kBigramPruneInfo), + Slice(old_payload.buffer())); + legacy_builder.add_raw_section(Slice(old_frame.buffer())); + ByteSink legacy_sink; + ASSERT_TRUE(legacy_builder.finish(&legacy_sink).ok()); + PerIndexMetaReader legacy; + ASSERT_TRUE(PerIndexMetaReader::open(Slice(legacy_sink.buffer()), &legacy).ok()); + EXPECT_EQ(legacy.bigram_prune_min_df(), 640U); + EXPECT_EQ(legacy.bigram_prune_max_df(), 0U); +} + +TEST(SniiPerIndexMeta, HeaderStartsWithMetaFormatVersion) { + auto bytes = BuildMeta(7, "x", {"a"}, + {{.offset = 0, .length = 1, .n_entries = 1, .flags = 0, .checksum = 0}}); + ASSERT_GE(bytes.size(), 2U); + // u16 meta_format_version, little-endian, is the first field. + ByteSource src {Slice(bytes)}; + uint16_t ver = 0; + ASSERT_TRUE(src.get_fixed16(&ver).ok()); + EXPECT_EQ(ver, kMetaFormatVersion); +} + +TEST(SniiPerIndexMeta, HeaderCrcCorruptionDetected) { + auto bytes = BuildMeta(7, "title", {"a", "b"}, + {{.offset = 0, .length = 1, .n_entries = 1, .flags = 0, .checksum = 0}}); + ASSERT_GE(bytes.size(), 3U); + // Flip a byte inside the header (index_id varint region after the u16 version). + bytes[3] ^= 0xFF; + PerIndexMetaReader reader; + EXPECT_TRUE(PerIndexMetaReader::open(Slice(bytes), &reader) + .is()); +} + +TEST(SniiPerIndexMeta, SubSectionCrcCorruptionDetected) { + std::vector sample_terms = {"alpha", "kappa"}; + std::vector dict_refs = { + {.offset = 4096, .length = 1024, .n_entries = 10, .flags = 0, .checksum = 0x11111111U}}; + auto bytes = BuildMeta(7, "title", sample_terms, dict_refs); + // Corrupt a byte deep in the block (well past the header, inside a framed + // sub-section payload). The framer CRC of that sub-section must catch it. + ASSERT_GT(bytes.size(), 40U); + bytes[bytes.size() - 10] ^= 0xFF; + PerIndexMetaReader reader; + EXPECT_TRUE(PerIndexMetaReader::open(Slice(bytes), &reader) + .is()); +} + +TEST(SniiPerIndexMeta, UnknownOptionalSectionSkipped) { + // Build a normal meta block, then splice an unknown-type framed section in + // front of the SectionRefs by rebuilding manually: header + stats + sampled + + // dict + unknown + section_refs. The reader must skip the unknown section and + // still expose all the known ones. + PerIndexMetaBuilder builder(7, "title", 0); + builder.set_stats(SampleStats()); + builder.set_sampled_term_index(Slice(BuildSampled({"a", "b"}))); + builder.set_dict_block_directory(Slice( + BuildDict({{.offset = 0, .length = 1, .n_entries = 1, .flags = 0, .checksum = 0}}))); + builder.set_section_refs(SampleRefs()); + + // Inject an extra framed section with an unrecognized type id (200). + const uint8_t junk[] = {0xDE, 0xAD, 0xBE, 0xEF}; + ByteSink extra; + SectionFramer::write(extra, 200, Slice(junk, sizeof(junk))); + builder.add_raw_section(extra.view()); + + ByteSink sink; + ASSERT_TRUE(builder.finish(&sink).ok()); + auto bytes = sink.buffer(); + + PerIndexMetaReader reader; + ASSERT_TRUE(PerIndexMetaReader::open(Slice(bytes), &reader).ok()); + ExpectStatsEq(reader.stats(), SampleStats()); + ExpectRefsEq(reader.section_refs(), SampleRefs()); + std::vector sti_scratch; + Slice sti_frame; + ASSERT_TRUE(reader.sampled_term_index_frame(&sti_scratch, &sti_frame).ok()); + SampledTermIndexReader sti; + ASSERT_TRUE(SampledTermIndexReader::open(sti_frame, &sti).ok()); + EXPECT_EQ(sti.n_blocks(), 2U); + std::vector dbd_scratch; + Slice dbd_frame; + ASSERT_TRUE(reader.dict_block_directory_frame(&dbd_scratch, &dbd_frame).ok()); + DictBlockDirectoryReader dbd; + ASSERT_TRUE(DictBlockDirectoryReader::open(dbd_frame, &dbd).ok()); + EXPECT_EQ(dbd.n_blocks(), 1U); +} + +TEST(SniiPerIndexMeta, TruncatedHeaderRejected) { + auto bytes = BuildMeta(7, "title", {"a"}, + {{.offset = 0, .length = 1, .n_entries = 1, .flags = 0, .checksum = 0}}); + // Keep only one byte: cannot even read the u16 version. + std::vector truncated(bytes.begin(), bytes.begin() + 1); + PerIndexMetaReader reader; + EXPECT_FALSE(PerIndexMetaReader::open(Slice(truncated), &reader).ok()); +} + +TEST(SniiPerIndexMeta, NullSinkRejectedByFinish) { + PerIndexMetaBuilder builder(1, "x", 0); + builder.set_stats(SampleStats()); + builder.set_sampled_term_index(Slice(BuildSampled({"a"}))); + builder.set_dict_block_directory(Slice( + BuildDict({{.offset = 0, .length = 1, .n_entries = 1, .flags = 0, .checksum = 0}}))); + builder.set_section_refs(SampleRefs()); + EXPECT_TRUE(builder.finish(nullptr).is()); +} + +TEST(SniiPerIndexMeta, OpenNullReaderRejected) { + auto bytes = BuildMeta(1, "x", {"a"}, + {{.offset = 0, .length = 1, .n_entries = 1, .flags = 0, .checksum = 0}}); + EXPECT_TRUE(PerIndexMetaReader::open(Slice(bytes), nullptr) + .is()); +} + +// ---- G13: zstd-compressed embedded sti/dbd sections ---- + +namespace { + +// A large ascending vocabulary whose sampled-term frame exceeds the 4KB +// compression threshold. Fixed-width numeric tails keep lexicographic order. +std::vector ManyTerms(uint32_t n) { + std::vector terms; + terms.reserve(n); + for (uint32_t i = 0; i < n; ++i) { + terms.push_back("term_" + std::to_string(1000000 + i)); + } + return terms; +} + +// A large block-ref table whose directory frame exceeds the threshold. +std::vector ManyRefs(uint32_t n) { + std::vector refs; + refs.reserve(n); + for (uint32_t i = 0; i < n; ++i) { + refs.push_back({.offset = 4096 + static_cast(i) * 128, + .length = 128, + .n_entries = 3, + .flags = 0, + .checksum = i}); + } + return refs; +} + +// Walks a per-index meta block and returns the framed section type ids in +// emission order (header fields are consumed structurally, not verified here). +std::vector SectionTypesOf(Slice block) { + ByteSource src(block); + uint16_t ver = 0; + EXPECT_TRUE(src.get_fixed16(&ver).ok()); + uint64_t index_id = 0; + EXPECT_TRUE(src.get_varint64(&index_id).ok()); + uint32_t suffix_len = 0; + EXPECT_TRUE(src.get_varint32(&suffix_len).ok()); + Slice suffix; + EXPECT_TRUE(src.get_bytes(suffix_len, &suffix).ok()); + uint32_t flags = 0; + EXPECT_TRUE(src.get_fixed32(&flags).ok()); + uint32_t crc = 0; + EXPECT_TRUE(src.get_fixed32(&crc).ok()); + std::vector types; + while (!src.eof()) { + FramedSection sec; + EXPECT_TRUE(SectionFramer::read(src, &sec).ok()); + types.push_back(sec.type); + } + return types; +} + +bool ContainsType(const std::vector& types, SectionType t) { + return std::find(types.begin(), types.end(), static_cast(t)) != types.end(); +} + +// Builds a meta block whose sti/dbd sections are corrupt zstd carriers: the +// carrier frame CRC is VALID (SectionFramer computes it over the given payload), +// so only the zstd layer can catch the damage. `payload` = the carrier frame +// payload (varint64 uncomp_len + compressed bytes). +std::vector BuildMetaWithRawStiSection(Slice sti_carrier_payload) { + PerIndexMetaBuilder builder(1, "x", 0); + builder.set_stats(SampleStats()); + builder.set_dict_block_directory(Slice( + BuildDict({{.offset = 0, .length = 1, .n_entries = 1, .flags = 0, .checksum = 0}}))); + builder.set_section_refs(SampleRefs()); + ByteSink carrier; + SectionFramer::write(carrier, static_cast(SectionType::kSampledTermIndexZstd), + sti_carrier_payload); + builder.add_raw_section(carrier.view()); + ByteSink sink; + EXPECT_TRUE(builder.finish(&sink).ok()); + return sink.buffer(); +} + +} // namespace + +// Large sti/dbd frames are emitted as kSampledTermIndexZstd/kDictBlockDirectoryZstd +// carriers, the block shrinks vs the uncompressed control, and the materialized +// frames are BYTE-EXACT copies of the original framed sub-sections. +TEST(SniiPerIndexMetaZstd, LargeSectionsCompressedAndByteRoundTrip) { + const auto terms = ManyTerms(2000); + const auto refs = ManyRefs(2000); + const auto sti_raw = BuildSampled(terms); + const auto dbd_raw = BuildDict(refs); + ASSERT_GT(sti_raw.size(), kMetaSectionCompressMinBytes); + ASSERT_GT(dbd_raw.size(), kMetaSectionCompressMinBytes); + + const auto compressed = BuildMeta(7, "title", terms, refs); + std::vector control; + { + ScopedEnv off("SNII_META_COMPRESS_MIN", kCompressOff); + control = BuildMeta(7, "title", terms, refs); + } + // The compressed layout must be a real shrink over the raw layout. + EXPECT_LT(compressed.size(), control.size()); + std::cout << "[G13] per-index meta block: raw=" << control.size() + << "B zstd=" << compressed.size() << "B (sti_frame=" << sti_raw.size() + << "B dbd_frame=" << dbd_raw.size() << "B)\n"; + + const auto compressed_types = SectionTypesOf(Slice(compressed)); + EXPECT_TRUE(ContainsType(compressed_types, SectionType::kSampledTermIndexZstd)); + EXPECT_TRUE(ContainsType(compressed_types, SectionType::kDictBlockDirectoryZstd)); + EXPECT_FALSE(ContainsType(compressed_types, SectionType::kSampledTermIndex)); + EXPECT_FALSE(ContainsType(compressed_types, SectionType::kDictBlockDirectory)); + const auto control_types = SectionTypesOf(Slice(control)); + EXPECT_TRUE(ContainsType(control_types, SectionType::kSampledTermIndex)); + EXPECT_FALSE(ContainsType(control_types, SectionType::kSampledTermIndexZstd)); + + PerIndexMetaReader reader; + ASSERT_TRUE(PerIndexMetaReader::open(Slice(compressed), &reader).ok()); + std::vector sti_scratch; + Slice sti_frame; + ASSERT_TRUE(reader.sampled_term_index_frame(&sti_scratch, &sti_frame).ok()); + ASSERT_EQ(sti_frame.size(), sti_raw.size()); + EXPECT_EQ(std::memcmp(sti_frame.data(), sti_raw.data(), sti_raw.size()), 0); + std::vector dbd_scratch; + Slice dbd_frame; + ASSERT_TRUE(reader.dict_block_directory_frame(&dbd_scratch, &dbd_frame).ok()); + ASSERT_EQ(dbd_frame.size(), dbd_raw.size()); + EXPECT_EQ(std::memcmp(dbd_frame.data(), dbd_raw.data(), dbd_raw.size()), 0); + + // The materialized frames parse with the unchanged sub-module readers. + SampledTermIndexReader sti; + ASSERT_TRUE(SampledTermIndexReader::open(sti_frame, &sti).ok()); + EXPECT_EQ(sti.n_blocks(), 2000U); + bool maybe = false; + uint32_t ord = 0; + ASSERT_TRUE(sti.locate("term_1001500", &maybe, &ord).ok()); + EXPECT_TRUE(maybe); + EXPECT_EQ(ord, 1500U); + DictBlockDirectoryReader dbd; + ASSERT_TRUE(DictBlockDirectoryReader::open(dbd_frame, &dbd).ok()); + ASSERT_EQ(dbd.n_blocks(), 2000U); + BlockRef got {}; + ASSERT_TRUE(dbd.get(1500, &got).ok()); + EXPECT_EQ(got.offset, 4096U + 1500U * 128U); + EXPECT_EQ(got.checksum, 1500U); +} + +// Old-segment compatibility: a large meta built with compression disabled (the +// pre-G13 layout) still opens and materializes as in-place raw frames. +TEST(SniiPerIndexMetaZstd, UncompressedLargeSectionsStillRead) { + const auto terms = ManyTerms(2000); + const auto refs = ManyRefs(2000); + std::vector legacy; + { + ScopedEnv off("SNII_META_COMPRESS_MIN", kCompressOff); + legacy = BuildMeta(7, "title", terms, refs); + } + PerIndexMetaReader reader; + ASSERT_TRUE(PerIndexMetaReader::open(Slice(legacy), &reader).ok()); + std::vector sti_scratch; + Slice sti_frame; + ASSERT_TRUE(reader.sampled_term_index_frame(&sti_scratch, &sti_frame).ok()); + // Raw frame: a view into the block, no decompression scratch used. + EXPECT_TRUE(sti_scratch.empty()); + EXPECT_GE(sti_frame.data(), legacy.data()); + EXPECT_LE(sti_frame.data() + sti_frame.size(), legacy.data() + legacy.size()); + SampledTermIndexReader sti; + ASSERT_TRUE(SampledTermIndexReader::open(sti_frame, &sti).ok()); + EXPECT_EQ(sti.n_blocks(), 2000U); + std::vector dbd_scratch; + Slice dbd_frame; + ASSERT_TRUE(reader.dict_block_directory_frame(&dbd_scratch, &dbd_frame).ok()); + EXPECT_TRUE(dbd_scratch.empty()); + DictBlockDirectoryReader dbd; + ASSERT_TRUE(DictBlockDirectoryReader::open(dbd_frame, &dbd).ok()); + EXPECT_EQ(dbd.n_blocks(), 2000U); +} + +// Below the threshold nothing changes: the default build is byte-identical to a +// compression-disabled build (small/legacy segments keep the exact old layout). +TEST(SniiPerIndexMetaZstd, SmallSectionsStayRawByteIdentical) { + const std::vector terms = {"alpha", "kappa", "zeta"}; + const std::vector refs = { + {.offset = 4096, .length = 1024, .n_entries = 10, .flags = 0, .checksum = 0x11U}}; + const auto def = BuildMeta(7, "title", terms, refs); + std::vector off_bytes; + { + ScopedEnv off("SNII_META_COMPRESS_MIN", kCompressOff); + off_bytes = BuildMeta(7, "title", terms, refs); + } + EXPECT_EQ(def, off_bytes); + const auto types = SectionTypesOf(Slice(def)); + EXPECT_TRUE(ContainsType(types, SectionType::kSampledTermIndex)); + EXPECT_TRUE(ContainsType(types, SectionType::kDictBlockDirectory)); + EXPECT_FALSE(ContainsType(types, SectionType::kSampledTermIndexZstd)); + EXPECT_FALSE(ContainsType(types, SectionType::kDictBlockDirectoryZstd)); +} + +// The threshold gate itself: sections between the zstd win size and the 4KB +// default stay raw by default but compress when the env lowers the threshold, +// and the forced layout still round-trips byte-exactly. +TEST(SniiPerIndexMetaZstd, ThresholdGateControlsCompression) { + const auto terms = ManyTerms(300); // ~2KB sti: below the 4KB default, above zstd win size + const auto refs = ManyRefs(300); + const auto sti_raw = BuildSampled(terms); + ASSERT_LT(sti_raw.size(), kMetaSectionCompressMinBytes); + + const auto def = BuildMeta(7, "title", terms, refs); + EXPECT_TRUE(ContainsType(SectionTypesOf(Slice(def)), SectionType::kSampledTermIndex)); + + std::vector forced; + { + ScopedEnv force("SNII_META_COMPRESS_MIN", "1"); + forced = BuildMeta(7, "title", terms, refs); + } + const auto forced_types = SectionTypesOf(Slice(forced)); + EXPECT_TRUE(ContainsType(forced_types, SectionType::kSampledTermIndexZstd)); + EXPECT_TRUE(ContainsType(forced_types, SectionType::kDictBlockDirectoryZstd)); + EXPECT_LT(forced.size(), def.size()); + + PerIndexMetaReader reader; + ASSERT_TRUE(PerIndexMetaReader::open(Slice(forced), &reader).ok()); + std::vector scratch; + Slice frame; + ASSERT_TRUE(reader.sampled_term_index_frame(&scratch, &frame).ok()); + ASSERT_EQ(frame.size(), sti_raw.size()); + EXPECT_EQ(std::memcmp(frame.data(), sti_raw.data(), sti_raw.size()), 0); +} + +// Garbage compressed bytes under a VALID carrier-frame crc must fail with +// kCorruption at materialization (the zstd layer catches what the crc cannot). +TEST(SniiPerIndexMetaZstd, CorruptZstdPayloadRejected) { + ByteSink payload; + payload.put_varint64(1024); // plausible uncomp_len + std::vector garbage(64, 0xAB); + payload.put_bytes(Slice(garbage)); + const auto bytes = BuildMetaWithRawStiSection(payload.view()); + + PerIndexMetaReader reader; + ASSERT_TRUE(PerIndexMetaReader::open(Slice(bytes), &reader).ok()); + std::vector scratch; + Slice frame; + EXPECT_TRUE(reader.sampled_term_index_frame(&scratch, &frame) + .is()); +} + +// Truncated compressed bytes and a wrong declared uncomp_len are both detected. +TEST(SniiPerIndexMetaZstd, TruncatedOrMisdeclaredZstdRejected) { + const auto sti_raw = BuildSampled(ManyTerms(500)); + std::vector comp; + ASSERT_TRUE(zstd_compress(Slice(sti_raw), 3, &comp).ok()); + + { + // Half the compressed stream, full declared length. + ByteSink payload; + payload.put_varint64(sti_raw.size()); + payload.put_bytes(Slice(comp.data(), comp.size() / 2)); + const auto bytes = BuildMetaWithRawStiSection(payload.view()); + PerIndexMetaReader reader; + ASSERT_TRUE(PerIndexMetaReader::open(Slice(bytes), &reader).ok()); + std::vector scratch; + Slice frame; + EXPECT_TRUE(reader.sampled_term_index_frame(&scratch, &frame) + .is()); + } + { + // Full compressed stream, under-declared length (decompress overflows dst). + ByteSink payload; + payload.put_varint64(sti_raw.size() - 1); + payload.put_bytes(Slice(comp)); + const auto bytes = BuildMetaWithRawStiSection(payload.view()); + PerIndexMetaReader reader; + ASSERT_TRUE(PerIndexMetaReader::open(Slice(bytes), &reader).ok()); + std::vector scratch; + Slice frame; + EXPECT_TRUE(reader.sampled_term_index_frame(&scratch, &frame) + .is()); + } +} + +// A declared uncomp_len outside (0, 256MB] is rejected before any allocation. +TEST(SniiPerIndexMetaZstd, ZstdUncompLenOutOfRangeRejected) { + for (const uint64_t bad_len : {uint64_t {0}, uint64_t {256ULL * 1024 * 1024 + 1}}) { + ByteSink payload; + payload.put_varint64(bad_len); + std::vector some(16, 0x01); + payload.put_bytes(Slice(some)); + const auto bytes = BuildMetaWithRawStiSection(payload.view()); + PerIndexMetaReader reader; + ASSERT_TRUE(PerIndexMetaReader::open(Slice(bytes), &reader).ok()); + std::vector scratch; + Slice frame; + EXPECT_TRUE(reader.sampled_term_index_frame(&scratch, &frame) + .is()) + << "uncomp_len=" << bad_len; + } +} diff --git a/be/test/storage/index/snii/format/prx_pod_test.cpp b/be/test/storage/index/snii/format/prx_pod_test.cpp new file mode 100644 index 00000000000000..37f2f352965dab --- /dev/null +++ b/be/test/storage/index/snii/format/prx_pod_test.cpp @@ -0,0 +1,1207 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/prx_pod.h" + +#include + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" +#include "storage/index/snii/encoding/pfor.h" +#include "storage/index/snii/encoding/section_framer.h" +#include "storage/index/snii/encoding/zstd_codec.h" +#include "storage/index/snii/format/format_constants.h" + +using doris::Status; // RETURN_IF_ERROR expands to bare Status +using doris::snii::ByteSink; +using doris::snii::ByteSource; +using doris::snii::pfor_encode; +using doris::snii::Slice; +using doris::snii::format::build_prx_window; +using doris::snii::format::build_prx_window_flat; +using doris::snii::format::kFrqBaseUnit; +using doris::snii::format::PrxCodec; +using doris::snii::format::read_prx_window; +using doris::snii::format::read_prx_window_csr; +using doris::snii::format::read_prx_window_csr_selective; + +// Integrated SNII reports corruption via doris ErrorCode::INVERTED_INDEX_FILE_CORRUPTED +// (the standalone build used doris::snii::StatusCode::kCorruption) and writer-side precondition +// violations via ErrorCode::INVALID_ARGUMENT; the negative tests below assert those codes. + +namespace { + +using PerDoc = std::vector>; + +// Encodes a uint32 array as the PFOR_runs wire form the prx PFOR payload uses: +// fixed-size runs of kFrqBaseUnit values, run count derived by the decoder from +// the total length (so it is not stored). Mirrors the in-source +// encode_pfor_runs so a test can hand-assemble a CRC-VALID PFOR payload and +// exercise the INNER decode branches (sum check / trailing bytes) instead of +// the outer CRC/codec. +void AppendPforRuns(const std::vector& values, ByteSink* out) { + const size_t n = values.size(); + for (size_t off = 0; off < n; off += kFrqBaseUnit) { + const size_t run = (n - off < kFrqBaseUnit) ? (n - off) : kFrqBaseUnit; + pfor_encode(values.data() + off, run, out); + } +} + +// Assembles a complete CRC-valid PFOR window: codec, uncomp_len, payload, crc. +// The payload header fields (doc_count, total_pos) are passed explicitly so a +// test can declare values that disagree with the encoded runs, and `trailing` +// lets a test pad the declared payload with undecodable bytes. The CRC always +// matches the emitted frame, so reads fail (if at all) on an INNER payload +// check, never CRC. +std::vector MakePforWindow(uint32_t doc_count, uint32_t total_pos, + const std::vector& freqs, + const std::vector& deltas, + const std::vector& trailing = {}) { + ByteSink payload; + payload.put_varint32(doc_count); + payload.put_varint32(total_pos); + AppendPforRuns(freqs, &payload); + AppendPforRuns(deltas, &payload); + payload.put_bytes(Slice(trailing)); + + ByteSink framed; + framed.put_u8(static_cast(PrxCodec::kPfor)); + framed.put_varint32(static_cast(payload.view().size())); + framed.put_bytes(payload.view()); + + ByteSink full; + full.put_bytes(framed.view()); + full.put_fixed32(doris::snii::crc32c(framed.view())); + return full.buffer(); +} + +// Flattens per-doc lists into (flat positions, freqs) the way the accumulator +// stores them, so the flat builder can be checked for byte-identity. +void Flatten(const PerDoc& in, std::vector* flat, std::vector* freqs) { + flat->clear(); + freqs->clear(); + for (const auto& doc : in) { + freqs->push_back(static_cast(doc.size())); + flat->insert(flat->end(), doc.begin(), doc.end()); + } +} + +// Build data above/below the raw threshold consistent with the production path; +// provides a stable, controlled round-trip helper. +Status RoundTrip(const PerDoc& in, int level, PerDoc* out) { + ByteSink sink; + RETURN_IF_ERROR(build_prx_window(in, level, &sink)); + ByteSource src(sink.view()); + RETURN_IF_ERROR(read_prx_window(&src, out)); + return Status::OK(); +} + +} // namespace + +// Single doc with ascending positions: delta-encoded round-trip must be +// lossless. +TEST(SniiPrxPod, SingleDocRoundTrip) { + PerDoc in = {{3, 7, 7, 10, 100}}; // includes duplicate positions (delta=0) + PerDoc out; + ASSERT_TRUE(RoundTrip(in, -1, &out).ok()); + EXPECT_EQ(out, in); +} + +// The FLAT builder (used by the writer to avoid materializing a +// vector-of-vectors for high-df terms) must produce BYTE-IDENTICAL window bytes +// to the per-doc builder for the same logical positions, at every codec level. +// This is the load-bearing guarantee that the flat refactor keeps .prx +// byte-identical. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiPrxPod, FlatBuilderMatchesPerDocBytes) { + const std::vector cases = { + {}, // 0 docs + {{}, {3}, {}, {}, {1, 2}}, // empty docs interleaved + {{0, 5, 12}, {1}, {2, 2, 9, 9, 40}}, // duplicate positions (delta 0) + {{3, 7, 7, 10, 100}}, // single doc + }; + for (const auto& in : cases) { + std::vector flat, freqs; + Flatten(in, &flat, &freqs); + for (int level : {-1, 0, 3}) { + ByteSink per_doc_sink, flat_sink; + ASSERT_TRUE(build_prx_window(in, level, &per_doc_sink).ok()); + ASSERT_TRUE(build_prx_window_flat(flat, freqs, level, &flat_sink).ok()); + const Slice a = per_doc_sink.view(); + const Slice b = flat_sink.view(); + ASSERT_EQ(a.size(), b.size()) << "level=" << level; + EXPECT_EQ(0, std::memcmp(a.data(), b.data(), a.size())) << "level=" << level; + // The flat-built window still decodes back to the original per-doc lists. + PerDoc out; + ByteSource src(flat_sink.view()); + ASSERT_TRUE(read_prx_window(&src, &out).ok()); + EXPECT_EQ(out, in) << "level=" << level; + } + } +} + +// A large window built flat (auto codec) must equal the per-doc build. The auto +// path picks the smaller of PFOR vs zstd; for these deltas it compresses (not +// raw), proving the flat path is byte-identical to the per-doc path through the +// same auto codec. +TEST(SniiPrxPod, FlatBuilderMatchesPerDocLargePfor) { + PerDoc in; + for (uint32_t d = 0; d < 300; ++d) { + in.push_back({d, d + 1U, d + 2U}); + } + std::vector flat, freqs; + Flatten(in, &flat, &freqs); + ByteSink per_doc_sink, flat_sink; + ASSERT_TRUE(build_prx_window(in, -1, &per_doc_sink).ok()); + ASSERT_TRUE(build_prx_window_flat(flat, freqs, -1, &flat_sink).ok()); + const Slice a = per_doc_sink.view(); + const Slice b = flat_sink.view(); + ASSERT_EQ(a.size(), b.size()); + EXPECT_EQ(0, std::memcmp(a.data(), b.data(), a.size())); + EXPECT_NE(a.data()[0], static_cast(doris::snii::format::PrxCodec::kRaw)); + // Round-trips losslessly back to the per-doc lists. + PerDoc out; + ByteSource src(flat_sink.view()); + ASSERT_TRUE(read_prx_window(&src, &out).ok()); + EXPECT_EQ(out, in); +} + +TEST(SniiPrxPod, CsrPforRoundTripMatchesPerDoc) { + PerDoc in; + for (uint32_t d = 0; d < 300; ++d) { + if (d % 7 == 0) { + in.emplace_back(); + } else if (d % 5 == 0) { + in.push_back({d, d + 2U, d + 9U}); + } else { + in.push_back({d + 1U}); + } + } + std::vector flat, freqs; + Flatten(in, &flat, &freqs); + + ByteSink sink; + ASSERT_TRUE(build_prx_window_flat(flat, freqs, -1, &sink).ok()); + ASSERT_EQ(sink.view().data()[0], static_cast(PrxCodec::kPfor)); + + std::vector pos_flat = {999U}; + std::vector pos_off = {777U}; + ByteSource src(sink.view()); + ASSERT_TRUE(read_prx_window_csr(&src, &pos_flat, &pos_off).ok()); + + ASSERT_EQ(pos_off.size(), in.size() + 1); + EXPECT_EQ(pos_off.front(), 0U); + EXPECT_EQ(pos_off.back(), pos_flat.size()); + PerDoc got; + got.reserve(in.size()); + for (size_t i = 0; i < in.size(); ++i) { + got.emplace_back(pos_flat.begin() + pos_off[i], pos_flat.begin() + pos_off[i + 1]); + } + EXPECT_EQ(got, in); +} + +TEST(SniiPrxPod, SelectiveCsrPforReturnsOnlyRequestedDocs) { + PerDoc in; + for (uint32_t d = 0; d < 320; ++d) { + if (d % 11 == 0) { + in.emplace_back(); + } else if (d % 7 == 0) { + in.push_back({d, d + 3U, d + 8U}); + } else { + in.push_back({d + 1U}); + } + } + std::vector flat, freqs; + Flatten(in, &flat, &freqs); + + ByteSink sink; + ASSERT_TRUE(build_prx_window_flat(flat, freqs, -1, &sink).ok()); + ASSERT_EQ(sink.view().data()[0], static_cast(PrxCodec::kPfor)); + + const std::vector ordinals = {0, 1, 7, 77, 255, 319}; + std::vector pos_flat = {999U}; + std::vector pos_off = {777U}; + ByteSource src(sink.view()); + ASSERT_TRUE(read_prx_window_csr_selective(&src, ordinals, &pos_flat, &pos_off).ok()); + + ASSERT_EQ(pos_off.size(), ordinals.size() + 1); + EXPECT_EQ(pos_off.front(), 0U); + EXPECT_EQ(pos_off.back(), pos_flat.size()); + PerDoc got; + got.reserve(ordinals.size()); + for (size_t i = 0; i < ordinals.size(); ++i) { + got.emplace_back(pos_flat.begin() + pos_off[i], pos_flat.begin() + pos_off[i + 1]); + } + PerDoc want; + for (uint32_t ordinal : ordinals) { + want.push_back(in[ordinal]); + } + EXPECT_EQ(got, want); +} + +TEST(SniiPrxPod, SelectiveCsrRejectsInvalidOrdinals) { + PerDoc in = {{1}, {2}, {3}}; + std::vector flat, freqs; + Flatten(in, &flat, &freqs); + ByteSink sink; + ASSERT_TRUE(build_prx_window_flat(flat, freqs, -1, &sink).ok()); + + std::vector pos_flat; + std::vector pos_off; + { + const std::vector unsorted = {1, 1}; + ByteSource src(sink.view()); + EXPECT_TRUE(read_prx_window_csr_selective(&src, unsorted, &pos_flat, &pos_off) + .is()); + } + { + const std::vector out_of_range = {3}; + ByteSource src(sink.view()); + EXPECT_TRUE(read_prx_window_csr_selective(&src, out_of_range, &pos_flat, &pos_off) + .is()); + } +} + +// Multiple docs, positions ascending within each doc, no shared baseline across +// docs. +TEST(SniiPrxPod, MultiDocRoundTrip) { + PerDoc in = {{0, 5, 12}, {1}, {2, 2, 9, 9, 40}, {7, 8, 9, 10}}; + PerDoc out; + ASSERT_TRUE(RoundTrip(in, -1, &out).ok()); + EXPECT_EQ(out, in); +} + +// Empty positions: supports both 0 docs and 0 positions within a doc. +TEST(SniiPrxPod, EmptyPositions) { + PerDoc empty_window; // 0 docs + PerDoc out1; + ASSERT_TRUE(RoundTrip(empty_window, -1, &out1).ok()); + EXPECT_EQ(out1, empty_window); + + PerDoc with_empty_docs = {{}, {3}, {}, {}, {1, 2}}; // contains empty docs + PerDoc out2; + ASSERT_TRUE(RoundTrip(with_empty_docs, -1, &out2).ok()); + EXPECT_EQ(out2, with_empty_docs); +} + +// Small window (level<0 auto) now uses PFOR: the auto codec always bit-packs +// deltas (PFOR is cheap and competitive even for small windows; no zstd, no +// size-based raw fallback). codec byte == kPfor and it round-trips. +TEST(SniiPrxPod, SmallWindowUsesPfor) { + PerDoc in = {{1, 2, 3}, {4, 5}}; + ByteSink sink; + ASSERT_TRUE(build_prx_window(in, -1, &sink).ok()); + + ByteSource src(sink.view()); + uint8_t codec = 0xFF; + ASSERT_TRUE(src.get_u8(&codec).ok()); + EXPECT_EQ(codec, static_cast(doris::snii::format::PrxCodec::kPfor)); + + PerDoc out; + ByteSource rt(sink.view()); + ASSERT_TRUE(read_prx_window(&rt, &out).ok()); + EXPECT_EQ(out, in); +} + +// Large window (all-1 deltas) auto-encodes as PFOR; for tiny constant deltas +// the 1-bit-packed PFOR payload is much smaller than the forced raw varint +// encoding. +TEST(SniiPrxPod, LargeWindowTriggersPforAndIsSmaller) { + PerDoc in; + in.reserve(64); + for (int d = 0; d < 64; ++d) { + std::vector doc; + uint32_t p = 0; + for (int i = 0; i < 256; ++i) { + p += 1; // all-1 deltas: pack to ~1 bit each + doc.push_back(p); + } + in.push_back(std::move(doc)); + } + + ByteSink auto_sink; + ASSERT_TRUE(build_prx_window(in, -1, &auto_sink).ok()); + ByteSource probe(auto_sink.view()); + uint8_t codec = 0xFF; + ASSERT_TRUE(probe.get_u8(&codec).ok()); + // Auto path compresses with the smaller of PFOR vs zstd; assert it is not raw. + EXPECT_NE(codec, static_cast(doris::snii::format::PrxCodec::kRaw)); + + ByteSink raw_sink; + ASSERT_TRUE(build_prx_window(in, /*level=*/0, &raw_sink).ok()); + ByteSource raw_probe(raw_sink.view()); + uint8_t raw_codec = 0xFF; + ASSERT_TRUE(raw_probe.get_u8(&raw_codec).ok()); + EXPECT_EQ(raw_codec, static_cast(doris::snii::format::PrxCodec::kRaw)); + + EXPECT_LT(auto_sink.size(), raw_sink.size()); + + // The PFOR path restores losslessly. + PerDoc out; + ByteSource z(auto_sink.view()); + ASSERT_TRUE(read_prx_window(&z, &out).ok()); + EXPECT_EQ(out, in); +} + +// PFOR round-trips arbitrary multi-doc windows (including empty docs, duplicate +// positions, and large jumps that force PFOR exceptions) losslessly. +TEST(SniiPrxPod, PforRoundTripVariety) { + const std::vector cases = { + {{0, 5, 12}, {1}, {2, 2, 9, 9, 40}, {7, 8, 9, 10}}, + {{}, {3}, {}, {}, {1, 2}}, + {{0, 1000000, 1000001}, {5}}, // large jump => PFOR exception + {{3, 7, 7, 10, 100}}, // duplicate positions (delta 0) + }; + for (const auto& in : cases) { + ByteSink sink; + ASSERT_TRUE(build_prx_window(in, -1, &sink).ok()); + ByteSource probe(sink.view()); + uint8_t codec = 0xFF; + ASSERT_TRUE(probe.get_u8(&codec).ok()); + EXPECT_EQ(codec, static_cast(doris::snii::format::PrxCodec::kPfor)); + PerDoc out; + ByteSource src(sink.view()); + ASSERT_TRUE(read_prx_window(&src, &out).ok()); + EXPECT_EQ(out, in); + } +} + +// level=0 explicitly forces raw (no compression), useful for testing and the +// oversized single-doc degenerate path. +TEST(SniiPrxPod, ExplicitRawLevelRoundTrip) { + PerDoc in = {{10, 20, 30}, {40}}; + PerDoc out; + ASSERT_TRUE(RoundTrip(in, /*level=*/0, &out).ok()); + EXPECT_EQ(out, in); +} + +// CRC corruption is detectable: flipping any byte in the payload causes read to +// return Corruption. +TEST(SniiPrxPod, CrcCorruptionDetected) { + PerDoc in = {{1, 2, 3, 4, 5}, {6, 7, 8}}; + ByteSink sink; + ASSERT_TRUE(build_prx_window(in, -1, &sink).ok()); + + std::vector bytes = sink.buffer(); + ASSERT_GT(bytes.size(), 1U); + bytes.back() ^= 0xFF; // corrupt the last byte of the payload + + Slice corrupt(bytes); + ByteSource src(corrupt); + PerDoc out; + Status s = read_prx_window(&src, &out); + EXPECT_FALSE(s.ok()); + EXPECT_TRUE(s.is()); +} + +// A corrupted codec byte (invalid codec value) should also be rejected. +TEST(SniiPrxPod, InvalidCodecRejected) { + PerDoc in = {{1, 2}}; + ByteSink sink; + ASSERT_TRUE(build_prx_window(in, -1, &sink).ok()); + + std::vector bytes = sink.buffer(); + bytes[0] = 0x7F; // invalid codec (bits 0-5 exceed known values) + + Slice corrupt(bytes); + ByteSource src(corrupt); + PerDoc out; + Status s = read_prx_window(&src, &out); + EXPECT_FALSE(s.ok()); +} + +// Truncated input (insufficient payload) should return an error rather than +// crash. +TEST(SniiPrxPod, TruncatedInputRejected) { + PerDoc in = {{1, 2, 3, 4, 5, 6, 7, 8}}; + ByteSink sink; + ASSERT_TRUE(build_prx_window(in, -1, &sink).ok()); + + std::vector bytes = sink.buffer(); + bytes.resize(bytes.size() / 2); // truncate to half + + Slice truncated(bytes); + ByteSource src(truncated); + PerDoc out; + Status s = read_prx_window(&src, &out); + EXPECT_FALSE(s.ok()); +} + +// DoS prevention: an uncomp_len corrupted to a huge value must be rejected +// before allocation/decompression. +TEST(SniiPrxPod, OversizedUncompLenRejected) { + ByteSink sink; + sink.put_u8(static_cast(doris::snii::format::PrxCodec::kZstd)); + sink.put_varint32(300U * 1024 * 1024); // > 256MiB window limit + // No need to construct the subsequent comp_len/payload/crc — the cap check + // triggers immediately after reading uncomp_len. + ByteSource src(sink.view()); + PerDoc out; + Status s = read_prx_window(&src, &out); + EXPECT_TRUE(s.is()); +} + +// DoS prevention: a CRC-VALID raw frame whose decoded doc_count is absurd must +// return Corruption, not a giant reserve()/assign() -> std::bad_alloc. Distinct +// from the CRC and uncomp_len tests above, which are caught BEFORE the inner +// doc_count is read. +TEST(SniiPrxPod, OversizedDocCountRejected) { + ByteSink payload; + payload.put_varint32(0x02000000U); // doc_count = 33M, > kMaxWindowDocs (1<<24) + ByteSink framed; + framed.put_u8(static_cast(doris::snii::format::PrxCodec::kRaw)); + framed.put_varint32(static_cast(payload.view().size())); // uncomp_len + framed.put_bytes(payload.view()); + ByteSink full; + full.put_bytes(framed.view()); + full.put_fixed32(doris::snii::crc32c(framed.view())); // valid crc over codec+uncomp_len+payload + ByteSource src(full.view()); + PerDoc out; + Status s = read_prx_window(&src, &out); + EXPECT_TRUE(s.is()) << s.to_string(); +} + +// Sanity: the MakePforWindow helper builds a window the production reader +// accepts, so the negative variants below isolate the ONE field they tamper +// with (not a helper bug). doc_count=2, total_pos=3, freqs sum to total_pos, +// deltas match. +TEST(SniiPrxPod, CraftedPforWindowRoundTrips) { + // doc0 = {5, 6} (deltas 5,1), doc1 = {9} (delta 9); pos_counts {2,1}, + // total 3. + auto bytes = MakePforWindow(/*doc_count=*/2, /*total_pos=*/3, + /*freqs=*/ {2, 1}, /*deltas=*/ {5, 1, 9}); + Slice s(bytes); + ByteSource src(s); + PerDoc out; + ASSERT_TRUE(read_prx_window(&src, &out).ok()); + PerDoc expected = {{5, 6}, {9}}; + EXPECT_EQ(out, expected); +} + +// CRC-VALID PFOR frame whose declared total_pos disagrees with sum(pos_counts): +// the reader must reach the INNER "pos_count sum mismatch" branch (after the +// cap checks and after decoding the freqs runs) and return Corruption -- NOT a +// CRC or codec rejection. freqs sum to 3 but total_pos is declared 4. +TEST(SniiPrxPod, PforPosCountSumMismatchRejected) { + auto bytes = MakePforWindow(/*doc_count=*/2, /*total_pos=*/4, + /*freqs=*/ {2, 1}, /*deltas=*/ {5, 1, 9}); + Slice slice(bytes); + ByteSource src(slice); + PerDoc out; + Status s = read_prx_window(&src, &out); + EXPECT_TRUE(s.is()) << s.to_string(); +} + +// CRC-VALID PFOR frame with extra bytes appended INSIDE the declared payload +// (uncomp_len covers them, so the CRC is over them too). After +// decode_pfor_payload consumes the real doc_count/total_pos/freqs/deltas, the +// source is not at EOF, so the INNER "trailing bytes after pfor payload" branch +// must fire -> Corruption. +TEST(SniiPrxPod, PforTrailingBytesRejected) { + auto bytes = MakePforWindow(/*doc_count=*/2, /*total_pos=*/3, + /*freqs=*/ {2, 1}, /*deltas=*/ {5, 1, 9}, + /*trailing=*/ {0xAA, 0xBB, 0xCC}); + Slice slice(bytes); + ByteSource src(slice); + PerDoc out; + Status s = read_prx_window(&src, &out); + EXPECT_TRUE(s.is()) << s.to_string(); +} + +// FLAT builder precondition: a (positions_flat, freqs) mismatch where +// sum(freqs) overruns positions_flat would index flat[off+i] past the span end. +// The guard must return a CLEAN InvalidArgument (never OOB / UB) at every flat +// codec level. Here freqs sum to 5 but only 3 positions are supplied. +TEST(SniiPrxPod, FlatBuilderShortPositionsRejectedNotOob) { + std::vector positions_flat = {1, 2, 3}; // 3 positions + std::vector freqs = {2, 3}; // claims 5 positions + for (int level : {-1, 0, 3}) { // pfor, raw, zstd flat encoders all guard + ByteSink sink; + Status s = build_prx_window_flat(positions_flat, freqs, level, &sink); + EXPECT_TRUE(s.is()) + << "level=" << level << " msg=" << s.to_string(); + EXPECT_EQ(sink.size(), 0U) << "level=" << level; // nothing emitted on reject + } +} + +// FLAT builder precondition (other direction): sum(freqs) < positions_flat +// leaves trailing positions unused -- also a writer-side mismatch. The guard +// requires EXACT equality, so this is rejected too (a silently-dropped position +// is a bug). +TEST(SniiPrxPod, FlatBuilderExtraPositionsRejected) { + std::vector positions_flat = {1, 2, 3, 4, 5}; // 5 positions + std::vector freqs = {2, 1}; // only addresses 3 + for (int level : {-1, 0, 3}) { + ByteSink sink; + Status s = build_prx_window_flat(positions_flat, freqs, level, &sink); + EXPECT_TRUE(s.is()) + << "level=" << level << " msg=" << s.to_string(); + } +} + +// --------------------------------------------------------------------------- +// T14: auto-mode (kAutoZstd) single-encode -- the auto builders derive per-doc +// deltas once and only materialize the throwaway raw plaintext payload when it is +// large enough (>= 512 B) to make a zstd attempt worthwhile. The tests below +// cover the deterministic raw-build counter seam plus the byte/codec equivalence +// guarantees that prove zero on-disk format change. +// --------------------------------------------------------------------------- + +namespace { + +using doris::snii::format::testing::prx_raw_build_count; +using doris::snii::format::testing::reset_prx_raw_build_count; + +// Byte length of put_varint32(v); mirrors the in-source varint32_size so the +// brute-force codec recomputation below can size frames exactly. +size_t VarintSize(uint32_t v) { + size_t n = 1; + while (v >= 128) { + v >>= 7; + ++n; + } + return n; +} + +// Auto constants restated from prx_pod.cpp (anonymous there). The brute-force +// codec check must use the SAME threshold/level write_auto_pfor_or_zstd uses. +constexpr size_t kAutoZstdMinBytes = 512; +constexpr int kDefaultZstdLevel = 3; + +// Per-doc -> flat delta stream (first position of each doc absolute, the rest +// delta-within-doc): the exact sequence both .prx payload encoders serialize. +std::vector FlatDeltas(const std::vector& flat, + const std::vector& freqs) { + std::vector deltas; + deltas.reserve(flat.size()); + size_t off = 0; + for (uint32_t fc : freqs) { + uint32_t prev = 0; + for (uint32_t i = 0; i < fc; ++i) { + const uint32_t pos = flat[off + i]; + deltas.push_back(i == 0 ? pos : pos - prev); + prev = pos; + } + off += fc; + } + return deltas; +} + +// Independently recompute the codec byte the auto builder must emit, replicating +// write_auto_pfor_or_zstd: try zstd only when the EXACT plaintext payload size +// reaches the threshold, and choose zstd only when its frame is strictly smaller +// than the PFOR frame. +uint8_t BruteForceAutoCodec(const std::vector& freqs, + const std::vector& deltas) { + ByteSink pfor_payload; + pfor_payload.put_varint32(static_cast(freqs.size())); + pfor_payload.put_varint32(static_cast(deltas.size())); + AppendPforRuns(freqs, &pfor_payload); + AppendPforRuns(deltas, &pfor_payload); + const size_t pfor_size = pfor_payload.view().size(); + + ByteSink plain; + plain.put_varint32(static_cast(freqs.size())); + size_t off = 0; + for (uint32_t fc : freqs) { + plain.put_varint32(fc); + for (uint32_t i = 0; i < fc; ++i) { + plain.put_varint32(deltas[off + i]); + } + off += fc; + } + const size_t plain_size = plain.view().size(); + + const auto kPfor = static_cast(PrxCodec::kPfor); + const auto kZstd = static_cast(PrxCodec::kZstd); + if (plain_size >= kAutoZstdMinBytes) { + std::vector comp; + EXPECT_TRUE(doris::snii::zstd_compress(plain.view(), kDefaultZstdLevel, &comp).ok()); + const size_t zstd_frame = 1 + VarintSize(static_cast(plain_size)) + + VarintSize(static_cast(comp.size())) + comp.size() + + sizeof(uint32_t); + const size_t pfor_frame = + 1 + VarintSize(static_cast(pfor_size)) + pfor_size + sizeof(uint32_t); + if (zstd_frame < pfor_frame) { + return kZstd; + } + } + return kPfor; +} + +// One doc whose raw plaintext payload is EXACTLY `target` bytes. Positions +// {0,1,...,fc-1} make every delta a 1-byte varint (0 then 1s), so the plaintext +// is varint32_size(doc_count=1) + varint32_size(fc) + fc. With fc in +// [128, 16383] the fc varint is 2 bytes, so plaintext == 3 + fc => fc = target-3. +std::vector SingleDocWithPlaintextSize(size_t target) { + EXPECT_GE(target, 131U); // need fc >= 128 so the count varint is 2 bytes + const auto fc = static_cast(target - 3); + std::vector doc(fc); + for (uint32_t i = 0; i < fc; ++i) { + doc[i] = i; + } + return doc; +} + +} // namespace + +// [perf-deterministic] A sub-threshold window (<512 B plaintext) must NOT +// materialize the throwaway raw plaintext payload: the auto path emits PFOR +// directly. On the pre-optimization code (which always materialized) this counter +// would be 1 -> this is the RED guard for the single-encode change. +TEST(SniiPrxPodCounterTest, SmallWindowSkipsRawPlaintextBuild) { + PerDoc in = {{1}, {3, 5}, {2}}; + std::vector flat, freqs; + Flatten(in, &flat, &freqs); + + reset_prx_raw_build_count(); + ByteSink sink; + ASSERT_TRUE(build_prx_window_flat(flat, freqs, -1, &sink).ok()); + EXPECT_EQ(prx_raw_build_count(), 0U); + + // codec is PFOR and the window still round-trips. + EXPECT_EQ(sink.view().data()[0], static_cast(PrxCodec::kPfor)); + PerDoc out; + ByteSource src(sink.view()); + ASSERT_TRUE(read_prx_window(&src, &out).ok()); + EXPECT_EQ(out, in); +} + +// [perf-deterministic] A >=512 B window materializes the raw plaintext EXACTLY +// once (needed for the zstd-vs-pfor comparison) -- not zero (would skip a viable +// zstd) and not twice (the old double-encode). +TEST(SniiPrxPodCounterTest, LargeWindowStillBuildsRawPlaintextOnce) { + PerDoc in = {SingleDocWithPlaintextSize(512)}; + std::vector flat, freqs; + Flatten(in, &flat, &freqs); + + reset_prx_raw_build_count(); + ByteSink sink; + ASSERT_TRUE(build_prx_window_flat(flat, freqs, -1, &sink).ok()); + EXPECT_EQ(prx_raw_build_count(), 1U); +} + +// [perf-deterministic] N consecutive small windows materialize ZERO raw +// plaintext payloads in aggregate (the per-window throwaway ByteSink is the +// allocation this task removes for the common Zipfian case). +TEST(SniiPrxPodCounterTest, ManySmallWindowsBuildZeroRawPlaintext) { + reset_prx_raw_build_count(); + for (uint32_t w = 0; w < 64; ++w) { + PerDoc in = {{w}, {w + 1U, w + 3U}, {w + 2U}}; + std::vector flat, freqs; + Flatten(in, &flat, &freqs); + ByteSink sink; + ASSERT_TRUE(build_prx_window_flat(flat, freqs, -1, &sink).ok()); + } + EXPECT_EQ(prx_raw_build_count(), 0U); +} + +// [perf-deterministic] The per-doc (vector) builder funnels through the SAME auto +// path as the flat builder: small window materializes nothing, large window once. +TEST(SniiPrxPodCounterTest, VectorBuilderSharesSingleEncodePath) { + reset_prx_raw_build_count(); + PerDoc small = {{1, 2, 3}, {4, 5}}; + ByteSink small_sink; + ASSERT_TRUE(build_prx_window(small, -1, &small_sink).ok()); + EXPECT_EQ(prx_raw_build_count(), 0U); + + reset_prx_raw_build_count(); + PerDoc large = {SingleDocWithPlaintextSize(512)}; + ByteSink large_sink; + ASSERT_TRUE(build_prx_window(large, -1, &large_sink).ok()); + EXPECT_EQ(prx_raw_build_count(), 1U); +} + +// [functional/perf] The flat and per-doc auto builders must produce BYTE-IDENTICAL +// windows for the same logical positions, across sizes spanning the 512-byte zstd +// threshold (empty, single, tiny, just-below, exactly-at, well-above). This is the +// load-bearing proof that the single-encode refactor changed zero on-disk bytes. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiPrxPodTest, FlatAutoProducesByteIdenticalOutputAcrossSizes) { + std::vector cases; + cases.emplace_back(); // empty window + cases.push_back({{7}}); // single doc/pos + cases.push_back({{1}, {3, 5}, {2}}); // tiny (<512 plaintext) + cases.push_back({{}, {3}, {}, {}, {1, 2}}); // empty docs interleaved + cases.push_back({SingleDocWithPlaintextSize(511)}); // just below threshold + cases.push_back({SingleDocWithPlaintextSize(512)}); // exactly at threshold + { + PerDoc big; // well above threshold + for (uint32_t d = 0; d < 280; ++d) { + big.push_back({d, d + 1U, d + 2U}); + } + cases.push_back(std::move(big)); + } + + for (const auto& in : cases) { + std::vector flat, freqs; + Flatten(in, &flat, &freqs); + ByteSink per_doc_sink, flat_sink; + ASSERT_TRUE(build_prx_window(in, -1, &per_doc_sink).ok()); + ASSERT_TRUE(build_prx_window_flat(flat, freqs, -1, &flat_sink).ok()); + const Slice a = per_doc_sink.view(); + const Slice b = flat_sink.view(); + ASSERT_EQ(a.size(), b.size()) << "doc_count=" << in.size(); + EXPECT_EQ(0, std::memcmp(a.data(), b.data(), a.size())) << "doc_count=" << in.size(); + // Auto mode never emits the forced raw codec. + ASSERT_GT(a.size(), 0U); + EXPECT_NE(a.data()[0], static_cast(PrxCodec::kRaw)) << "doc_count=" << in.size(); + // The flat-built window round-trips back to the original per-doc lists. + PerDoc out; + ByteSource src(flat_sink.view()); + ASSERT_TRUE(read_prx_window(&src, &out).ok()); + EXPECT_EQ(out, in) << "doc_count=" << in.size(); + } +} + +// [functional/perf] At the EXACT 511- vs 512-byte plaintext boundary the codec +// the builder emits must equal an independent brute-force frame-size comparison. +// This guards the risk that an estimated (rather than exact) plaintext size flips +// the codec choice near the threshold and silently changes the on-disk bytes. +TEST(SniiPrxPodTest, AutoCodecChoiceMatchesBruteForceAtThreshold) { + for (size_t target : {size_t {511}, size_t {512}}) { + PerDoc in = {SingleDocWithPlaintextSize(target)}; + std::vector flat, freqs; + Flatten(in, &flat, &freqs); + + // Confirm the construction lands on exactly the intended plaintext size. + const std::vector deltas = FlatDeltas(flat, freqs); + size_t plain_size = VarintSize(static_cast(freqs.size())); + for (uint32_t fc : freqs) { + plain_size += VarintSize(fc); + } + for (uint32_t d : deltas) { + plain_size += VarintSize(d); + } + ASSERT_EQ(plain_size, target); + + ByteSink sink; + ASSERT_TRUE(build_prx_window_flat(flat, freqs, -1, &sink).ok()); + const uint8_t emitted = sink.view().data()[0]; + EXPECT_EQ(emitted, BruteForceAutoCodec(freqs, deltas)) << "target=" << target; + if (target < kAutoZstdMinBytes) { + // Below the threshold zstd is never attempted: must be plain PFOR. + EXPECT_EQ(emitted, static_cast(PrxCodec::kPfor)); + } + // Round-trips regardless of the chosen codec. + PerDoc out; + ByteSource src(sink.view()); + ASSERT_TRUE(read_prx_window(&src, &out).ok()); + EXPECT_EQ(out, in) << "target=" << target; + } +} + +// [functional] F-RT-small: a small flat window round-trips through the CSR reader +// (codec=pfor path) back to the original per-doc positions. +TEST(SniiPrxPodTest, SmallFlatWindowRoundTripsViaCsr) { + PerDoc in = {{1}, {3, 5}, {2}}; + std::vector flat, freqs; + Flatten(in, &flat, &freqs); + ByteSink sink; + ASSERT_TRUE(build_prx_window_flat(flat, freqs, -1, &sink).ok()); + EXPECT_EQ(sink.view().data()[0], static_cast(PrxCodec::kPfor)); + + std::vector pos_flat, pos_off; + ByteSource src(sink.view()); + ASSERT_TRUE(read_prx_window_csr(&src, &pos_flat, &pos_off).ok()); + ASSERT_EQ(pos_off.size(), in.size() + 1); + PerDoc out; + for (size_t d = 0; d < in.size(); ++d) { + out.emplace_back(pos_flat.begin() + pos_off[d], pos_flat.begin() + pos_off[d + 1]); + } + EXPECT_EQ(out, in); +} + +// [functional] F-RT-large: a >=512 B flat window (exercising the zstd-vs-pfor +// pick) round-trips losslessly through the per-doc reader. +TEST(SniiPrxPodTest, LargeFlatWindowRoundTrips) { + PerDoc in; + for (uint32_t d = 0; d < 280; ++d) { + in.push_back({d, d + 4U, d + 9U}); + } + std::vector flat, freqs; + Flatten(in, &flat, &freqs); + ByteSink sink; + ASSERT_TRUE(build_prx_window_flat(flat, freqs, -1, &sink).ok()); + EXPECT_NE(sink.view().data()[0], static_cast(PrxCodec::kRaw)); + PerDoc out; + ByteSource src(sink.view()); + ASSERT_TRUE(read_prx_window(&src, &out).ok()); + EXPECT_EQ(out, in); +} + +// [functional/boundary] F-EMPTY: a 0-doc window builds OK, materializes no raw +// plaintext, decodes to an empty result. +TEST(SniiPrxPodTest, EmptyFlatWindowRoundTripsWithNoRawBuild) { + std::vector flat, freqs; // 0 docs, 0 positions + reset_prx_raw_build_count(); + ByteSink sink; + ASSERT_TRUE(build_prx_window_flat(flat, freqs, -1, &sink).ok()); + EXPECT_EQ(prx_raw_build_count(), 0U); + EXPECT_EQ(sink.view().data()[0], static_cast(PrxCodec::kPfor)); + PerDoc out; + ByteSource src(sink.view()); + ASSERT_TRUE(read_prx_window(&src, &out).ok()); + EXPECT_TRUE(out.empty()); +} + +// [functional/boundary] F-SINGLE: a single doc with a single position round-trips. +TEST(SniiPrxPodTest, SingleDocSinglePositionFlatRoundTrips) { + std::vector flat = {42}; + std::vector freqs = {1}; + ByteSink sink; + ASSERT_TRUE(build_prx_window_flat(flat, freqs, -1, &sink).ok()); + PerDoc out; + ByteSource src(sink.view()); + ASSERT_TRUE(read_prx_window(&src, &out).ok()); + PerDoc expected = {{42}}; + EXPECT_EQ(out, expected); +} + +// [error] F-ERR-asc: descending positions within a doc are rejected by the +// single delta walk -- the ascending check is not lost when the second encode is +// skipped. Nothing is emitted. +TEST(SniiPrxPodTest, NonAscendingPositionsRejected) { + std::vector flat = {5, 3}; // within one doc, descending + std::vector freqs = {2}; + ByteSink sink; + Status s = build_prx_window_flat(flat, freqs, -1, &sink); + EXPECT_TRUE(s.is()) << s.to_string(); + EXPECT_EQ(sink.size(), 0U); +} + +// [error] F-ERR-part: a (flat, freqs) partition mismatch (sum(freqs) != size) is +// rejected before any indexing -- the partition check is preserved. +TEST(SniiPrxPodTest, PartitionMismatchRejected) { + std::vector flat = {1, 2, 3}; + std::vector freqs = {2, 3}; // sum 5 != 3 + ByteSink sink; + Status s = build_prx_window_flat(flat, freqs, -1, &sink); + EXPECT_TRUE(s.is()) << s.to_string(); + EXPECT_EQ(sink.size(), 0U); +} + +// [error] F-NULL: a null sink is rejected by both auto builders before any work. +TEST(SniiPrxPodTest, NullSinkRejected) { + std::vector flat = {1, 2}; + std::vector freqs = {2}; + Status s = build_prx_window_flat(flat, freqs, -1, nullptr); + EXPECT_TRUE(s.is()) << s.to_string(); + + PerDoc per_doc = {{1, 2}}; + Status s2 = build_prx_window(per_doc, -1, nullptr); + EXPECT_TRUE(s2.is()) << s2.to_string(); +} + +// =========================================================================== +// T22 -- single-copy framing (write_pfor / write_raw / write_zstd_compressed + +// SectionFramer::write) and the bitpack capacity reserve. The framing refactor +// writes [codec/type][len][payload] DIRECTLY into the caller's sink and crc's +// that span in place (no temp ByteSink), so these tests pin the on-disk bytes to +// an INDEPENDENT hand-assembly of the wire frame. A wrong crc range, a view() +// taken at the wrong time, or a wrong start offset shows up as a byte or Status +// mismatch. All bytes MUST stay identical to the pre-refactor output. +// =========================================================================== + +namespace { + +using doris::snii::FramedSection; +using doris::snii::pfor_decode; +using doris::snii::SectionFramer; + +// Appends the 4-byte crc32c(prefix) tail to `prefix`, returning the full on-disk +// frame. Mirrors the reader's crc coverage: the crc is over +// [codec/type][len][payload] ONLY (the bytes already in prefix), never itself. +std::vector WithCrc(const ByteSink& prefix) { + ByteSink full; + full.put_bytes(prefix.view()); + full.put_fixed32(doris::snii::crc32c(prefix.view())); + return full.buffer(); +} + +// The RAW/ZSTD plaintext payload (encode_payload_flat wire form): VInt doc_count, +// then per doc VInt pos_count followed by that doc's position deltas (VInt). +ByteSink RawPlaintext(const std::vector& flat, const std::vector& freqs) { + const std::vector deltas = FlatDeltas(flat, freqs); + ByteSink plain; + plain.put_varint32(static_cast(freqs.size())); + size_t off = 0; + for (uint32_t fc : freqs) { + plain.put_varint32(fc); + for (uint32_t i = 0; i < fc; ++i) { + plain.put_varint32(deltas[off + i]); + } + off += fc; + } + return plain; +} + +// Deterministic pseudo-random uint32 run whose values fit in `w` bits, with a +// couple of exact-`w`-bit values forced so choose_width sees width w. Round-trip +// must be lossless regardless of the width PFOR actually picks. +std::vector MakeWidthRun(uint8_t w, size_t n) { + std::vector run(n, 0U); + uint32_t mask = 0; + if (w >= 32) { + mask = 0xFFFFFFFFU; + } else if (w != 0) { + mask = (1U << w) - 1U; + } + uint32_t state = 0x12345678U ^ (static_cast(w) << 24) ^ static_cast(n); + for (size_t i = 0; i < n; ++i) { + state = state * 1664525U + 1013904223U; // LCG + run[i] = state & mask; + } + if (n > 0 && w > 0) { + run[0] = mask; // force a full-width value + run[n / 2] = mask; // and one mid-run (exercises exceptions if a smaller w is picked) + } + return run; +} + +} // namespace + +// [perf-deterministic] PRX-BYTE-PFOR: an auto (<512 B) window emits PFOR and its +// on-disk bytes equal an independent [kPfor][len][payload][crc] hand-assembly, +// proving write_pfor's single-copy framing is byte-identical. +TEST(SniiPrxPodTest, PforFramingProducesByteIdenticalOutput) { + std::vector flat = {1, 2, 3, 4, 5}; + std::vector freqs = {3, 2}; + ByteSink sink; + ASSERT_TRUE(build_prx_window_flat(flat, freqs, -1, &sink).ok()); + ASSERT_EQ(sink.view().data()[0], static_cast(PrxCodec::kPfor)); + + const std::vector deltas = FlatDeltas(flat, freqs); + ByteSink payload; + payload.put_varint32(static_cast(freqs.size())); + payload.put_varint32(static_cast(deltas.size())); + AppendPforRuns(freqs, &payload); + AppendPforRuns(deltas, &payload); + ByteSink framed; + framed.put_u8(static_cast(PrxCodec::kPfor)); + framed.put_varint32(static_cast(payload.view().size())); + framed.put_bytes(payload.view()); + + EXPECT_EQ(sink.buffer(), WithCrc(framed)); +} + +// [perf-deterministic] PRX-BYTE-RAW: level=0 forces raw; the bytes equal an +// independent [kRaw][len][plaintext][crc] assembly (write_raw single-copy). +TEST(SniiPrxPodTest, RawFramingProducesByteIdenticalOutput) { + std::vector flat = {10, 20, 30, 40}; + std::vector freqs = {3, 1}; + ByteSink sink; + ASSERT_TRUE(build_prx_window_flat(flat, freqs, /*level=*/0, &sink).ok()); + ASSERT_EQ(sink.view().data()[0], static_cast(PrxCodec::kRaw)); + + ByteSink plain = RawPlaintext(flat, freqs); + ByteSink framed; + framed.put_u8(static_cast(PrxCodec::kRaw)); + framed.put_varint32(static_cast(plain.view().size())); + framed.put_bytes(plain.view()); + + EXPECT_EQ(sink.buffer(), WithCrc(framed)); +} + +// [perf-deterministic] PRX-BYTE-ZSTD: level>0 forces zstd; the bytes equal an +// independent [kZstd][uncomp_len][comp_len][zstd(plaintext)][crc] assembly +// (write_zstd_compressed single-copy). zstd is deterministic at a fixed level. +TEST(SniiPrxPodTest, ZstdFramingProducesByteIdenticalOutput) { + std::vector flat, freqs; + for (uint32_t d = 0; d < 200; ++d) { + flat.push_back(d); + freqs.push_back(1); + } + ByteSink sink; + ASSERT_TRUE(build_prx_window_flat(flat, freqs, /*level=*/3, &sink).ok()); + ASSERT_EQ(sink.view().data()[0], static_cast(PrxCodec::kZstd)); + + ByteSink plain = RawPlaintext(flat, freqs); + std::vector comp; + ASSERT_TRUE(doris::snii::zstd_compress(plain.view(), /*level=*/3, &comp).ok()); + ByteSink framed; + framed.put_u8(static_cast(PrxCodec::kZstd)); + framed.put_varint32(static_cast(plain.view().size())); + framed.put_varint32(static_cast(comp.size())); + framed.put_bytes(Slice(comp)); + + EXPECT_EQ(sink.buffer(), WithCrc(framed)); + + // Round-trips losslessly back to the original per-doc positions. + PerDoc out; + ByteSource src(sink.view()); + ASSERT_TRUE(read_prx_window(&src, &out).ok()); + ASSERT_EQ(out.size(), flat.size()); + for (uint32_t d = 0; d < flat.size(); ++d) { + ASSERT_EQ(out[d].size(), 1U); + EXPECT_EQ(out[d][0], flat[d]); + } +} + +// [perf-deterministic] Two windows appended to ONE sink: the second window's crc +// must cover only its own [codec][len][payload] (start offset != 0), and each +// window's bytes must be identical to building it standalone. This is the guard +// that write_raw / write_pfor use `start = sink->size()` (not absolute 0). +TEST(SniiPrxPodTest, MultipleWindowsInOneSinkAreFramedByteIdentical) { + std::vector flat_a = {1, 2, 3}, freqs_a = {2, 1}; // -> raw + std::vector flat_b = {5, 6}, freqs_b = {1, 1}; // -> pfor + ByteSink combined; + ASSERT_TRUE(build_prx_window_flat(flat_a, freqs_a, /*level=*/0, &combined).ok()); + const size_t boundary = combined.size(); + ASSERT_TRUE(build_prx_window_flat(flat_b, freqs_b, -1, &combined).ok()); + + ByteSink only_a, only_b; + ASSERT_TRUE(build_prx_window_flat(flat_a, freqs_a, /*level=*/0, &only_a).ok()); + ASSERT_TRUE(build_prx_window_flat(flat_b, freqs_b, -1, &only_b).ok()); + ASSERT_EQ(boundary, only_a.size()); + ASSERT_EQ(combined.size(), only_a.size() + only_b.size()); + const std::vector& c = combined.buffer(); + EXPECT_EQ(0, std::memcmp(c.data(), only_a.buffer().data(), only_a.size())); + EXPECT_EQ(0, std::memcmp(c.data() + boundary, only_b.buffer().data(), only_b.size())); + + // Both windows read back sequentially (crc verified per window). + ByteSource src(combined.view()); + PerDoc a, b; + ASSERT_TRUE(read_prx_window(&src, &a).ok()); + ASSERT_TRUE(read_prx_window(&src, &b).ok()); + EXPECT_TRUE(src.eof()); + PerDoc want_a = {{1, 2}, {3}}; + PerDoc want_b = {{5}, {6}}; + EXPECT_EQ(a, want_a); + EXPECT_EQ(b, want_b); +} + +// [functional/error] PRX-CRC-CORRUPT: flipping a byte inside the raw payload (the +// crc-covered [codec][len][payload] span) is detected as Corruption -- confirms +// write_raw's crc range is exactly [codec][len][payload]. +TEST(SniiPrxPodTest, RawFrameCrcCorruptionDetected) { + std::vector flat = {10, 20, 30, 40}; + std::vector freqs = {3, 1}; + ByteSink sink; + ASSERT_TRUE(build_prx_window_flat(flat, freqs, /*level=*/0, &sink).ok()); + ASSERT_EQ(sink.view().data()[0], static_cast(PrxCodec::kRaw)); + + std::vector bytes = sink.buffer(); + ASSERT_GT(bytes.size(), 5U); + bytes[bytes.size() - 5] ^= 0xFF; // last payload byte, just before the 4-byte crc + ByteSource src((Slice(bytes))); + PerDoc out; + Status s = read_prx_window(&src, &out); + EXPECT_TRUE(s.is()) << s.to_string(); +} + +// [perf-deterministic] SF-BYTE: SectionFramer::write bytes equal an independent +// [type][varint64 len][payload][crc] hand-assembly (single-copy framing). +TEST(SniiPrxPodTest, SectionFramerProducesByteIdenticalOutput) { + const std::vector payload_bytes = {0xDE, 0xAD, 0xBE, 0xEF, 0x00, 0x11, 0x22}; + const uint8_t type = 7; + ByteSink sink; + SectionFramer::write(sink, type, Slice(payload_bytes)); + + ByteSink framed; + framed.put_u8(type); + framed.put_varint64(payload_bytes.size()); + framed.put_bytes(Slice(payload_bytes)); + EXPECT_EQ(sink.buffer(), WithCrc(framed)); +} + +// [perf-deterministic] SectionFramer::write into a NON-empty sink frames ONLY its +// own region: the crc covers [type][len][payload] starting at `start`, not the +// pre-existing prefix. Guards the start-offset in the single-copy rewrite. +TEST(SniiPrxPodTest, SectionFramerFramesOnlyItsOwnRegionInNonEmptySink) { + const std::vector prefix = {0xAA, 0xBB}; + const std::vector payload_bytes = {0x01, 0x02, 0x03, 0x04}; + ByteSink sink; + sink.put_bytes(Slice(prefix)); + SectionFramer::write(sink, /*type=*/5, Slice(payload_bytes)); + + ByteSink framed; + framed.put_u8(5); + framed.put_varint64(payload_bytes.size()); + framed.put_bytes(Slice(payload_bytes)); + const std::vector expected_region = WithCrc(framed); + + const std::vector& got = sink.buffer(); + ASSERT_EQ(got.size(), prefix.size() + expected_region.size()); + EXPECT_EQ(0, std::memcmp(got.data(), prefix.data(), prefix.size())); + EXPECT_EQ(0, std::memcmp(got.data() + prefix.size(), expected_region.data(), + expected_region.size())); +} + +// [functional] SF-RT: two sections written into one sink (second at start != 0) +// both read back with the correct type + payload and the crc verifies. +TEST(SniiPrxPodTest, SectionFramerWriteReadRoundTripAcrossSections) { + const std::vector p0 = {1, 2, 3}; + const std::vector p1 = {9, 8, 7, 6, 5}; + ByteSink sink; + SectionFramer::write(sink, /*type=*/3, Slice(p0)); + SectionFramer::write(sink, /*type=*/9, Slice(p1)); + + ByteSource src(sink.view()); + FramedSection s0, s1; + ASSERT_TRUE(SectionFramer::read(src, &s0).ok()); + ASSERT_TRUE(SectionFramer::read(src, &s1).ok()); + EXPECT_TRUE(src.eof()); + EXPECT_EQ(s0.type, 3); + EXPECT_EQ(s1.type, 9); + ASSERT_EQ(s0.payload.size(), p0.size()); + ASSERT_EQ(s1.payload.size(), p1.size()); + EXPECT_EQ(0, std::memcmp(s0.payload.data(), p0.data(), p0.size())); + EXPECT_EQ(0, std::memcmp(s1.payload.data(), p1.data(), p1.size())); +} + +// [functional/error] A corrupted SectionFramer payload byte is caught by the +// section crc on read (crc range == [type][len][payload]). +TEST(SniiPrxPodTest, SectionFramerCrcCorruptionDetected) { + const std::vector payload_bytes = {0x10, 0x20, 0x30, 0x40, 0x50}; + ByteSink sink; + SectionFramer::write(sink, /*type=*/4, Slice(payload_bytes)); + std::vector bytes = sink.buffer(); + ASSERT_GT(bytes.size(), 5U); + bytes[bytes.size() - 5] ^= 0xFF; // last payload byte, before the crc + ByteSource src((Slice(bytes))); + FramedSection out; + Status s = SectionFramer::read(src, &out); + EXPECT_TRUE(s.is()) << s.to_string(); +} + +// [functional/boundary] BITPACK-RT: pfor_encode -> pfor_decode round-trips for +// every value width w in [0,32] at run lengths n = 1, 255, 256 (the kFrqBaseUnit +// boundary). Proves the bitpack reserve (capacity-only) leaves the packed bytes +// bit-exact and the decode path recovers every value. +TEST(SniiPrxPodTest, PforBitpackRoundTripAcrossWidths) { + for (uint8_t w = 0; w <= 32; ++w) { + for (size_t n : {size_t {1}, size_t {255}, size_t {256}}) { + const std::vector run = MakeWidthRun(w, n); + ByteSink sink; + pfor_encode(run.data(), n, &sink); + std::vector decoded(n, 0xFFFFFFFFU); + ByteSource src(sink.view()); + ASSERT_TRUE(pfor_decode(&src, n, decoded.data()).ok()) + << "w=" << static_cast(w) << " n=" << n; + EXPECT_TRUE(src.eof()) << "w=" << static_cast(w) << " n=" << n; + EXPECT_EQ(decoded, run) << "w=" << static_cast(w) << " n=" << n; + } + } +} diff --git a/be/test/storage/index/snii/format/sampled_term_index_test.cpp b/be/test/storage/index/snii/format/sampled_term_index_test.cpp new file mode 100644 index 00000000000000..acb5bf84103357 --- /dev/null +++ b/be/test/storage/index/snii/format/sampled_term_index_test.cpp @@ -0,0 +1,257 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/sampled_term_index.h" + +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/section_framer.h" +#include "storage/index/snii/format/format_constants.h" + +using namespace doris::snii; // NOLINT +using namespace doris::snii::format; // NOLINT +using doris::Status; + +namespace { + +// Build a SampledTermIndex byte buffer from an ordered set of first_terms. +std::vector BuildIndex(const std::vector& first_terms) { + SampledTermIndexBuilder builder; + for (const auto& t : first_terms) { + builder.add_block_first_term(t); + } + ByteSink sink; + builder.finish(&sink); + return sink.buffer(); +} + +// Convenience wrapper to open a reader. +SampledTermIndexReader OpenOrDie(const std::vector& bytes) { + SampledTermIndexReader reader; + Status s = SampledTermIndexReader::open(Slice(bytes), &reader); + EXPECT_TRUE(s.ok()) << s.to_string(); + return reader; +} + +} // namespace + +// Multiple blocks: locate returns the correct ordinal for each first_term. +TEST(SniiSampledTermIndex, LocateExactFirstTermHitsOrdinal) { + const std::vector terms = {"alpha", "delta", "kappa", "omega"}; + auto bytes = BuildIndex(terms); + auto reader = OpenOrDie(bytes); + ASSERT_EQ(reader.n_blocks(), 4U); + + for (uint32_t i = 0; i < terms.size(); ++i) { + bool maybe_present = false; + uint32_t ord = 0xFFFFFFFFU; + ASSERT_TRUE(reader.locate(terms[i], &maybe_present, &ord).ok()); + EXPECT_TRUE(maybe_present) << "term=" << terms[i]; + EXPECT_EQ(ord, i) << "term=" << terms[i]; + } +} + +// target falls between two first_terms → returns the ordinal of the lower one. +TEST(SniiSampledTermIndex, LocateBetweenReturnsLowerOrdinal) { + const std::vector terms = {"alpha", "delta", "kappa", "omega"}; + auto bytes = BuildIndex(terms); + auto reader = OpenOrDie(bytes); + + bool maybe_present = false; + uint32_t ord = 0; + // "echo" is between "delta"(1) and "kappa"(2) → should land in block 1. + ASSERT_TRUE(reader.locate("echo", &maybe_present, &ord).ok()); + EXPECT_TRUE(maybe_present); + EXPECT_EQ(ord, 1U); + + // "beta" is between "alpha"(0) and "delta"(1) → block 0. + ASSERT_TRUE(reader.locate("beta", &maybe_present, &ord).ok()); + EXPECT_TRUE(maybe_present); + EXPECT_EQ(ord, 0U); + + // "zzz" > "omega"(3, the LAST block's first term) routes to the last block: the + // last block can hold terms greater than its first term, so locate must return a + // candidate (find_term then confirms). This is a router, not an exact set. + ASSERT_TRUE(reader.locate("zzz", &maybe_present, &ord).ok()); + EXPECT_TRUE(maybe_present); + EXPECT_EQ(ord, 3U); +} + +// target < min_term → maybe_present=false (not-present signal). +TEST(SniiSampledTermIndex, LocateBelowMinIsOutOfRange) { + const std::vector terms = {"banana", "cherry", "mango"}; + auto bytes = BuildIndex(terms); + auto reader = OpenOrDie(bytes); + + bool maybe_present = true; + uint32_t ord = 12345; + ASSERT_TRUE(reader.locate("apple", &maybe_present, &ord).ok()); + EXPECT_FALSE(maybe_present); +} + +// target > the last block's first term routes to the LAST block (that block may +// contain terms greater than its first term; find_term confirms presence). The +// router must NOT reject such a target, or present tail-of-last-block terms would +// be falsely reported absent. +TEST(SniiSampledTermIndex, LocateAboveLastFirstTermRoutesToLastBlock) { + const std::vector terms = {"banana", "cherry", "mango"}; + auto bytes = BuildIndex(terms); + auto reader = OpenOrDie(bytes); + + bool maybe_present = false; + uint32_t ord = 0; + ASSERT_TRUE(reader.locate("zebra", &maybe_present, &ord).ok()); + EXPECT_TRUE(maybe_present); + EXPECT_EQ(ord, 2U); +} + +// target == min_term / == max_term boundary cases should both hit and be in-range. +TEST(SniiSampledTermIndex, LocateBoundaryTermsInRange) { + const std::vector terms = {"banana", "cherry", "mango"}; + auto bytes = BuildIndex(terms); + auto reader = OpenOrDie(bytes); + + bool maybe_present = false; + uint32_t ord = 0; + ASSERT_TRUE(reader.locate("banana", &maybe_present, &ord).ok()); + EXPECT_TRUE(maybe_present); + EXPECT_EQ(ord, 0U); + + ASSERT_TRUE(reader.locate("mango", &maybe_present, &ord).ok()); + EXPECT_TRUE(maybe_present); + EXPECT_EQ(ord, 2U); +} + +// Single block: min==max==the only first_term. +TEST(SniiSampledTermIndex, SingleBlock) { + const std::vector terms = {"solo"}; + auto bytes = BuildIndex(terms); + auto reader = OpenOrDie(bytes); + ASSERT_EQ(reader.n_blocks(), 1U); + + bool maybe_present = false; + uint32_t ord = 99; + ASSERT_TRUE(reader.locate("solo", &maybe_present, &ord).ok()); + EXPECT_TRUE(maybe_present); + EXPECT_EQ(ord, 0U); + + // "solz" > "solo" routes to the single (last) block: that block may hold terms + // greater than its first term, so locate returns block 0 (find_term confirms). + ASSERT_TRUE(reader.locate("solz", &maybe_present, &ord).ok()); + EXPECT_TRUE(maybe_present); + EXPECT_EQ(ord, 0U); + + // "sola" < "solo" → before the first block, so it cannot exist (out of range). + ASSERT_TRUE(reader.locate("sola", &maybe_present, &ord).ok()); + EXPECT_FALSE(maybe_present); +} + +// Terms sharing a long common prefix (verify front coding round-trip correctness). +TEST(SniiSampledTermIndex, SharedPrefixTermsRoundTrip) { + const std::vector terms = {"international", "internationalize", + "internationalized", "internet", "interoperate"}; + auto bytes = BuildIndex(terms); + auto reader = OpenOrDie(bytes); + ASSERT_EQ(reader.n_blocks(), 5U); + + for (uint32_t i = 0; i < terms.size(); ++i) { + bool maybe_present = false; + uint32_t ord = 0; + ASSERT_TRUE(reader.locate(terms[i], &maybe_present, &ord).ok()); + EXPECT_TRUE(maybe_present) << "term=" << terms[i]; + EXPECT_EQ(ord, i) << "term=" << terms[i]; + } + + // "internationalizes" is between idx2 and idx3 → lower ordinal 2. + bool maybe_present = false; + uint32_t ord = 0; + ASSERT_TRUE(reader.locate("internationalizes", &maybe_present, &ord).ok()); + EXPECT_TRUE(maybe_present); + EXPECT_EQ(ord, 2U); +} + +// Terms containing null bytes / high-bit bytes, compared by unsigned byte order. +TEST(SniiSampledTermIndex, BinarySafeTerms) { + std::vector terms; + terms.emplace_back("\x01\x00z", 3); + terms.emplace_back("\x80\x00", 2); + terms.emplace_back("\xFF", 1); + auto bytes = BuildIndex(terms); + auto reader = OpenOrDie(bytes); + ASSERT_EQ(reader.n_blocks(), 3U); + + bool maybe_present = false; + uint32_t ord = 0; + ASSERT_TRUE(reader.locate(std::string("\x80\x00", 2), &maybe_present, &ord).ok()); + EXPECT_TRUE(maybe_present); + EXPECT_EQ(ord, 1U); +} + +// The first byte is the SectionFramer type and must be kSampledTermIndex. +TEST(SniiSampledTermIndex, FramedAsSampledTermIndexType) { + auto bytes = BuildIndex({"alpha", "beta"}); + ASSERT_GE(bytes.size(), 1U); + EXPECT_EQ(bytes[0], static_cast(SectionType::kSampledTermIndex)); +} + +// CRC checksum: flipping one byte in the payload → open returns Corruption. +TEST(SniiSampledTermIndex, DetectsCorruption) { + auto bytes = BuildIndex({"alpha", "delta", "kappa"}); + ASSERT_GE(bytes.size(), 4U); + bytes[3] ^= 0xFF; // Flip a byte in the payload region (skip the type+len prefix) + SampledTermIndexReader reader; + Status s = SampledTermIndexReader::open(Slice(bytes), &reader); + EXPECT_FALSE(s.ok()) << s.to_string(); +} + +// Truncation: drop the trailing byte → open fails. +TEST(SniiSampledTermIndex, DetectsTruncation) { + auto bytes = BuildIndex({"alpha", "delta", "kappa"}); + bytes.pop_back(); + SampledTermIndexReader reader; + Status s = SampledTermIndexReader::open(Slice(bytes), &reader); + EXPECT_FALSE(s.ok()); +} + +// A wrong section type should be rejected. +TEST(SniiSampledTermIndex, WrongSectionTypeRejected) { + ByteSink sink; + const uint8_t p[] = {0, 0}; + SectionFramer::write(sink, static_cast(SectionType::kXFilter), Slice(p, 2)); + SampledTermIndexReader reader; + Status s = SampledTermIndexReader::open(sink.view(), &reader); + EXPECT_FALSE(s.ok()); +} + +// Empty builder (n_blocks=0): valid build, locate on any term is out of range. +TEST(SniiSampledTermIndex, EmptyIndexLocateOutOfRange) { + auto bytes = BuildIndex({}); + auto reader = OpenOrDie(bytes); + ASSERT_EQ(reader.n_blocks(), 0U); + + bool maybe_present = true; + uint32_t ord = 7; + ASSERT_TRUE(reader.locate("anything", &maybe_present, &ord).ok()); + EXPECT_FALSE(maybe_present); +} diff --git a/be/test/storage/index/snii/format/stats_block_test.cpp b/be/test/storage/index/snii/format/stats_block_test.cpp new file mode 100644 index 00000000000000..d03dad3516f61e --- /dev/null +++ b/be/test/storage/index/snii/format/stats_block_test.cpp @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/stats_block.h" + +#include + +#include + +#include "common/status.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/section_framer.h" +#include "storage/index/snii/format/format_constants.h" + +using namespace doris::snii; +using namespace doris::snii::format; +using doris::Status; + +namespace { + +// Encode then decode; assert every field is equal one by one. +void ExpectRoundTrip(const StatsBlock& in) { + ByteSink sink; + encode_stats_block(in, &sink); + ByteSource src(sink.view()); + StatsBlock out {}; + ASSERT_TRUE(decode_stats_block(&src, &out).ok()); + EXPECT_EQ(out.doc_count, in.doc_count); + EXPECT_EQ(out.indexed_doc_count, in.indexed_doc_count); + EXPECT_EQ(out.term_count, in.term_count); + EXPECT_EQ(out.sum_total_term_freq, in.sum_total_term_freq); + EXPECT_EQ(out.null_count, in.null_count); + EXPECT_TRUE(src.eof()); +} + +} // namespace + +TEST(SniiStatsBlock, RoundTripTypicalValues) { + StatsBlock sb {}; + sb.doc_count = 1000; + sb.indexed_doc_count = 980; + sb.term_count = 54321; + sb.sum_total_term_freq = 1234567; + sb.null_count = 20; + ExpectRoundTrip(sb); +} + +TEST(SniiStatsBlock, RoundTripZeroes) { + StatsBlock sb {}; // All zeros: empty segment is valid + ExpectRoundTrip(sb); +} + +TEST(SniiStatsBlock, RoundTripNear2Pow63) { + StatsBlock sb {}; + const uint64_t kBig = (1ULL << 63) - 1; // Large-value boundary + sb.doc_count = kBig; + sb.indexed_doc_count = kBig - 1; + sb.term_count = (1ULL << 63) + 7; // High bit set must also round-trip correctly + sb.sum_total_term_freq = UINT64_MAX; + sb.null_count = (1ULL << 62); + ExpectRoundTrip(sb); +} + +TEST(SniiStatsBlock, FramedAsStatsBlockType) { + StatsBlock sb {}; + sb.doc_count = 7; + ByteSink sink; + encode_stats_block(sb, &sink); + // First byte is the SectionFramer type field and must equal kStatsBlock. + ASSERT_GE(sink.size(), 1U); + EXPECT_EQ(sink.buffer()[0], static_cast(SectionType::kStatsBlock)); +} + +TEST(SniiStatsBlock, DetectsCorruption) { + StatsBlock sb {}; + sb.doc_count = 42; + sb.indexed_doc_count = 40; + sb.term_count = 9; + sb.sum_total_term_freq = 1000; + sb.null_count = 2; + ByteSink sink; + encode_stats_block(sb, &sink); + + auto bytes = sink.buffer(); + // Flip one byte in the payload area (skip the type+len prefix); CRC must detect the corruption. + ASSERT_GE(bytes.size(), 3U); + bytes[2] ^= 0xFF; + Slice corrupted(bytes); + ByteSource src(corrupted); + StatsBlock out {}; + // SectionFramer reports CRC failure as INVERTED_INDEX_FILE_CORRUPTED. + EXPECT_TRUE( + decode_stats_block(&src, &out).is()); +} + +TEST(SniiStatsBlock, DetectsTruncation) { + StatsBlock sb {}; + sb.doc_count = 100; + ByteSink sink; + encode_stats_block(sb, &sink); + auto bytes = sink.buffer(); + bytes.pop_back(); // Truncate the last byte + Slice truncated(bytes); + ByteSource src(truncated); + StatsBlock out {}; + EXPECT_FALSE(decode_stats_block(&src, &out).ok()); +} + +TEST(SniiStatsBlock, WrongSectionTypeRejected) { + // Write a non-StatsBlock section via the framer; decode must reject it. + ByteSink sink; + const uint8_t p[] = {1, 2, 3}; + SectionFramer::write(sink, static_cast(SectionType::kXFilter), Slice(p, 3)); + ByteSource src(sink.view()); + StatsBlock out {}; + Status s = decode_stats_block(&src, &out); + EXPECT_FALSE(s.ok()); +} diff --git a/be/test/storage/index/snii/format/tail_meta_region_test.cpp b/be/test/storage/index/snii/format/tail_meta_region_test.cpp new file mode 100644 index 00000000000000..da03a8a1f4b8c5 --- /dev/null +++ b/be/test/storage/index/snii/format/tail_meta_region_test.cpp @@ -0,0 +1,130 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/tail_meta_region.h" + +#include + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" + +using namespace doris::snii; +using doris::Status; +using doris::snii::format::TailMetaRegionBuilder; +using doris::snii::format::TailMetaRegionReader; + +namespace { + +std::vector Pattern(size_t n, uint8_t base) { + std::vector v(n); + for (size_t i = 0; i < n; ++i) { + v[i] = static_cast(base + i); + } + return v; +} + +} // namespace + +// Round-trip: add per-index meta blocks (opaque payloads), then locate each by +// (index_id, suffix) and verify the exact bytes come back. +TEST(SniiTailMetaRegion, RoundTripLocate) { + auto a = Pattern(10, 0); + auto b = Pattern(20, 100); + + TailMetaRegionBuilder builder; + builder.add_index(7, "", Slice(a)); + builder.add_index(7, "suffix", Slice(b)); + + ByteSink sink; + builder.finish(&sink); + + TailMetaRegionReader reader; + ASSERT_TRUE(TailMetaRegionReader::open(sink.view(), &reader).ok()); + EXPECT_EQ(reader.n_logical_indexes(), 2U); + + bool found = false; + Slice block; + ASSERT_TRUE(reader.find(7, "", &found, &block).ok()); + ASSERT_TRUE(found); + ASSERT_EQ(block.size(), a.size()); + EXPECT_EQ(block[0], 0U); + EXPECT_EQ(block[9], 9U); + + ASSERT_TRUE(reader.find(7, "suffix", &found, &block).ok()); + ASSERT_TRUE(found); + ASSERT_EQ(block.size(), b.size()); + EXPECT_EQ(block[0], 100U); + + // Missing key. + ASSERT_TRUE(reader.find(99, "", &found, &block).ok()); + EXPECT_FALSE(found); +} + +// A single-index region round-trips. +TEST(SniiTailMetaRegion, SingleIndex) { + auto a = Pattern(5, 42); + TailMetaRegionBuilder builder; + builder.add_index(1, "x", Slice(a)); + ByteSink sink; + builder.finish(&sink); + + TailMetaRegionReader reader; + ASSERT_TRUE(TailMetaRegionReader::open(sink.view(), &reader).ok()); + EXPECT_EQ(reader.n_logical_indexes(), 1U); + bool found = false; + Slice block; + ASSERT_TRUE(reader.find(1, "x", &found, &block).ok()); + ASSERT_TRUE(found); + EXPECT_EQ(block[0], 42U); +} + +// Corruption of the region (a byte inside the meta_region_checksum coverage) is +// detected at open(). +TEST(SniiTailMetaRegion, DetectsCorruption) { + auto a = Pattern(10, 0); + TailMetaRegionBuilder builder; + builder.add_index(7, "", Slice(a)); + ByteSink sink; + builder.finish(&sink); + + std::vector bytes = sink.buffer(); + ASSERT_GT(bytes.size(), 8U); + bytes[bytes.size() / 2] ^= 0xFF; // flip a covered byte + + TailMetaRegionReader reader; + Status s = TailMetaRegionReader::open(Slice(bytes), &reader); + EXPECT_TRUE(s.is()); +} + +// Empty region (no logical indexes) round-trips. +TEST(SniiTailMetaRegion, EmptyRegion) { + TailMetaRegionBuilder builder; + ByteSink sink; + builder.finish(&sink); + TailMetaRegionReader reader; + ASSERT_TRUE(TailMetaRegionReader::open(sink.view(), &reader).ok()); + EXPECT_EQ(reader.n_logical_indexes(), 0U); + bool found = true; + Slice block; + ASSERT_TRUE(reader.find(1, "", &found, &block).ok()); + EXPECT_FALSE(found); +} diff --git a/be/test/storage/index/snii/format/tail_pointer_test.cpp b/be/test/storage/index/snii/format/tail_pointer_test.cpp new file mode 100644 index 00000000000000..97d886dda0587d --- /dev/null +++ b/be/test/storage/index/snii/format/tail_pointer_test.cpp @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/format/tail_pointer.h" + +#include + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/format_constants.h" + +using namespace doris::snii; +using namespace doris::snii::format; +using doris::Status; + +namespace { + +// Encode then decode through the fixed-size on-disk layout and assert that every +// field round-trips and the encoded length matches the advertised fixed size. +void ExpectRoundTrip(const TailPointer& in) { + ByteSink sink; + ASSERT_TRUE(encode_tail_pointer(in, &sink).ok()); + EXPECT_EQ(sink.size(), tail_pointer_size()); + TailPointer out {}; + ASSERT_TRUE(decode_tail_pointer(sink.view(), &out).ok()); + EXPECT_EQ(out.meta_region_offset, in.meta_region_offset); + EXPECT_EQ(out.meta_region_length, in.meta_region_length); + EXPECT_EQ(out.hot_off, in.hot_off); + EXPECT_EQ(out.meta_region_checksum, in.meta_region_checksum); + EXPECT_EQ(out.bootstrap_header_checksum, in.bootstrap_header_checksum); +} + +TailPointer MakeTypical() { + TailPointer tp {}; + tp.meta_region_offset = 0x0011223344556677ULL; + tp.meta_region_length = 4096; + tp.hot_off = 0x00000000DEADBEEFULL; + tp.meta_region_checksum = 0xABCD1234U; + tp.bootstrap_header_checksum = 0x0BADC0DEU; + return tp; +} + +} // namespace + +TEST(SniiTailPointer, RoundTripTypicalValues) { + ExpectRoundTrip(MakeTypical()); +} + +TEST(SniiTailPointer, RoundTripZeroes) { + TailPointer tp {}; // empty / absent hot region (hot_off == 0) must round-trip. + ExpectRoundTrip(tp); +} + +TEST(SniiTailPointer, RoundTripMaxValues) { + TailPointer tp {}; + tp.meta_region_offset = UINT64_MAX; + tp.meta_region_length = UINT64_MAX - 1; + tp.hot_off = (1ULL << 63) | 7ULL; // high bit set must survive fixed64. + tp.meta_region_checksum = UINT32_MAX; + tp.bootstrap_header_checksum = UINT32_MAX - 3; + ExpectRoundTrip(tp); +} + +// The advertised fixed size must equal the actual encoded byte count, so a reader +// can read exactly the last tail_pointer_size() bytes of the file. +TEST(SniiTailPointer, FixedSizeMatchesEncodedLength) { + ByteSink sink; + ASSERT_TRUE(encode_tail_pointer(MakeTypical(), &sink).ok()); + EXPECT_EQ(sink.size(), tail_pointer_size()); +} + +// The on-disk tail_pointer_size byte must record the same fixed size value. +TEST(SniiTailPointer, EmbeddedSizeByteMatchesFixedSize) { + ByteSink sink; + ASSERT_TRUE(encode_tail_pointer(MakeTypical(), &sink).ok()); + auto bytes = sink.buffer(); + // Layout: ...[u8 tail_pointer_size][u32 tail_checksum]; the size byte is the + // 5th byte from the end. + ASSERT_GE(bytes.size(), 5U); + EXPECT_EQ(bytes[bytes.size() - 5], static_cast(tail_pointer_size())); +} + +TEST(SniiTailPointer, WrongMagicRejected) { + ByteSink sink; + ASSERT_TRUE(encode_tail_pointer(MakeTypical(), &sink).ok()); + auto bytes = sink.buffer(); + ASSERT_GE(bytes.size(), 4U); + bytes[0] ^= 0xFFU; // corrupt the magic. + // Recompute the trailing tail_checksum is NOT done: but even if we wanted to + // isolate the magic check, corrupting magic alone must be rejected. + TailPointer out {}; + Status s = decode_tail_pointer(Slice(bytes), &out); + EXPECT_FALSE(s.ok()); +} + +TEST(SniiTailPointer, TailChecksumMismatchRejected) { + ByteSink sink; + ASSERT_TRUE(encode_tail_pointer(MakeTypical(), &sink).ok()); + auto bytes = sink.buffer(); + // Flip a payload byte (the meta_region_offset region) without fixing the + // trailing tail_checksum; the checksum must catch it. + ASSERT_GE(bytes.size(), 10U); + bytes[8] ^= 0xFFU; + TailPointer out {}; + Status s = decode_tail_pointer(Slice(bytes), &out); + EXPECT_FALSE(s.ok()); +} + +TEST(SniiTailPointer, TruncatedInputRejected) { + ByteSink sink; + ASSERT_TRUE(encode_tail_pointer(MakeTypical(), &sink).ok()); + auto bytes = sink.buffer(); + bytes.pop_back(); // one byte short of the fixed size. + TailPointer out {}; + Status s = decode_tail_pointer(Slice(bytes), &out); + EXPECT_FALSE(s.ok()); +} + +TEST(SniiTailPointer, EmptyInputRejected) { + TailPointer out {}; + Status s = decode_tail_pointer(Slice(), &out); + EXPECT_FALSE(s.ok()); +} + +// A reader may hand decode_tail_pointer more than the fixed size (e.g. when the +// file is short and it read extra leading bytes). Decoding the exact-size suffix +// must succeed; passing a longer buffer must be rejected as not the fixed size. +TEST(SniiTailPointer, LongerThanFixedSizeRejected) { + ByteSink sink; + ASSERT_TRUE(encode_tail_pointer(MakeTypical(), &sink).ok()); + auto bytes = sink.buffer(); + bytes.push_back(0x00); // one trailing extra byte. + TailPointer out {}; + Status s = decode_tail_pointer(Slice(bytes), &out); + EXPECT_FALSE(s.ok()); +} diff --git a/be/test/storage/index/snii/io/batch_range_fetcher_test.cpp b/be/test/storage/index/snii/io/batch_range_fetcher_test.cpp new file mode 100644 index 00000000000000..faab93419868e5 --- /dev/null +++ b/be/test/storage/index/snii/io/batch_range_fetcher_test.cpp @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/io/batch_range_fetcher.h" + +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/io/metered_file_reader.h" + +using namespace doris::snii; +using doris::Status; +using doris::snii::io::BatchRangeFetcher; +using doris::snii::io::LocalFileReader; +using doris::snii::io::LocalFileWriter; +using doris::snii::io::MeteredFileReader; + +namespace { + +std::string MakeRampFile() { + const std::string path = "/tmp/snii_brf_ramp.bin"; + LocalFileWriter w; + EXPECT_TRUE(w.open(path).ok()); + std::vector data(256); + for (int i = 0; i < 256; ++i) { + data[i] = static_cast(i); + } + EXPECT_TRUE(w.append(Slice(data)).ok()); + EXPECT_TRUE(w.finalize().ok()); + return path; +} + +} // namespace + +// Disjoint ranges: each handle returns its own bytes; the whole fetch is one +// serial round on the metered reader. +TEST(SniiBatchRangeFetcher, DisjointRanges) { + LocalFileReader inner; + ASSERT_TRUE(inner.open(MakeRampFile()).ok()); + MeteredFileReader m(&inner, 16); + + BatchRangeFetcher f(&m); + size_t h0 = f.add(0, 4); + size_t h1 = f.add(100, 4); + size_t h2 = f.add(200, 4); + ASSERT_TRUE(f.fetch().ok()); + + EXPECT_EQ(f.get(h0)[0], 0U); + EXPECT_EQ(f.get(h1)[0], 100U); + EXPECT_EQ(f.get(h2)[3], 203U); + EXPECT_EQ(m.metrics().serial_rounds, 1U); // single batched round +} + +// Overlapping requests coalesce into one physical read; bytes still map back. +TEST(SniiBatchRangeFetcher, OverlappingCoalesced) { + LocalFileReader inner; + ASSERT_TRUE(inner.open(MakeRampFile()).ok()); + MeteredFileReader m(&inner, 16); + + BatchRangeFetcher f(&m); + size_t h0 = f.add(0, 4); // [0,4) + size_t h1 = f.add(2, 4); // [2,6) overlaps + size_t h2 = f.add(5, 3); // [5,8) adjacent/overlaps + ASSERT_TRUE(f.fetch().ok()); + + EXPECT_EQ(f.get(h0)[0], 0U); + EXPECT_EQ(f.get(h1)[0], 2U); + EXPECT_EQ(f.get(h2)[0], 5U); + EXPECT_EQ(f.get(h2)[2], 7U); + // Coalesced into a single physical read -> one read_at_call on the metered reader. + EXPECT_EQ(m.metrics().read_at_calls, 1U); +} + +// clear() lets the fetcher be reused for a new round. +TEST(SniiBatchRangeFetcher, ClearAndReuse) { + LocalFileReader inner; + ASSERT_TRUE(inner.open(MakeRampFile()).ok()); + MeteredFileReader m(&inner, 16); + + BatchRangeFetcher f(&m); + f.add(0, 4); + ASSERT_TRUE(f.fetch().ok()); + f.clear(); + EXPECT_EQ(f.pending(), 0U); + size_t h = f.add(64, 8); + ASSERT_TRUE(f.fetch().ok()); + EXPECT_EQ(f.get(h)[0], 64U); +} + +TEST(SniiBatchRangeFetcher, RejectsOverflowingRangeEnd) { + LocalFileReader inner; + ASSERT_TRUE(inner.open(MakeRampFile()).ok()); + + BatchRangeFetcher f(&inner); + f.add(std::numeric_limits::max() - 1, 8); + const Status st = f.fetch(); + // Integrated fetch() reports range-end overflow as INVERTED_INDEX_FILE_CORRUPTED. + EXPECT_TRUE(st.is()) << st.to_string(); +} + +TEST(SniiBatchRangeFetcher, RejectsNullReaderAtFetch) { + BatchRangeFetcher f(nullptr); + f.add(0, 1); + const Status st = f.fetch(); + EXPECT_TRUE(st.is()) << st.to_string(); +} diff --git a/be/test/storage/index/snii/io/local_file_test.cpp b/be/test/storage/index/snii/io/local_file_test.cpp new file mode 100644 index 00000000000000..4b575c3e7739dc --- /dev/null +++ b/be/test/storage/index/snii/io/local_file_test.cpp @@ -0,0 +1,143 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/io/local_file.h" + +#include + +#include +#include +#include + +#include "common/status.h" + +using namespace doris::snii; +using doris::snii::io::LocalFileReader; +using doris::snii::io::LocalFileWriter; + +namespace { + +std::string TempPath(const char* name) { + return std::string("/tmp/snii_io_test_") + name + ".bin"; +} + +} // namespace + +TEST(SniiLocalFile, AppendThenReadBack) { + const std::string path = TempPath("append"); + { + LocalFileWriter w; + ASSERT_TRUE(w.open(path).ok()); + const uint8_t a[] = {1, 2, 3, 4}; + const uint8_t b[] = {5, 6, 7, 8}; + ASSERT_TRUE(w.append(Slice(a, 4)).ok()); + ASSERT_TRUE(w.append(Slice(b, 4)).ok()); + EXPECT_EQ(w.bytes_written(), 8U); + ASSERT_TRUE(w.finalize().ok()); + } + + LocalFileReader r; + ASSERT_TRUE(r.open(path).ok()); + EXPECT_EQ(r.size(), 8U); + std::vector out; + ASSERT_TRUE(r.read_at(2, 4, &out).ok()); + ASSERT_EQ(out.size(), 4U); + EXPECT_EQ(out[0], 3U); + EXPECT_EQ(out[3], 6U); +} + +TEST(SniiLocalFile, ReadPastEndFails) { + const std::string path = TempPath("past_end"); + { + LocalFileWriter w; + ASSERT_TRUE(w.open(path).ok()); + const uint8_t a[] = {1, 2, 3}; + ASSERT_TRUE(w.append(Slice(a, 3)).ok()); + ASSERT_TRUE(w.finalize().ok()); + } + LocalFileReader r; + ASSERT_TRUE(r.open(path).ok()); + std::vector out; + EXPECT_FALSE(r.read_at(2, 10, &out).ok()); +} + +TEST(SniiLocalFile, OpenMissingFails) { + LocalFileReader r; + EXPECT_FALSE(r.open("/tmp/snii_io_test_does_not_exist_zzz.bin").ok()); +} + +// The buffered writer must produce byte-identical output regardless of how the +// stream is chopped: tiny appends that overflow the 256 KiB buffer, a single +// over-buffer append (the direct-to-fd fast path), and mixes of the two. +TEST(SniiLocalFile, BufferedAppendsByteIdentical) { + const std::string path = TempPath("buffered"); + // Build a deterministic reference stream larger than the 256 KiB buffer so the + // buffer flushes several times and at least one big append bypasses it. + std::vector expected; + expected.reserve(1U << 20); + uint32_t x = 0xC0FFEEU; + auto next = [&]() { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + return static_cast(x); + }; + for (size_t i = 0; i < (1U << 20); ++i) { + expected.push_back(next()); + } + + { + LocalFileWriter w; + ASSERT_TRUE(w.open(path).ok()); + size_t off = 0; + // Phase 1: many small appends (700 B) -> repeated buffer overflow/flush. + while (off + 700 <= 600U * 1024) { + ASSERT_TRUE(w.append(Slice(expected.data() + off, 700)).ok()); + off += 700; + } + // Phase 2: one append larger than the buffer (direct-to-fd path). + const size_t big = 300U * 1024; + ASSERT_TRUE(w.append(Slice(expected.data() + off, big)).ok()); + off += big; + // Phase 3: drain the remainder in a final small append. + ASSERT_TRUE(w.append(Slice(expected.data() + off, expected.size() - off)).ok()); + EXPECT_EQ(w.bytes_written(), expected.size()); + ASSERT_TRUE(w.finalize().ok()); + } + + LocalFileReader r; + ASSERT_TRUE(r.open(path).ok()); + ASSERT_EQ(r.size(), expected.size()); + std::vector got; + ASSERT_TRUE(r.read_at(0, expected.size(), &got).ok()); + EXPECT_EQ(got, expected); +} + +// Empty appends are no-ops and an unused open->finalize yields an empty file. +TEST(SniiLocalFile, EmptyAppendsAndFinalize) { + const std::string path = TempPath("empty_buf"); + { + LocalFileWriter w; + ASSERT_TRUE(w.open(path).ok()); + ASSERT_TRUE(w.append(Slice(nullptr, 0)).ok()); + EXPECT_EQ(w.bytes_written(), 0U); + ASSERT_TRUE(w.finalize().ok()); + } + LocalFileReader r; + ASSERT_TRUE(r.open(path).ok()); + EXPECT_EQ(r.size(), 0U); +} diff --git a/be/test/storage/index/snii/io/metered_file_reader_test.cpp b/be/test/storage/index/snii/io/metered_file_reader_test.cpp new file mode 100644 index 00000000000000..a9fc344fb56d04 --- /dev/null +++ b/be/test/storage/index/snii/io/metered_file_reader_test.cpp @@ -0,0 +1,181 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/io/metered_file_reader.h" + +#include + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/io/local_file.h" + +using namespace doris::snii; +using doris::Status; +using doris::snii::io::LocalFileReader; +using doris::snii::io::LocalFileWriter; +using doris::snii::io::MeteredFileReader; +using doris::snii::io::Range; + +namespace { + +// Writes 256 bytes (byte[i] = i) to a temp file and returns its path. +std::string MakeRampFile() { + const std::string path = "/tmp/snii_metered_ramp.bin"; + LocalFileWriter w; + EXPECT_TRUE(w.open(path).ok()); + std::vector data(256); + for (int i = 0; i < 256; ++i) { + data[i] = static_cast(i); + } + EXPECT_TRUE(w.append(Slice(data)).ok()); + EXPECT_TRUE(w.finalize().ok()); + return path; +} + +} // namespace + +// Single reads: first read to a block is a cache miss (1 round, 1 GET, 1 block of +// remote bytes); a second read to the same 16-byte block is a hit (no new round). +TEST(SniiMeteredFileReader, SingleReadCacheAccounting) { + LocalFileReader inner; + ASSERT_TRUE(inner.open(MakeRampFile()).ok()); + MeteredFileReader m(&inner, /*block_size=*/16); + + std::vector out; + ASSERT_TRUE(m.read_at(0, 4, &out).ok()); + EXPECT_EQ(out[0], 0U); + EXPECT_EQ(out[3], 3U); + EXPECT_EQ(m.metrics().read_at_calls, 1U); + EXPECT_EQ(m.metrics().serial_rounds, 1U); + EXPECT_EQ(m.metrics().range_gets, 1U); + EXPECT_EQ(m.metrics().remote_bytes, 16U); + + // Same block (offset 8..11) -> cache hit. + ASSERT_TRUE(m.read_at(8, 4, &out).ok()); + EXPECT_EQ(m.metrics().read_at_calls, 2U); + EXPECT_EQ(m.metrics().serial_rounds, 1U); // unchanged + EXPECT_EQ(m.metrics().range_gets, 1U); + EXPECT_EQ(m.metrics().remote_bytes, 16U); + + // Different block (offset 20 -> block 1) -> miss. + ASSERT_TRUE(m.read_at(20, 4, &out).ok()); + EXPECT_EQ(out[0], 20U); + EXPECT_EQ(m.metrics().serial_rounds, 2U); + EXPECT_EQ(m.metrics().range_gets, 2U); + EXPECT_EQ(m.metrics().remote_bytes, 32U); +} + +// A read spanning 3 contiguous blocks is one round and one coalesced GET. +TEST(SniiMeteredFileReader, SpanMultipleBlocksCoalesced) { + LocalFileReader inner; + ASSERT_TRUE(inner.open(MakeRampFile()).ok()); + MeteredFileReader m(&inner, 16); + + std::vector out; + ASSERT_TRUE(m.read_at(0, 40, &out).ok()); // blocks 0,1,2 + EXPECT_EQ(out.size(), 40U); + EXPECT_EQ(m.metrics().serial_rounds, 1U); + EXPECT_EQ(m.metrics().range_gets, 1U); + EXPECT_EQ(m.metrics().remote_bytes, 48U); // 3 * 16 +} + +// A batch of reads to non-adjacent blocks: one serial round, one GET per run. +TEST(SniiMeteredFileReader, BatchNonAdjacent) { + LocalFileReader inner; + ASSERT_TRUE(inner.open(MakeRampFile()).ok()); + MeteredFileReader m(&inner, 16); + + std::vector ranges = {{.offset = 0, .len = 4}, + {.offset = 100, .len = 4}, + {.offset = 200, .len = 4}}; // blocks 0, 6, 12 + std::vector> outs; + ASSERT_TRUE(m.read_batch(ranges, &outs).ok()); + ASSERT_EQ(outs.size(), 3U); + EXPECT_EQ(outs[1][0], 100U); + EXPECT_EQ(m.metrics().read_at_calls, 3U); + EXPECT_EQ(m.metrics().serial_rounds, 1U); // one batch = one round + EXPECT_EQ(m.metrics().range_gets, 3U); // 3 disjoint runs + EXPECT_EQ(m.metrics().remote_bytes, 48U); +} + +// A batch of reads to adjacent blocks coalesces into a single GET. +TEST(SniiMeteredFileReader, BatchAdjacentCoalesced) { + LocalFileReader inner; + ASSERT_TRUE(inner.open(MakeRampFile()).ok()); + MeteredFileReader m(&inner, 16); + + std::vector ranges = {{.offset = 0, .len = 4}, + {.offset = 16, .len = 4}, + {.offset = 32, .len = 4}}; // blocks 0,1,2 + std::vector> outs; + ASSERT_TRUE(m.read_batch(ranges, &outs).ok()); + EXPECT_EQ(m.metrics().read_at_calls, 3U); + EXPECT_EQ(m.metrics().serial_rounds, 1U); + EXPECT_EQ(m.metrics().range_gets, 1U); // coalesced + EXPECT_EQ(m.metrics().remote_bytes, 48U); +} + +// reset_metrics clears both counters and the resident cache (cold query). +TEST(SniiMeteredFileReader, ResetClearsCacheAndCounters) { + LocalFileReader inner; + ASSERT_TRUE(inner.open(MakeRampFile()).ok()); + MeteredFileReader m(&inner, 16); + + std::vector out; + ASSERT_TRUE(m.read_at(0, 4, &out).ok()); + m.reset_metrics(); + EXPECT_EQ(m.metrics().read_at_calls, 0U); + EXPECT_EQ(m.metrics().serial_rounds, 0U); + // Cache cleared -> same read misses again. + ASSERT_TRUE(m.read_at(0, 4, &out).ok()); + EXPECT_EQ(m.metrics().serial_rounds, 1U); + EXPECT_EQ(m.metrics().remote_bytes, 16U); +} + +TEST(SniiMeteredFileReader, InvalidRangeDoesNotPolluteMetrics) { + LocalFileReader inner; + ASSERT_TRUE(inner.open(MakeRampFile()).ok()); + MeteredFileReader m(&inner, 16); + + std::vector out; + const Status st = m.read_at(250, 16, &out); + EXPECT_TRUE(st.is()) << st.to_string(); + EXPECT_EQ(m.metrics().read_at_calls, 0U); + EXPECT_EQ(m.metrics().serial_rounds, 0U); + EXPECT_EQ(m.metrics().range_gets, 0U); + EXPECT_EQ(m.metrics().remote_bytes, 0U); + EXPECT_EQ(m.metrics().total_request_bytes, 0U); +} + +TEST(SniiMeteredFileReader, InvalidBatchRangeDoesNotPolluteMetrics) { + LocalFileReader inner; + ASSERT_TRUE(inner.open(MakeRampFile()).ok()); + MeteredFileReader m(&inner, 16); + + std::vector> outs; + const Status st = + m.read_batch({Range {.offset = 0, .len = 4}, Range {.offset = 250, .len = 16}}, &outs); + EXPECT_TRUE(st.is()) << st.to_string(); + EXPECT_EQ(m.metrics().read_at_calls, 0U); + EXPECT_EQ(m.metrics().serial_rounds, 0U); + EXPECT_EQ(m.metrics().range_gets, 0U); + EXPECT_EQ(m.metrics().remote_bytes, 0U); + EXPECT_EQ(m.metrics().total_request_bytes, 0U); +} diff --git a/be/test/storage/index/snii/query/boolean_query_test.cpp b/be/test/storage/index/snii/query/boolean_query_test.cpp new file mode 100644 index 00000000000000..f89533442b1c10 --- /dev/null +++ b/be/test/storage/index/snii/query/boolean_query_test.cpp @@ -0,0 +1,337 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/boolean_query.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/io/metered_file_reader.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +using namespace doris::snii; +using namespace doris::snii::reader; +using namespace doris::snii::writer; +using doris::Status; + +namespace { + +std::string TempPath() { + static int counter = 0; + return "/tmp/snii_boolean_query_" + std::to_string(getpid()) + "_" + std::to_string(counter++) + + ".idx"; +} + +struct Corpus { + uint32_t doc_count = 900; + std::vector> docs; + + std::vector term_docs(const std::string& term) const { + std::vector out; + for (uint32_t d = 0; d < docs.size(); ++d) { + if (std::find(docs[d].begin(), docs[d].end(), term) != docs[d].end()) { + out.push_back(d); + } + } + return out; + } + + std::vector any_docs(const std::vector& terms) const { + std::set ids; + for (const auto& term : terms) { + const std::vector docs_for_term = term_docs(term); + ids.insert(docs_for_term.begin(), docs_for_term.end()); + } + return {ids.begin(), ids.end()}; + } +}; + +Corpus BuildCorpus() { + Corpus c; + c.docs.resize(c.doc_count); + for (uint32_t d = 0; d < c.doc_count; ++d) { + std::vector& toks = c.docs[d]; + if (d < 700) { + toks.emplace_back("aa_hot"); + } + if (d < 700) { + toks.emplace_back("aa_warm"); + } + if (d < 700) { + toks.emplace_back("aa_loud"); + } + if (d % 17 == 0) { + toks.emplace_back("aa_rare"); + } + if (d % 41 == 0) { + toks.emplace_back("aa_sparse"); + } + char filler[16]; + std::snprintf(filler, sizeof(filler), "zz_%03u", d % 200); + toks.emplace_back(filler); + } + return c; +} + +Corpus BuildDenseAndCorpus() { + Corpus c; + c.doc_count = 9000; + c.docs.resize(c.doc_count); + for (uint32_t d = 0; d < c.doc_count; ++d) { + std::vector& toks = c.docs[d]; + toks.emplace_back("aa_dense"); + if (d % 1024 == 0) { + toks.emplace_back("aa_rare_driver"); + } + char filler[16]; + std::snprintf(filler, sizeof(filler), "zz_%03u", d % 200); + toks.emplace_back(filler); + } + return c; +} + +void WriteCorpus(const Corpus& c, const std::string& path) { + SpimiTermBuffer buf(/*has_positions=*/false); + for (uint32_t d = 0; d < c.docs.size(); ++d) { + const auto& toks = c.docs[d]; + for (uint32_t pos = 0; pos < toks.size(); ++pos) { + buf.add_token(toks[pos], d, pos); + } + } + + SniiIndexInput in; + in.index_id = 1; + in.index_suffix = "body"; + in.config = doris::snii::format::IndexConfig::kDocsOnly; + in.doc_count = c.doc_count; + in.terms = buf.finalize_sorted(); + in.target_dict_block_bytes = 256; + + io::LocalFileWriter writer; + ASSERT_TRUE(writer.open(path).ok()); + SniiCompoundWriter compound(&writer); + ASSERT_TRUE(compound.add_logical_index(in).ok()); + ASSERT_TRUE(compound.finish().ok()); +} + +LogicalIndexReader OpenIndex(io::LocalFileReader* file, SniiSegmentReader* segment, + const std::string& path) { + EXPECT_TRUE(file->open(path).ok()); + EXPECT_TRUE(SniiSegmentReader::open(file, segment).ok()); + LogicalIndexReader idx; + EXPECT_TRUE(segment->open_index(1, "body", &idx).ok()); + return idx; +} + +LogicalIndexReader OpenMeteredIndex(io::MeteredFileReader* file, SniiSegmentReader* segment) { + EXPECT_TRUE(SniiSegmentReader::open(file, segment).ok()); + LogicalIndexReader idx; + EXPECT_TRUE(segment->open_index(1, "body", &idx).ok()); + return idx; +} + +class RecordingSink final : public query::DocIdSink { +public: + Status append_sorted(std::span docids) override { + ++chunks; + max_chunk = std::max(max_chunk, docids.size()); + out.insert(out.end(), docids.begin(), docids.end()); + return Status::OK(); + } + + Status append_range(uint32_t first, uint64_t last_exclusive) override { + ++chunks; + if (last_exclusive > first) { + max_chunk = std::max(max_chunk, static_cast(last_exclusive - first)); + for (uint64_t docid = first; docid < last_exclusive; ++docid) { + out.push_back(static_cast(docid)); + } + } + return Status::OK(); + } + + std::vector out; + size_t chunks = 0; + size_t max_chunk = 0; +}; + +class FailingSink final : public query::DocIdSink { +public: + Status append_sorted(std::span docids) override { + saw_docids = !docids.empty(); + return Status::IOError("sink append failed"); + } + + Status append_range(uint32_t first, uint64_t last_exclusive) override { + saw_docids = saw_docids || (last_exclusive > first); + return Status::IOError("sink append failed"); + } + + bool saw_docids = false; +}; + +} // namespace + +TEST(SniiBooleanOr, ReturnsSortedDeduplicatedUnion) { + const Corpus corpus = BuildCorpus(); + const std::string path = TempPath(); + WriteCorpus(corpus, path); + + io::LocalFileReader file; + SniiSegmentReader segment; + LogicalIndexReader idx = OpenIndex(&file, &segment, path); + + const std::vector terms = {"aa_hot", "aa_rare", "absent_token", "aa_hot", + "aa_sparse"}; + std::vector got; + ASSERT_TRUE(query::boolean_or(idx, terms, &got).ok()); + + EXPECT_TRUE(std::ranges::is_sorted(got)); + EXPECT_EQ(got, corpus.any_docs(terms)); + + std::vector single; + ASSERT_TRUE(query::term_query(idx, "aa_hot", &single).ok()); + std::vector single_or; + ASSERT_TRUE(query::boolean_or(idx, {"aa_hot"}, &single_or).ok()); + EXPECT_EQ(single_or, single); + + std::vector absent; + ASSERT_TRUE(query::boolean_or(idx, {"missing_a", "missing_b"}, &absent).ok()); + EXPECT_TRUE(absent.empty()); + + std::remove(path.c_str()); +} + +TEST(SniiBooleanOr, EmitsHighHitResultsInBulkChunks) { + const Corpus corpus = BuildCorpus(); + const std::string path = TempPath(); + WriteCorpus(corpus, path); + + io::LocalFileReader file; + SniiSegmentReader segment; + LogicalIndexReader idx = OpenIndex(&file, &segment, path); + + RecordingSink sink; + ASSERT_TRUE(query::boolean_or(idx, {"aa_hot"}, &sink).ok()); + + const std::vector want = corpus.any_docs({"aa_hot"}); + EXPECT_EQ(sink.out, want); + ASSERT_FALSE(want.empty()); + EXPECT_GT(sink.max_chunk, 1U); + EXPECT_LT(sink.chunks, want.size() / 100) + << "high-hit term results must be handed off in bulk, not per doc"; + + std::remove(path.c_str()); +} + +TEST(SniiBooleanOr, PropagatesSinkAppendFailure) { + const Corpus corpus = BuildCorpus(); + const std::string path = TempPath(); + WriteCorpus(corpus, path); + + io::LocalFileReader file; + SniiSegmentReader segment; + LogicalIndexReader idx = OpenIndex(&file, &segment, path); + + FailingSink sink; + const Status st = query::boolean_or(idx, {"aa_hot"}, &sink); + EXPECT_TRUE(st.is()) << st.to_string(); + EXPECT_TRUE(sink.saw_docids); + + std::remove(path.c_str()); +} + +TEST(SniiBooleanOr, BatchesWindowedPostingReadsByStage) { + const Corpus corpus = BuildCorpus(); + const std::string path = TempPath(); + WriteCorpus(corpus, path); + + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + io::MeteredFileReader metered(&local, /*block_size=*/128); + SniiSegmentReader segment; + LogicalIndexReader idx = OpenMeteredIndex(&metered, &segment); + + const std::vector terms = {"aa_hot", "aa_warm", "aa_loud"}; + for (const std::string& term : terms) { + bool found = false; + doris::snii::format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + ASSERT_TRUE(idx.lookup(term, &found, &entry, &frq_base, &prx_base).ok()); + ASSERT_TRUE(found) << term; + } + + const uint64_t serial_before = metered.metrics().serial_rounds; + std::vector got; + ASSERT_TRUE(query::boolean_or(idx, terms, &got).ok()); + const uint64_t serial_delta = metered.metrics().serial_rounds - serial_before; + + EXPECT_EQ(got, corpus.any_docs(terms)); + EXPECT_LE(serial_delta, 2U) << "windowed OR should batch posting reads by stage, not by term"; + + std::remove(path.c_str()); +} + +TEST(SniiBooleanAnd, SkipsDenseFullWindowsForRareDriver) { + const Corpus corpus = BuildDenseAndCorpus(); + const std::string path = TempPath(); + WriteCorpus(corpus, path); + + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + io::MeteredFileReader metered(&local, /*block_size=*/128); + SniiSegmentReader segment; + LogicalIndexReader idx = OpenMeteredIndex(&metered, &segment); + + std::vector hot_docs; + ASSERT_TRUE(query::term_query(idx, "aa_dense", &hot_docs).ok()); + const uint64_t hot_bytes = metered.metrics().total_request_bytes; + ASSERT_EQ(hot_docs, corpus.term_docs("aa_dense")); + ASSERT_GT(hot_bytes, 0U); + + metered.reset_metrics(); + std::vector got; + ASSERT_TRUE(query::boolean_and(idx, {"aa_dense", "aa_rare_driver"}, &got).ok()); + const uint64_t and_bytes = metered.metrics().total_request_bytes; + + std::vector want; + const std::vector rare = corpus.term_docs("aa_rare_driver"); + std::ranges::set_intersection(hot_docs, rare, std::back_inserter(want)); + EXPECT_EQ(got, want); + EXPECT_LT(and_bytes, hot_bytes / 4) + << "dense full-window AND should keep rare-driver candidates without " + "reading the dense term's dd regions"; + + std::remove(path.c_str()); +} diff --git a/be/test/storage/index/snii/query/byte_skip_test.cpp b/be/test/storage/index/snii/query/byte_skip_test.cpp new file mode 100644 index 00000000000000..e49d14b6f04482 --- /dev/null +++ b/be/test/storage/index/snii/query/byte_skip_test.cpp @@ -0,0 +1,467 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/io/metered_file_reader.h" +#include "storage/index/snii/query/bm25_scorer.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/scoring_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/reader/windowed_posting.h" +#include "storage/index/snii/stats/snii_stats_provider.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +// PHASE B differential + byte-reduction test (design spec sections 1.4 & 2). +// +// Builds an index (~4000 docs, positions + scoring) with a HIGH-DF term spanning +// MANY .frq windows plus low/mid-df terms and a planted 5-term phrase led by the +// high-df term. Over a MeteredFileReader it asserts: +// (a) term_query and phrase_query docids equal a brute-force ORACLE (unchanged +// by the freq-skip optimization); +// (b) a docid-only term_query on the high-df term requests STRICTLY FEWER .frq +// bytes than the pre-PhaseB full-window path (sum of per-window frq_len), +// because the freq region is skipped on the wire; +// (c) scoring_query STILL reads the FULL windows (freq region present -> its +// .frq request bytes match the full-window total, strictly above the +// docid-only path) and returns the correct top-K. +using namespace doris::snii; +using namespace doris::snii::format; +using namespace doris::snii::reader; +using namespace doris::snii::writer; +using doris::snii::query::Bm25Params; +using doris::snii::query::ScoredDoc; +using doris::snii::stats::SniiStatsProvider; + +namespace { + +std::string TempPath() { + static int counter = 0; + return "/tmp/snii_byte_skip_" + std::to_string(getpid()) + "_" + std::to_string(counter++) + + ".idx"; +} + +constexpr uint32_t kDocCount = 4000; +constexpr uint32_t kHiGap = 1; // aa_hi in (almost) every doc -> many windows +constexpr uint32_t kHiCount = 3600; // df of aa_hi (>>256 -> many windows) +// The 5-term phrase is planted only in the FIRST kPhraseSpan occurrences of +// aa_hi, so a low-df lead term concentrates candidates into the first windows. +constexpr uint32_t kPhraseSpan = 120; + +// aa_hi appears with VARIED, large frequency so its per-window freq region is +// big in absolute terms -- large enough to span whole cache blocks the docs-only +// path never touches, making the freq-skip visible in remote_bytes (not just in +// the raw wire-byte total). Frequencies cycle through a wide range. +uint32_t HiFreq(uint32_t occ) { + return 40U + (occ * 13U + 7U) % 200U; +} + +struct Corpus { + uint32_t doc_count = 0; + std::vector> docs; // docs[d] = ordered tokens + std::vector doc_len; // per-doc token count (for norms) + + bool phrase_in_doc(uint32_t d, const std::vector& phrase) const { + if (phrase.empty()) { + return false; + } + const auto& toks = docs[d]; + if (toks.size() < phrase.size()) { + return false; + } + for (size_t i = 0; i + phrase.size() <= toks.size(); ++i) { + bool match = true; + for (size_t k = 0; k < phrase.size(); ++k) { + if (toks[i + k] != phrase[k]) { + match = false; + break; + } + } + if (match) { + return true; + } + } + return false; + } + + std::vector term_oracle(const std::string& term) const { + std::vector out; + for (uint32_t d = 0; d < doc_count; ++d) { + const auto& toks = docs[d]; + if (std::ranges::find(toks, term) != toks.end()) { + out.push_back(d); + } + } + return out; + } + + std::vector phrase_oracle(const std::vector& phrase) const { + std::vector out; + for (uint32_t d = 0; d < doc_count; ++d) { + if (phrase_in_doc(d, phrase)) { + out.push_back(d); + } + } + return out; + } +}; + +Corpus BuildCorpus() { + Corpus c; + c.doc_count = kDocCount; + c.docs.resize(c.doc_count); + c.doc_len.assign(c.doc_count, 0); + const std::vector vocab = {"alpha", "bravo", "charlie", "delta"}; + + for (uint32_t d = 0; d < c.doc_count; ++d) { + std::vector& toks = c.docs[d]; + const bool is_hi = (d % kHiGap == 0) && (d / kHiGap < kHiCount); + if (is_hi) { + toks.emplace_back("aa_hi"); // high-df, position 0 + } + const uint32_t occ = d / kHiGap; // aa_hi occurrence ordinal + + if (is_hi && occ < kPhraseSpan) { + // 5-term phrase led by the high-df term, concentrated in early windows. + toks.emplace_back("aa_quick"); + toks.emplace_back("aa_brown"); + toks.emplace_back("aa_fox"); + toks.emplace_back("aa_jumps"); + } else if (d % 11 == 0) { + toks.emplace_back("aa_lazy"); + toks.emplace_back("aa_dog"); + } + // Mid-df term: every 5th doc (varied freq via repetition for scoring). + if (d % 5 == 0) { + toks.emplace_back("aa_mid"); + if (d % 15 == 0) { + toks.emplace_back("aa_mid"); + } + } + for (uint32_t k = 0; k < 2; ++k) { + toks.push_back(vocab[(d + k) % vocab.size()]); + } + if (d % 97 == 0) { + toks.emplace_back("aa_rare"); + } + // Filler vocabulary to occupy the lexicographic tail blocks. + char nm[16]; + std::snprintf(nm, sizeof(nm), "zz_%03u", d % 400); + toks.emplace_back(nm); + // Extra aa_hi repetitions AFTER position 0's phrase, giving aa_hi a varied, + // larger freq so its per-window freq region carries real bytes. These trail + // the phrase tokens so the consecutive phrase at position 0 is preserved. + if (is_hi) { + const uint32_t extra = HiFreq(occ) - 1U; // total freq = HiFreq(occ) + for (uint32_t k = 0; k < extra; ++k) { + toks.emplace_back("aa_hi"); + } + } + c.doc_len[d] = toks.size(); + } + return c; +} + +void WriteCorpus(const Corpus& c, const std::string& path) { + SpimiTermBuffer buf(/*has_positions=*/true); + for (uint32_t d = 0; d < c.doc_count; ++d) { + const auto& toks = c.docs[d]; + for (uint32_t pos = 0; pos < toks.size(); ++pos) { + buf.add_token(toks[pos], d, pos); + } + } + std::vector terms = buf.finalize_sorted(); + + SniiIndexInput in; + in.index_id = 1; + in.index_suffix = "body"; + in.config = IndexConfig::kDocsPositionsScoring; // positions (phrase) + norms (scoring) + in.doc_count = c.doc_count; + in.terms = std::move(terms); + in.target_dict_block_bytes = 256; + + in.encoded_norms.resize(c.doc_count); + for (uint32_t d = 0; d < c.doc_count; ++d) { + in.encoded_norms[d] = doris::snii::query::encode_norm(c.doc_len[d]); + } + + io::LocalFileWriter w; + ASSERT_TRUE(w.open(path).ok()); + SniiCompoundWriter cw(&w); + ASSERT_TRUE(cw.add_logical_index(in).ok()); + ASSERT_TRUE(cw.finish().ok()); +} + +// Grouped-block byte totals of a windowed term (design 1.6): docs = dd-block +// length (sum of per-window dd_disk_len); full = dd-block + freq-block lengths +// (what scoring reads). Also returns prelude_len and window count. +struct FrqByteTotals { + uint64_t full = 0; // dd-block + freq-block on-disk bytes + uint64_t docs = 0; // dd-block on-disk bytes (the contiguous docs-only run) + uint64_t prelude_len = 0; + uint32_t windows = 0; +}; + +FrqByteTotals MeasureFrqTotals(const LogicalIndexReader& idx, const std::string& term) { + FrqByteTotals t; + DictEntry entry; + uint64_t frq_base = 0, prx_base = 0; + bool found = false; + EXPECT_TRUE(idx.lookup(term, &found, &entry, &frq_base, &prx_base).ok()); + EXPECT_TRUE(found); + EXPECT_EQ(entry.enc, DictEntryEnc::kWindowed); + t.prelude_len = entry.prelude_len; + FrqPreludeReader prelude; + EXPECT_TRUE(fetch_windowed_prelude(idx, entry, frq_base, &prelude).ok()); + t.windows = prelude.n_windows(); + t.docs = prelude.dd_block_len(); + t.full = prelude.dd_block_len() + prelude.freq_block_len(); + return t; +} + +// Reference BM25 ranking computed directly from the corpus (df/idf/tf + norms). +std::vector ReferenceRanking(const Corpus& c, const std::vector& terms, + uint32_t k, const Bm25Params& params) { + uint64_t sum_ttf = 0; + for (uint64_t dl : c.doc_len) { + sum_ttf += dl; + } + const double avgdl = static_cast(sum_ttf) / std::max(1, c.doc_count); + + std::unordered_map scores; + for (const auto& term : terms) { + std::map plist; // docid -> freq + for (uint32_t d = 0; d < c.doc_count; ++d) { + uint32_t f = 0; + for (const auto& t : c.docs[d]) { + if (t == term) { + ++f; + } + } + if (f > 0) { + plist[d] = f; + } + } + if (plist.empty()) { + continue; + } + const uint64_t df = plist.size(); + const double idf = + std::log(1.0 + (static_cast(c.doc_count) - df + 0.5) / (df + 0.5)); + for (const auto& [docid, freq] : plist) { + // Match the index's quantization (encode -> decode) so scores compare exact. + const double dl = doris::snii::query::decode_norm( + doris::snii::query::encode_norm(c.doc_len[docid])); + const double denom = freq + params.k1 * (1.0 - params.b + params.b * dl / avgdl); + scores[docid] += idf * (freq * (params.k1 + 1.0)) / denom; + } + } + + std::vector all; + all.reserve(scores.size()); + for (const auto& [docid, s] : scores) { + all.push_back({docid, s}); + } + std::ranges::sort(all, [](const ScoredDoc& a, const ScoredDoc& b) { + if (a.score != b.score) { + return a.score > b.score; + } + return a.docid < b.docid; + }); + if (all.size() > k) { + all.resize(k); + } + return all; +} + +} // namespace + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiByteSkip, DocidAndPhraseSkipFreqWhileScoringKeepsIt) { + Corpus c = BuildCorpus(); + const std::string path = TempPath(); + WriteCorpus(c, path); + + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + // Fine-grained cache block (< a window's freq region) so the per-window + // freq-skip shows up in remote_bytes too: each window's freq region spans whole + // cache blocks that the docs-only path never touches. + io::MeteredFileReader metered(&local, /*block_size=*/64); + + SniiSegmentReader seg; + ASSERT_TRUE(SniiSegmentReader::open(&metered, &seg).ok()); + LogicalIndexReader idx; + ASSERT_TRUE(seg.open_index(1, "body", &idx).ok()); + ASSERT_EQ(idx.stats().doc_count, c.doc_count); + + // The high-df term must span many windows for the freq-skip to be meaningful. + const FrqByteTotals hi = MeasureFrqTotals(idx, "aa_hi"); + ASSERT_GE(hi.windows, 4U) << "high-df term should span multiple windows"; + // PhaseA format invariant: the docs-only prefix is strictly smaller than the + // full window total (the freq region occupies real bytes). + ASSERT_LT(hi.docs, hi.full) << "docs-only prefix not smaller: docs=" << hi.docs + << " full=" << hi.full; + + // ---- (a) term_query / phrase_query docids == ORACLE (unchanged) ----------- + const std::vector terms_to_check = {"aa_hi", "aa_mid", "aa_quick", "aa_lazy", + "aa_rare"}; + for (const auto& term : terms_to_check) { + std::vector got; + ASSERT_TRUE(query::term_query(idx, term, &got).ok()) << term; + EXPECT_TRUE(std::ranges::is_sorted(got)) << term; + EXPECT_EQ(got, c.term_oracle(term)) << "term_query oracle mismatch: " << term; + } + + const std::vector> phrases = { + {"aa_hi", "aa_quick", "aa_brown", "aa_fox", "aa_jumps"}, // 5-term, hi lead + {"aa_hi", "aa_quick"}, // hi + mid + {"aa_quick", "aa_brown", "aa_fox", "aa_jumps"}, // no high-df term + {"aa_lazy", "aa_dog"}, // scattered phrase + {"aa_brown", "aa_fox"}, // sub-phrase + {"aa_fox", "aa_brown"}, // reversed -> absent + {"aa_hi", "aa_lazy"}, // present terms, absent phrase + }; + for (const auto& p : phrases) { + std::string label; + for (const auto& w : p) { + label += w + " "; + } + std::vector got; + ASSERT_TRUE(query::phrase_query(idx, p, &got).ok()) << label; + EXPECT_TRUE(std::ranges::is_sorted(got)) << label; + EXPECT_EQ(got, c.phrase_oracle(p)) << "phrase oracle mismatch: [" << label << "]"; + } + + // ---- (b) docid-only term_query on aa_hi skips the freq region ------------- + // Format-level proof: the docs-only prefixes total far fewer bytes than the + // full windows (the freq region is real and is what the wire-level skip drops). + const uint64_t full_window_frq = hi.full; // pre-PhaseB .frq byte total + EXPECT_LT(hi.docs * 100, full_window_frq * 85) + << "freq-skip format reduction too small: docs=" << hi.docs + << " full=" << full_window_frq; + // Both A and B below share the SAME lookup + prelude overhead; the ONLY + // difference is whether each window fetches its docs-only prefix or the full + // window. So (B - A) isolates exactly the .frq freq-region bytes the docid-only + // path skips. A and B are measured live over the metered reader. + // + // A = PhaseB docid-only term_query (want_freq=false everywhere). + metered.reset_metrics(); + std::vector hi_docs; + ASSERT_TRUE(query::term_query(idx, "aa_hi", &hi_docs).ok()); + const io::IoMetrics a = metered.metrics(); + ASSERT_EQ(hi_docs, c.term_oracle("aa_hi")); + + // B = the pre-PhaseB full-window baseline: same lookup + prelude, then read + // every window IN FULL (want_freq=true). This is exactly what term_query used + // to fetch before the freq-skip landed. + metered.reset_metrics(); + { + DictEntry entry; + uint64_t fb = 0, pb = 0; + bool found = false; + ASSERT_TRUE(idx.lookup("aa_hi", &found, &entry, &fb, &pb).ok()); + ASSERT_TRUE(found); + DecodedPosting full_posting; + ASSERT_TRUE(read_windowed_posting(idx, entry, fb, pb, /*want_positions=*/false, + /*want_freq=*/true, &full_posting) + .ok()); + EXPECT_EQ(full_posting.docids, c.term_oracle("aa_hi")); + EXPECT_EQ(full_posting.freqs.size(), full_posting.docids.size()) + << "want_freq=true must decode per-doc freqs"; + } + const io::IoMetrics b = metered.metrics(); + + // The docid-only path requests STRICTLY FEWER bytes on the wire (freq skipped). + EXPECT_LT(a.total_request_bytes, b.total_request_bytes) + << "docid-only did not skip freq bytes: a=" << a.total_request_bytes + << " b=" << b.total_request_bytes; + // The skipped amount equals the full-vs-docs .frq delta (sum(frq_len-frq_docs_len)). + const uint64_t saved = b.total_request_bytes - a.total_request_bytes; + EXPECT_EQ(saved, hi.full - hi.docs) << "skipped bytes != freq-region size: saved=" << saved + << " freq_region=" << (hi.full - hi.docs); + // remote_bytes (cache-block granular) is also strictly below the full-window + // baseline: the freq region occupies whole cache blocks the docs path never hits. + EXPECT_LT(a.remote_bytes, b.remote_bytes) + << "docid-only remote_bytes not below full-window: a=" << a.remote_bytes + << " b=" << b.remote_bytes; + + // ---- (c) scoring_query STILL reads full windows (freq present) ------------ + SniiStatsProvider stats; + ASSERT_TRUE(SniiStatsProvider::open(&idx, &stats).ok()); + const Bm25Params params; // defaults + const std::vector score_terms = {"aa_hi", "aa_mid", "aa_rare"}; + const uint32_t kTopK = 10; + + std::vector exhaustive, wand; + ASSERT_TRUE(query::scoring_query_exhaustive(idx, stats, score_terms, kTopK, params, &exhaustive) + .ok()); + ASSERT_TRUE(query::scoring_query_wand(idx, stats, score_terms, kTopK, params, &wand).ok()); + + const std::vector ref = ReferenceRanking(c, score_terms, kTopK, params); + ASSERT_EQ(exhaustive.size(), ref.size()); + for (size_t i = 0; i < ref.size(); ++i) { + EXPECT_EQ(exhaustive[i].docid, ref[i].docid) << "scoring docid mismatch at " << i; + EXPECT_NEAR(exhaustive[i].score, ref[i].score, 1e-9) << "scoring score mismatch at " << i; + } + // WAND must equal the exhaustive ranking exactly (docid + score). + ASSERT_EQ(wand.size(), exhaustive.size()); + for (size_t i = 0; i < wand.size(); ++i) { + EXPECT_EQ(wand[i].docid, exhaustive[i].docid) << "wand vs exhaustive at " << i; + EXPECT_NEAR(wand[i].score, exhaustive[i].score, 1e-9) << "wand vs exhaustive at " << i; + } + + // Scoring reads the FULL .frq windows for the high-df term (freq region present), + // so a single-term scoring query requests STRICTLY MORE bytes than the + // docid-only term_query A (which skipped the freq region). Same lookup + .frq + // section; scoring additionally pulls the freq region (and norms/prelude), so + // its request total exceeds the docid-only path by at least the freq region. + metered.reset_metrics(); + std::vector hi_only; + ASSERT_TRUE( + query::scoring_query_exhaustive(idx, stats, {"aa_hi"}, kTopK, params, &hi_only).ok()); + const io::IoMetrics score_io = metered.metrics(); + const uint64_t score_frq_request = score_io.total_request_bytes; + const uint64_t term_frq_request = a.total_request_bytes; + EXPECT_GE(score_frq_request, term_frq_request + (hi.full - hi.docs)) + << "scoring did not read the full freq region: scoring=" << score_frq_request + << " docid=" << term_frq_request << " freq_region=" << (hi.full - hi.docs); + EXPECT_GT(score_frq_request, term_frq_request) + << "scoring should read MORE .frq bytes than docid-only: scoring=" << score_frq_request + << " docid=" << term_frq_request; + + std::remove(path.c_str()); +} diff --git a/be/test/storage/index/snii/query/count_query_test.cpp b/be/test/storage/index/snii/query/count_query_test.cpp new file mode 100644 index 00000000000000..83d1f57bb36eba --- /dev/null +++ b/be/test/storage/index/snii/query/count_query_test.cpp @@ -0,0 +1,437 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/count_query.h" + +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "roaring/roaring.hh" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/null_bitmap.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" +#include "storage/index/snii_query_test_util.h" + +// G02 count-only fast path: the count answered from dict-entry df alone must +// EQUAL the full posting decode on the shared 9000-doc fixture, and every +// unsafe shape must fall through (*handled = false) so the generic paths keep +// owning the semantics. The count_fastpath_hits seam (BE_TEST) pins that hits +// are counted exactly when a dict-only answer was produced and never on a +// fall-through or on the ordinary decode paths. +using namespace doris::snii; +using namespace doris::snii::snii_test; +using doris::snii::query::count_only_term_df; +using doris::snii::query::count_only_two_term_phrase_bigram_df; +using doris::snii::query::fabricate_null_disjoint_count_bitmap; +using doris::snii::query::phrase_query; +using doris::snii::query::term_query; +namespace qinternal = doris::snii::query::internal; + +namespace { + +constexpr uint32_t kPruneThreshold = 64; + +struct Fixture { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; +}; + +void build_fixture(Fixture* f, bool include_bigrams, uint32_t bigram_prune_min_df, + uint64_t bigram_prune_max_df = 0) { + assert_ok(build_reader(&f->file, &f->segment_reader, &f->index_reader, include_bigrams, + bigram_prune_min_df, bigram_prune_max_df)); +} + +uint64_t full_decode_term_count(const reader::LogicalIndexReader& idx, const std::string& term) { + std::vector docids; + assert_ok(term_query(idx, term, &docids)); + return docids.size(); +} + +uint64_t full_decode_phrase_count(const reader::LogicalIndexReader& idx, + const std::vector& terms) { + std::vector docids; + assert_ok(phrase_query(idx, terms, &docids)); + return docids.size(); +} + +void reset_query_counters() { + qinternal::query_test_counters() = qinternal::QueryTestCounters {}; +} + +uint64_t count_hits() { + return qinternal::query_test_counters().count_fastpath_hits; +} + +} // namespace + +// Single-term df from the dict equals the full posting decode for every +// posting layout in the fixture (dense/windowed, sparse, tiny, tail) and for +// an absent term (0). The seam counts one hit per dict-only answer and none +// for the decodes. +TEST(SniiCountQuery, TermDfEqualsFullDecode) { + Fixture plain; + build_fixture(&plain, /*include_bigrams=*/false, /*threshold=*/0); + + const std::vector terms {"failed", "driver", "almost", "sparse_left", + "sparse_right", "needle", "123", "trace", + "repeat", "nosuchterm"}; + + reset_query_counters(); + uint64_t expected_hits = 0; + for (const std::string& term : terms) { + const uint64_t decode_count = full_decode_term_count(plain.index_reader, term); + EXPECT_EQ(count_hits(), expected_hits) << "decode path must not touch the seam"; + + uint64_t df_count = 0; + assert_ok(count_only_term_df(plain.index_reader, term, &df_count)); + EXPECT_EQ(df_count, decode_count) << "term: " << term; + ++expected_hits; + EXPECT_EQ(count_hits(), expected_hits) << "term: " << term; + } + // Spot-pin the fixture dfs so a fixture drift cannot silently weaken the test. + uint64_t df = 0; + assert_ok(count_only_term_df(plain.index_reader, "failed", &df)); + EXPECT_EQ(df, 9000U); + assert_ok(count_only_term_df(plain.index_reader, "nosuchterm", &df)); + EXPECT_EQ(df, 0U); +} + +// 2-term phrase with a SURVIVING bigram: on the G01-pruned segment the hot +// pair (df 9000 >= threshold) is answered from the bigram entry's df, equal to +// the full phrase execution on the pruned segment AND on the unpruned control. +TEST(SniiCountQuery, PhraseHotBigramDfEqualsFullDecode) { + Fixture control, pruned; + build_fixture(&control, /*include_bigrams=*/true, /*threshold=*/0); + build_fixture(&pruned, /*include_bigrams=*/true, kPruneThreshold); + + const uint64_t truth = full_decode_phrase_count(pruned.index_reader, {"repeat", "repeat"}); + EXPECT_EQ(truth, 9000U); + EXPECT_EQ(full_decode_phrase_count(control.index_reader, {"repeat", "repeat"}), truth); + + reset_query_counters(); + bool handled = false; + uint64_t count = 0; + assert_ok(count_only_two_term_phrase_bigram_df(pruned.index_reader, "repeat", "repeat", + &handled, &count)); + EXPECT_TRUE(handled); + EXPECT_EQ(count, truth); + EXPECT_EQ(count_hits(), 1U); + + // A low-df pair materialized on the LEGACY control segment is also a dict + // HIT there and must equal its positional truth. + handled = false; + count = 0; + assert_ok(count_only_two_term_phrase_bigram_df(control.index_reader, "failed", "order", + &handled, &count)); + EXPECT_TRUE(handled); + EXPECT_EQ(count, full_decode_phrase_count(control.index_reader, {"failed", "order"})); + EXPECT_EQ(count, 3U); + EXPECT_EQ(count_hits(), 2U); +} + +// Guard: a PRUNED bigram (dict miss on a segment whose meta declares a +// threshold) must fall through -- the miss is ambiguous and only the generic +// phrase path may answer it. The seam stays untouched. +TEST(SniiCountQuery, PrunedBigramFallsThrough) { + Fixture pruned; + build_fixture(&pruned, /*include_bigrams=*/true, kPruneThreshold); + ASSERT_EQ(pruned.index_reader.bigram_prune_min_df(), kPruneThreshold); + + reset_query_counters(); + bool handled = true; + uint64_t count = 42; + assert_ok(count_only_two_term_phrase_bigram_df(pruned.index_reader, "failed", "order", &handled, + &count)); + EXPECT_FALSE(handled); + EXPECT_EQ(count, 0U); + EXPECT_EQ(count_hits(), 0U); + + // The generic path still owns the correct answer. + EXPECT_EQ(full_decode_phrase_count(pruned.index_reader, {"failed", "order"}), 3U); +} + +// Guard (G15): a MAX-pruned bigram is a dict miss on a segment whose meta +// declares ONLY the upper threshold -- the count fast path must NOT claim the +// pruned pair's (absent) df: it falls through untouched (seam 0) and the full +// phrase execution keeps owning the 9000-doc answer. A pair still INSIDE the +// band fastpaths as before on the same segment. +TEST(SniiCountQuery, MaxPrunedBigramFallsThroughSurvivorStillFastpaths) { + constexpr uint64_t kMaxDf = 1800; + Fixture maxpruned; + build_fixture(&maxpruned, /*include_bigrams=*/true, /*bigram_prune_min_df=*/0, kMaxDf); + ASSERT_EQ(maxpruned.index_reader.bigram_prune_min_df(), 0U); + ASSERT_EQ(maxpruned.index_reader.bigram_prune_max_df(), kMaxDf); + + reset_query_counters(); + bool handled = true; + uint64_t count = 42; + // (repeat,repeat) df 9000 > 1800 was pruned: the dict miss is ambiguous, so + // no dict-only answer may be produced -- the outputs stay at the contract's + // zeroed defaults (never the pruned pair's 9000) and the seam stays cold. + assert_ok(count_only_two_term_phrase_bigram_df(maxpruned.index_reader, "repeat", "repeat", + &handled, &count)); + EXPECT_FALSE(handled); + EXPECT_EQ(count, 0U); + EXPECT_EQ(count_hits(), 0U); + + // The generic phrase path still owns the correct full-cardinality answer. + EXPECT_EQ(full_decode_phrase_count(maxpruned.index_reader, {"repeat", "repeat"}), 9000U); + + // (failed,order) df 3 <= 1800 survived the max-only prune: dict-df answer + // fires and equals the positional truth. + handled = false; + count = 0; + assert_ok(count_only_two_term_phrase_bigram_df(maxpruned.index_reader, "failed", "order", + &handled, &count)); + EXPECT_TRUE(handled); + EXPECT_EQ(count, 3U); + EXPECT_EQ(count_hits(), 1U); +} + +// Guards: legacy dict miss (pair never materialized), a bigram-less segment, +// and non-bigram-indexable terms all fall through conservatively. +TEST(SniiCountQuery, LegacyMissAndNonIndexableFallThrough) { + Fixture control, nobigram; + build_fixture(&control, /*include_bigrams=*/true, /*threshold=*/0); + build_fixture(&nobigram, /*include_bigrams=*/false, /*threshold=*/0); + + reset_query_counters(); + bool handled = true; + uint64_t count = 42; + // Legacy segment, pair not materialized by the fixture: even though legacy + // semantics say "miss == empty", the count path leaves that to phrase_query. + assert_ok(count_only_two_term_phrase_bigram_df(control.index_reader, "driver", "failed", + &handled, &count)); + EXPECT_FALSE(handled); + + // Segment built without bigrams at all. + handled = true; + assert_ok(count_only_two_term_phrase_bigram_df(nobigram.index_reader, "repeat", "repeat", + &handled, &count)); + EXPECT_FALSE(handled); + + // "123" is not bigram-indexable (non-alpha): no synthetic term to look up. + handled = true; + assert_ok(count_only_two_term_phrase_bigram_df(control.index_reader, "123", "order", &handled, + &count)); + EXPECT_FALSE(handled); + + EXPECT_EQ(count_hits(), 0U); +} + +// Guard: a positionless (kDocsOnly) index must fall through on the phrase +// shape -- the normal phrase path errors there and the count path must not +// mask it. The single-term df path stays valid on the same index. +TEST(SniiCountQuery, DocsOnlyIndexPhraseFallsThroughTermStillCounts) { + MemoryFile file; + writer::SniiIndexInput input; + input.index_id = 3; + input.index_suffix = "DocsOnly"; + input.config = format::IndexConfig::kDocsOnly; + input.doc_count = 100; + input.terms = {make_term("alpha", docs_with_one_position(0, 60, 0)), + make_term("bravo", docs_with_one_position(10, 25, 1))}; + std::ranges::sort(input.terms, + [](const writer::TermPostings& lhs, const writer::TermPostings& rhs) { + return lhs.term < rhs.term; + }); + writer::SniiCompoundWriter writer(&file); + assert_ok(writer.add_logical_index(input)); + assert_ok(writer.finish()); + + reader::SniiSegmentReader segment_reader; + assert_ok(reader::SniiSegmentReader::open(&file, &segment_reader)); + reader::LogicalIndexReader idx; + assert_ok(segment_reader.open_index(input.index_id, input.index_suffix, &idx)); + ASSERT_FALSE(idx.has_positions()); + + reset_query_counters(); + bool handled = true; + uint64_t count = 42; + assert_ok(count_only_two_term_phrase_bigram_df(idx, "alpha", "bravo", &handled, &count)); + EXPECT_FALSE(handled); + EXPECT_EQ(count_hits(), 0U); + + uint64_t df = 0; + assert_ok(count_only_term_df(idx, "alpha", &df)); + EXPECT_EQ(df, full_decode_term_count(idx, "alpha")); + EXPECT_EQ(df, 60U); + assert_ok(count_only_term_df(idx, "bravo", &df)); + EXPECT_EQ(df, 15U); + EXPECT_EQ(count_hits(), 2U); +} + +// --- fabricate_null_disjoint_count_bitmap truth table ----------------------- +// The fabricated bitmap must (a) hold exactly `count` ids, (b) be DISJOINT +// from the null bitmap so the caller-side FunctionMatchBase -> +// InvertedIndexResultBitmap::mask_out_null subtraction is a provable no-op, +// and (c) stay inside [0, count + |nulls|), which never exceeds the segment's +// [0, num_rows) row space because df counts only non-null docs. + +TEST(SniiCountQuery, FabricateNoNullsIsDenseRange) { + roaring::Roaring nulls; + roaring::Roaring out; + assert_ok(fabricate_null_disjoint_count_bitmap(100, nulls, &out)); + roaring::Roaring expected; + expected.addRange(0, 100); + EXPECT_EQ(out, expected); +} + +TEST(SniiCountQuery, FabricateNullPrefixShiftsRange) { + roaring::Roaring nulls; + nulls.addRange(0, 100); + roaring::Roaring out; + assert_ok(fabricate_null_disjoint_count_bitmap(50, nulls, &out)); + roaring::Roaring expected; + expected.addRange(100, 150); + EXPECT_EQ(out, expected); + EXPECT_TRUE((out & nulls).isEmpty()); +} + +TEST(SniiCountQuery, FabricateInterleavedNullsStaysDisjointAndExact) { + roaring::Roaring nulls; + for (uint32_t id = 0; id < 200; id += 2) { + nulls.add(id); // 100 even nulls + } + roaring::Roaring out; + assert_ok(fabricate_null_disjoint_count_bitmap(70, nulls, &out)); + EXPECT_EQ(out.cardinality(), 70U); + EXPECT_TRUE((out & nulls).isEmpty()); + // Exactly the first 70 odd (non-null) ids. + roaring::Roaring expected; + for (uint32_t id = 1; id < 140; id += 2) { + expected.add(id); + } + EXPECT_EQ(out, expected); + // The safety property itself: the null-bitmap subtraction the MATCH + // machinery applies to every index result must not change the count. + roaring::Roaring masked = out; + masked -= nulls; + EXPECT_EQ(masked.cardinality(), 70U); +} + +TEST(SniiCountQuery, FabricateNullsBeyondWindowIrrelevant) { + roaring::Roaring nulls; + nulls.add(1000000); + roaring::Roaring out; + assert_ok(fabricate_null_disjoint_count_bitmap(10, nulls, &out)); + roaring::Roaring expected; + expected.addRange(0, 10); + EXPECT_EQ(out, expected); +} + +TEST(SniiCountQuery, FabricateConsumesEntireNonNullWindow) { + // nulls {0..4}, count 5: every non-null id of the window [0, 10) is used. + roaring::Roaring nulls; + nulls.addRange(0, 5); + roaring::Roaring out; + assert_ok(fabricate_null_disjoint_count_bitmap(5, nulls, &out)); + roaring::Roaring expected; + expected.addRange(5, 10); + EXPECT_EQ(out, expected); +} + +TEST(SniiCountQuery, FabricateZeroCountIsEmpty) { + roaring::Roaring nulls; + nulls.addRange(0, 100); + roaring::Roaring out; + out.add(7); // stale content must be overwritten + assert_ok(fabricate_null_disjoint_count_bitmap(0, nulls, &out)); + EXPECT_TRUE(out.isEmpty()); +} + +TEST(SniiCountQuery, FabricateRejectsDocidDomainOverflow) { + // df + null count beyond the uint32 docid domain can only come from a + // corrupt index: it must surface as an error (the reader glue falls + // through to the row-accurate decode), never as a silently wrong bitmap. + roaring::Roaring nulls; + nulls.add(0); + roaring::Roaring out; + const uint64_t count = uint64_t(std::numeric_limits::max()) + 1; // 2^32 + doris::Status st = fabricate_null_disjoint_count_bitmap(count, nulls, &out); + EXPECT_FALSE(st.ok()); +} + +// A segment WITH a null bitmap end to end through the real writer/reader: +// df from the dict is unchanged by nulls (the writer adds NO postings for a +// null doc, so postings -- and df -- can never include null rows), and the +// fabricated bitmap built against the segment's REAL null bitmap keeps +// cardinality df through the caller-side null subtraction. This pins the +// contract that replaced the old "veto whenever a null section exists" guard. +TEST(SniiCountQuery, NullBitmapSegmentDfCountsAndFabricationSurvivesMasking) { + MemoryFile file; + writer::SniiIndexInput input; + input.index_id = 4; + input.index_suffix = "Nullable"; + input.config = format::IndexConfig::kDocsPositions; + input.doc_count = 100; + // Writer invariant mirrored by the fixture: null docids carry no postings + // (scalar add_nulls adds no tokens; a NULL array row is an empty range). + input.null_docids = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 50, 99}; + input.terms = {make_term("alpha", docs_with_one_position(10, 50, 0))}; + writer::SniiCompoundWriter writer(&file); + assert_ok(writer.add_logical_index(input)); + assert_ok(writer.finish()); + + reader::SniiSegmentReader segment_reader; + assert_ok(reader::SniiSegmentReader::open(&file, &segment_reader)); + reader::LogicalIndexReader idx; + assert_ok(segment_reader.open_index(input.index_id, input.index_suffix, &idx)); + ASSERT_GT(idx.section_refs().null_bitmap.length, 0U); + + // df is the exact match count despite the null rows. + uint64_t df = 0; + assert_ok(count_only_term_df(idx, "alpha", &df)); + EXPECT_EQ(df, full_decode_term_count(idx, "alpha")); + EXPECT_EQ(df, 40U); + + // Decode the segment's REAL null bitmap the same way the reader glue does. + const auto& ref = idx.section_refs().null_bitmap; + std::vector bytes; + assert_ok(idx.reader()->read_at(ref.offset, ref.length, &bytes)); + format::NullBitmapReader null_reader; + assert_ok(format::NullBitmapReader::open(Slice(bytes), &null_reader)); + roaring::Roaring nulls; + null_reader.copy_to(&nulls); + ASSERT_EQ(nulls.cardinality(), input.null_docids.size()); + + roaring::Roaring fabricated; + assert_ok(fabricate_null_disjoint_count_bitmap(df, nulls, &fabricated)); + EXPECT_EQ(fabricated.cardinality(), df); + EXPECT_TRUE((fabricated & nulls).isEmpty()); + roaring::Roaring masked = fabricated; + masked -= nulls; // FunctionMatchBase::mask_out_null equivalent + EXPECT_EQ(masked.cardinality(), df); + // With the 10 leading nulls the first 40 non-null ids are exactly [10, 50): + // pinned to catch select / removeRange off-by-ones against a real layout. + roaring::Roaring expected; + expected.addRange(10, 50); + EXPECT_EQ(fabricated, expected); +} diff --git a/be/test/storage/index/snii/query/docid_set_ops_test.cpp b/be/test/storage/index/snii/query/docid_set_ops_test.cpp new file mode 100644 index 00000000000000..eda9bf3dc7da65 --- /dev/null +++ b/be/test/storage/index/snii/query/docid_set_ops_test.cpp @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/internal/docid_set_ops.h" + +#include + +#include +#include +#include + +#include "common/status.h" + +namespace { + +std::vector Range(uint32_t begin, uint32_t end, uint32_t step = 1) { + std::vector out; + for (uint32_t v = begin; v < end; v += step) { + out.push_back(v); + } + return out; +} + +} // namespace + +TEST(SniiDocIdSetOps, UnionSortedManyDeduplicatesHighOverlapLists) { + std::vector> lists; + lists.reserve(257); + lists.push_back(Range(0, 10000)); + for (uint32_t i = 0; i < 256; ++i) { + lists.push_back(Range(i % 17, 10000, 17)); + } + + const std::vector got = doris::snii::query::internal::union_sorted_many(lists); + + EXPECT_EQ(got, Range(0, 10000)); + EXPECT_TRUE(std::ranges::is_sorted(got)); + EXPECT_EQ(std::ranges::adjacent_find(got), got.end()); +} + +TEST(SniiDocIdSetOps, UnionSortedManyMergesDisjointLists) { + const std::vector> lists = {{0, 3, 6}, {1, 4, 7}, {}, {2, 5, 8}}; + + const std::vector got = doris::snii::query::internal::union_sorted_many(lists); + + EXPECT_EQ(got, Range(0, 9)); +} diff --git a/be/test/storage/index/snii/query/pattern_query_test.cpp b/be/test/storage/index/snii/query/pattern_query_test.cpp new file mode 100644 index 00000000000000..ccd04047486470 --- /dev/null +++ b/be/test/storage/index/snii/query/pattern_query_test.cpp @@ -0,0 +1,240 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/io/metered_file_reader.h" +#include "storage/index/snii/query/regexp_query.h" +#include "storage/index/snii/query/wildcard_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +using namespace doris::snii; +using namespace doris::snii::reader; +using namespace doris::snii::writer; + +namespace { + +std::string TempPath() { + static int counter = 0; + return "/tmp/snii_pattern_query_" + std::to_string(getpid()) + "_" + std::to_string(counter++) + + ".idx"; +} + +bool WildcardMatch(std::string_view pattern, std::string_view text) { + std::vector prev(text.size() + 1, 0); + std::vector curr(text.size() + 1, 0); + prev[0] = 1; + for (char p : pattern) { + std::ranges::fill(curr, 0); + if (p == '*') { + curr[0] = prev[0]; + for (size_t i = 1; i <= text.size(); ++i) { + curr[i] = prev[i] || curr[i - 1]; + } + } else { + for (size_t i = 1; i <= text.size(); ++i) { + curr[i] = prev[i - 1] && (p == '?' || p == text[i - 1]); + } + } + prev.swap(curr); + } + return prev[text.size()] != 0; +} + +struct Corpus { + uint32_t doc_count = 0; + std::vector> docs; + + std::vector matching_docs( + const std::function& matches) const { + std::set ids; + for (uint32_t d = 0; d < docs.size(); ++d) { + for (const std::string& term : docs[d]) { + if (matches(term)) { + ids.insert(d); + } + } + } + return {ids.begin(), ids.end()}; + } +}; + +Corpus BuildMixedCorpus() { + Corpus c; + c.doc_count = 120; + c.docs.resize(c.doc_count); + for (uint32_t d = 0; d < c.doc_count; ++d) { + std::vector& terms = c.docs[d]; + if (d < 80) { + char term[16]; + std::snprintf(term, sizeof(term), "aa_%03u", d); + terms.emplace_back(term); + } + if (d < 80 && d % 2 == 0) { + terms.emplace_back("aa_even"); + } + if (d < 50) { + char term[16]; + std::snprintf(term, sizeof(term), "ab_%03u", d); + terms.emplace_back(term); + } + char filler[16]; + std::snprintf(filler, sizeof(filler), "zz_%03u", d); + terms.emplace_back(filler); + } + return c; +} + +Corpus BuildLowDfPrefixCorpus() { + Corpus c; + c.doc_count = 96; + c.docs.resize(c.doc_count); + for (uint32_t d = 0; d < c.doc_count; ++d) { + char term[16]; + std::snprintf(term, sizeof(term), "aa_%03u", d); + c.docs[d].emplace_back(term); + } + return c; +} + +void WriteCorpus(const Corpus& c, const std::string& path) { + SpimiTermBuffer buf(/*has_positions=*/false); + for (uint32_t d = 0; d < c.docs.size(); ++d) { + const std::vector& terms = c.docs[d]; + for (uint32_t pos = 0; pos < terms.size(); ++pos) { + buf.add_token(terms[pos], d, pos); + } + } + + SniiIndexInput in; + in.index_id = 1; + in.index_suffix = "body"; + in.config = doris::snii::format::IndexConfig::kDocsOnly; + in.doc_count = c.doc_count; + in.terms = buf.finalize_sorted(); + in.target_dict_block_bytes = 2048; + + io::LocalFileWriter writer; + ASSERT_TRUE(writer.open(path).ok()); + SniiCompoundWriter compound(&writer); + ASSERT_TRUE(compound.add_logical_index(in).ok()); + ASSERT_TRUE(compound.finish().ok()); +} + +LogicalIndexReader OpenIndex(io::LocalFileReader* file, SniiSegmentReader* segment, + const std::string& path) { + EXPECT_TRUE(file->open(path).ok()); + EXPECT_TRUE(SniiSegmentReader::open(file, segment).ok()); + LogicalIndexReader idx; + EXPECT_TRUE(segment->open_index(1, "body", &idx).ok()); + return idx; +} + +LogicalIndexReader OpenMeteredIndex(io::MeteredFileReader* file, SniiSegmentReader* segment) { + EXPECT_TRUE(SniiSegmentReader::open(file, segment).ok()); + LogicalIndexReader idx; + EXPECT_TRUE(segment->open_index(1, "body", &idx).ok()); + return idx; +} + +} // namespace + +TEST(SniiPatternQuery, WildcardMatchesOracle) { + const Corpus corpus = BuildMixedCorpus(); + const std::string path = TempPath(); + WriteCorpus(corpus, path); + + io::LocalFileReader file; + SniiSegmentReader segment; + LogicalIndexReader idx = OpenIndex(&file, &segment, path); + + for (const char* pattern : {"aa_*", "aa_00?", "aa_even", "zz_0??", "missing*"}) { + std::vector got; + ASSERT_TRUE(query::wildcard_query(idx, pattern, &got).ok()) << pattern; + EXPECT_TRUE(std::ranges::is_sorted(got)) << pattern; + EXPECT_EQ(got, corpus.matching_docs([&](std::string_view term) { + return WildcardMatch(pattern, term); + })) << pattern; + } + + std::remove(path.c_str()); +} + +TEST(SniiPatternQuery, RegexpMatchesOracle) { + const Corpus corpus = BuildMixedCorpus(); + const std::string path = TempPath(); + WriteCorpus(corpus, path); + + io::LocalFileReader file; + SniiSegmentReader segment; + LogicalIndexReader idx = OpenIndex(&file, &segment, path); + + for (const char* pattern : + {"aa_00[0-9]", "aa_even", "ab_0[0-9][0-9]", "zz_1..", "no_match.*"}) { + const std::regex re(pattern); + std::vector got; + ASSERT_TRUE(query::regexp_query(idx, pattern, &got).ok()) << pattern; + EXPECT_TRUE(std::ranges::is_sorted(got)) << pattern; + EXPECT_EQ(got, corpus.matching_docs([&](std::string_view term) { + return std::regex_match(term.begin(), term.end(), re); + })) << pattern; + } + + std::vector ignored; + EXPECT_FALSE(query::regexp_query(idx, "[", &ignored).ok()); + + std::remove(path.c_str()); +} + +TEST(SniiPatternQuery, WideWildcardUsesPrefixEnumerationWithoutPerTermLookup) { + const Corpus corpus = BuildLowDfPrefixCorpus(); + const std::string path = TempPath(); + WriteCorpus(corpus, path); + + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + io::MeteredFileReader metered(&local, /*block_size=*/4096); + SniiSegmentReader segment; + LogicalIndexReader idx = OpenMeteredIndex(&metered, &segment); + + metered.reset_metrics(); + std::vector got; + ASSERT_TRUE(query::wildcard_query(idx, "aa_*", &got).ok()); + + EXPECT_EQ(got, corpus.matching_docs( + [](std::string_view term) { return WildcardMatch("aa_*", term); })); + EXPECT_LT(metered.metrics().read_at_calls, corpus.doc_count / 3) + << "wildcard_query must reuse enumerated entries, not lookup every term again"; + + std::remove(path.c_str()); +} diff --git a/be/test/storage/index/snii/query/phrase_bigram_drain_df_gate_query_test.cpp b/be/test/storage/index/snii/query/phrase_bigram_drain_df_gate_query_test.cpp new file mode 100644 index 00000000000000..9b5bafa93f40dd --- /dev/null +++ b/be/test/storage/index/snii/query/phrase_bigram_drain_df_gate_query_test.cpp @@ -0,0 +1,239 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/logical_index_writer.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" +#include "storage/index/snii_query_test_util.h" + +// G06 verification gate at the QUERY level: phrase / phrase_prefix RESULTS +// from a build whose low-df pair-keyed bigrams were dropped by the DRAIN-SIDE +// df gate (sunk from process_term; plumbed automatically by +// LogicalIndexWriter::build_blocks from input.bigram_prune_min_df) are +// identical to a G01 STRING-KEYED control build over the same logical token +// stream -- whose bigrams only process_term's flush gate prunes -- across the +// two routings the gate can influence: +// hot : df >= threshold survivor -> answered DIRECTLY from its bigram +// posting in both builds (bigram_hits seam); +// pruned : df < threshold pair -> dropped at the DRAIN in the pair build, +// by process_term in the control -> no dict entry in either, both +// take the generic positions FALLBACK with equal results -- +// including an out-of-order docid REVISIT pair, which the drain +// gate must exempt (inexact counter) and hand to process_term. +// The two segments' BYTES are also asserted identical: at query time the gate +// is entirely invisible. +using namespace doris::snii; +using namespace doris::snii::snii_test; +using doris::snii::query::phrase_prefix_query; +using doris::snii::query::phrase_query; +namespace qinternal = doris::snii::query::internal; +namespace spimi_testing = doris::snii::writer::testing; + +namespace { + +constexpr uint32_t kDocCount = 400; +constexpr uint32_t kThreshold = 3; // (gone,away) df 2 and (twist,back) df 2 drop; + // (hot,pair) df 13 survives + +struct Fixture { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; +}; + +// STRING-KEYED control feed (the G01 writer behavior): these bigrams never +// enter the pair map, so the G06 drain gate cannot touch them -- process_term +// is their only pruner. +void feed_doc_strings(writer::SpimiTermBuffer* buf, uint32_t docid, std::string_view l, + std::string_view r, uint32_t pos_base = 0) { + buf->add_token(l, docid, pos_base); + buf->add_token(r, docid, pos_base + 1); + buf->add_bigram_token(l, r, docid, pos_base); +} + +// PAIR-KEYED feed, mirroring the production _add_value_tokens + +// _add_phrase_bigram_tokens split: intern the unigrams (capturing ids), then +// add the adjacent pair by id. +void feed_doc_ids(writer::SpimiTermBuffer* buf, uint32_t docid, std::string_view l, + std::string_view r, uint32_t pos_base = 0) { + const uint32_t lid = buf->add_token_returning_id(l, docid, pos_base); + const uint32_t rid = buf->add_token_returning_id(r, docid, pos_base + 1); + buf->add_bigram_token(lid, rid, docid, pos_base); +} + +// The shared logical token stream (identical in both builds). +template +void feed_stream(writer::SpimiTermBuffer* buf, FeedPair&& feed_pair) { + // Hot pair: df 13 >= threshold, answered directly from its bigram posting. + feed_pair(buf, 0, "hot", "pair", 0); + feed_pair(buf, 1, "hot", "pair", 0); + feed_pair(buf, 2, "hot", "pair", 0); + // Pruned pair: df 2 < threshold -> the G06 drain drop (pair build) / + // process_term df prune (control). Query falls back to unigram positions. + feed_pair(buf, 3, "gone", "away", 0); + feed_pair(buf, 4, "gone", "away", 0); + // Out-of-order REVISIT pair (docids 20, 8, 20): Term::ndocs overcounts + // (3 groups, coalesced df 2), so the drain gate must exempt it; both + // builds prune it at process_term on the exact coalesced df. + feed_pair(buf, 20, "twist", "back", 0); + feed_pair(buf, 8, "twist", "back", 0); + feed_pair(buf, 20, "twist", "back", 10); + // More hot occurrences (docids keep ascending past the revisit block). + for (uint32_t d = 50; d < 60; ++d) { + feed_pair(buf, d, "hot", "pair", 0); + } + // The sentinel, as the production writer feeds it at finish(). + buf->add_token(format::make_phrase_bigram_sentinel_term(), 0, 0); +} + +Status build_fixture(Fixture* f, writer::SpimiTermBuffer* buf) { + writer::SniiIndexInput input; + input.index_id = 23; + input.index_suffix = "Body"; + input.config = format::IndexConfig::kDocsPositions; + input.doc_count = kDocCount; + input.bigram_prune_min_df = kThreshold; + input.term_source = buf; + input.bigram_ever_dropped = buf->bigram_dropped_filter(); // null: no evictions here + + writer::SniiCompoundWriter writer(&f->file); + RETURN_IF_ERROR(writer.add_logical_index(input)); + RETURN_IF_ERROR(writer.finish()); + RETURN_IF_ERROR(reader::SniiSegmentReader::open(&f->file, &f->segment_reader)); + return f->segment_reader.open_index(input.index_id, input.index_suffix, &f->index_reader); +} + +std::vector run_phrase(const reader::LogicalIndexReader& idx, + const std::vector& terms) { + std::vector docids; + assert_ok(phrase_query(idx, terms, &docids)); + return docids; +} + +std::vector run_phrase_prefix(const reader::LogicalIndexReader& idx, + const std::vector& terms) { + std::vector docids; + assert_ok(phrase_prefix_query(idx, terms, &docids)); + return docids; +} + +void reset_query_counters() { + qinternal::query_test_counters() = qinternal::QueryTestCounters {}; +} + +} // namespace + +TEST(SniiPhraseBigramDrainDfGateQuery, ResultsEqualStringKeyedControlHotAndPruned) { + writer::SpimiTermBuffer pair_buf(/*has_positions=*/true); + writer::SpimiTermBuffer control_buf(/*has_positions=*/true); + pair_buf.configure_bigram_diet(0); + control_buf.configure_bigram_diet(0); + + feed_stream(&pair_buf, feed_doc_ids); + feed_stream(&control_buf, feed_doc_strings); + ASSERT_TRUE(pair_buf.status().ok()); + ASSERT_TRUE(control_buf.status().ok()); + + // The pair build's below-threshold pairs die at the DRAIN gate (seam: only + // (gone,away); the out-of-order (twist,back) is exempt and reaches + // process_term); no bloom is ever created by those drops. + spimi_testing::reset_bigram_drain_df_drops(); + Fixture pair_fix; + assert_ok(build_fixture(&pair_fix, &pair_buf)); + EXPECT_EQ(spimi_testing::bigram_drain_df_drops(), 1U); + EXPECT_EQ(pair_buf.bigram_dropped_filter(), nullptr); + + Fixture control_fix; + assert_ok(build_fixture(&control_fix, &control_buf)); + // String-keyed bigrams never enter the drain gate. + EXPECT_EQ(spimi_testing::bigram_drain_df_drops(), 1U); + + // The gate is invisible on disk: identical segment bytes. + ASSERT_EQ(pair_fix.file.size(), control_fix.file.size()); + std::vector pair_bytes; + std::vector control_bytes; + assert_ok(pair_fix.file.read_at(0, pair_fix.file.size(), &pair_bytes)); + assert_ok(control_fix.file.read_at(0, control_fix.file.size(), &control_bytes)); + EXPECT_EQ(pair_bytes, control_bytes); + + EXPECT_EQ(pair_fix.index_reader.bigram_prune_min_df(), kThreshold); + EXPECT_EQ(control_fix.index_reader.bigram_prune_min_df(), kThreshold); + + // HOT routing: direct bigram hit in BOTH builds, equal results. + const std::vector hot {"hot", "pair"}; + std::vector want_hot {0, 1, 2}; + for (uint32_t d = 50; d < 60; ++d) { + want_hot.push_back(d); + } + reset_query_counters(); + EXPECT_EQ(run_phrase(control_fix.index_reader, hot), want_hot); + EXPECT_EQ(qinternal::query_test_counters().bigram_hits, 1U); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 0U); + reset_query_counters(); + EXPECT_EQ(run_phrase(pair_fix.index_reader, hot), want_hot); + EXPECT_EQ(qinternal::query_test_counters().bigram_hits, 1U); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 0U); + + // PRUNED routing (drain-dropped in the pair build): dict miss on a + // pruning-declared segment -> generic positions fallback, equal results. + const std::vector pruned {"gone", "away"}; + const std::vector want_pruned {3, 4}; + reset_query_counters(); + EXPECT_EQ(run_phrase(control_fix.index_reader, pruned), want_pruned); + EXPECT_EQ(qinternal::query_test_counters().bigram_hits, 0U); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 1U); + reset_query_counters(); + EXPECT_EQ(run_phrase(pair_fix.index_reader, pruned), want_pruned); + EXPECT_EQ(qinternal::query_test_counters().bigram_hits, 0U); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 1U); + + // PRUNED routing, out-of-order revisit shape: exempt from the drain gate, + // pruned by process_term on the exact coalesced df in BOTH builds; the + // fallback recovers the coalesced doc set. + const std::vector revisit {"twist", "back"}; + const std::vector want_revisit {8, 20}; + reset_query_counters(); + EXPECT_EQ(run_phrase(control_fix.index_reader, revisit), want_revisit); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 1U); + reset_query_counters(); + EXPECT_EQ(run_phrase(pair_fix.index_reader, revisit), want_revisit); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 1U); + + // PHRASE_PREFIX over the same segments: identical results for a hot head + // ("hot pai" -> "hot pair") and a drain-dropped head ("gone aw" -> + // "gone away"). + const std::vector hot_prefix {"hot", "pai"}; + EXPECT_EQ(run_phrase_prefix(control_fix.index_reader, hot_prefix), want_hot); + EXPECT_EQ(run_phrase_prefix(pair_fix.index_reader, hot_prefix), want_hot); + const std::vector pruned_prefix {"gone", "aw"}; + EXPECT_EQ(run_phrase_prefix(control_fix.index_reader, pruned_prefix), want_pruned); + EXPECT_EQ(run_phrase_prefix(pair_fix.index_reader, pruned_prefix), want_pruned); +} diff --git a/be/test/storage/index/snii/query/phrase_bigram_pair_key_query_test.cpp b/be/test/storage/index/snii/query/phrase_bigram_pair_key_query_test.cpp new file mode 100644 index 00000000000000..d3794bdc0aeec7 --- /dev/null +++ b/be/test/storage/index/snii/query/phrase_bigram_pair_key_query_test.cpp @@ -0,0 +1,282 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/logical_index_writer.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" +#include "storage/index/snii_query_test_util.h" + +// G05 verification gate: phrase / phrase_prefix query RESULTS from a build fed +// through the PAIR-KEYED bigram path (add_token_returning_id + +// add_bigram_token(left_id, right_id) -- what the production column writer now +// does) are identical to a G01 STRING-KEYED control build over the same logical +// token stream, across every routing: +// hot : df>=2 survivor -> answered DIRECTLY from its bigram posting in +// both builds (bigram_hits seam); +// pruned : df==1 pair, below the df threshold in BOTH builds -> no dict +// entry, generic positions FALLBACK, same results; +// evicted-reappearing (pair build only capped): the G04 vocab cap evicted the +// pair at df==1 under PAIR KEYING, the bloom (inserted via the +// piecewise content hash) drops the reappeared term at flush, and +// the cold fallback recovers the identical doc set the uncapped +// string-keyed control answers directly; +// phrase_prefix : the tail-expansion path over the same segments agrees. +using namespace doris::snii; +using namespace doris::snii::snii_test; +using doris::snii::query::phrase_prefix_query; +using doris::snii::query::phrase_query; +namespace qinternal = doris::snii::query::internal; +namespace wtesting = doris::snii::writer::testing; + +namespace { + +constexpr uint32_t kDocCount = 400; +constexpr uint32_t kThreshold = 2; // df==1 bigrams pruned in BOTH builds (the + // pruned-fallback routing); the reappeared + // victim's df is far above it, isolating the + // BLOOM as its only dropper in the pair build + +const std::string kHotTerm = format::make_phrase_bigram_term("hot", "pair"); +const std::string kVictimTerm = format::make_phrase_bigram_term("gone", "away"); + +struct Fixture { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; +}; + +// STRING-KEYED control feed (the G01/HEAD writer behavior). +void feed_doc_strings(writer::SpimiTermBuffer* buf, uint32_t docid, std::string_view l, + std::string_view r) { + buf->add_token(l, docid, 0); + buf->add_token(r, docid, 1); + buf->add_bigram_token(l, r, docid, 0); +} + +// PAIR-KEYED feed, mirroring the production _add_value_tokens + +// _add_phrase_bigram_tokens split: intern the unigrams (capturing ids), then +// add the adjacent pair by id. +void feed_doc_ids(writer::SpimiTermBuffer* buf, uint32_t docid, std::string_view l, + std::string_view r) { + const uint32_t lid = buf->add_token_returning_id(l, docid, 0); + const uint32_t rid = buf->add_token_returning_id(r, docid, 1); + buf->add_bigram_token(lid, rid, docid, 0); +} + +std::pair filler_pair(uint32_t i) { + std::string l = "u"; + l += static_cast('a' + (i / 26) % 26); + l += static_cast('a' + i % 26); + l += static_cast('a' + (i / 676) % 26); + std::string r = "v"; + r += static_cast('a' + i % 26); + r += static_cast('a' + (i / 26) % 26); + r += static_cast('a' + (i / 676) % 26); + return {std::move(l), std::move(r)}; +} + +// Feeds the IDENTICAL logical token stream into the (capped) pair-keyed buffer +// and the (uncapped) string-keyed control. The filler phase keeps feeding +// unique tail pairs until the capped pair build provably evicted the victim +// (probed by the COMPOSED string -- the pair eviction must have bloomed the +// identical content hash). +void feed_stream(writer::SpimiTermBuffer* pair_capped, writer::SpimiTermBuffer* str_control) { + auto both = [&](uint32_t docid, std::string_view l, std::string_view r) { + feed_doc_ids(pair_capped, docid, l, r); + feed_doc_strings(str_control, docid, l, r); + }; + // Hot pair reaches df==2 before any cap pressure -> never evictable. + both(0, "hot", "pair"); + both(1, "hot", "pair"); + // A df==1 pair that stays df==1: pruned by the df threshold in BOTH builds. + both(2, "rare", "once"); + // The victim: one doc -> df==1 when the tail pressure arrives. Fed + // explicitly so the pair build's unigram ids are captured ONCE here and + // reused by the reappearance below (identical token streams in both builds). + const uint32_t gone = pair_capped->add_token_returning_id("gone", 4, 0); + const uint32_t away = pair_capped->add_token_returning_id("away", 4, 1); + pair_capped->add_bigram_token(gone, away, 4, 0); + feed_doc_strings(str_control, 4, "gone", "away"); + // Unique tail pairs until the victim is evicted from the capped pair build. + uint32_t docid = 5; + for (uint32_t i = 0; i < 2000 && docid < 190; ++i) { + if (pair_capped->bigram_dropped_filter() != nullptr && + pair_capped->bigram_dropped_filter()->maybe_contains(kVictimTerm)) { + break; + } + const auto [l, r] = filler_pair(i); + both(docid++, l, r); + } + // The victim REAPPEARS: bigram tokens back-to-back first (the second add + // re-establishes df==2 immunity deterministically -- a term is never + // evicted by its own add's sweep), then the unigrams for those docs. The + // pair build reuses the unigram ids captured at doc 4 (unigram ids are + // stable for the buffer's lifetime). + for (uint32_t d = 200; d < 220; ++d) { + pair_capped->add_bigram_token(gone, away, d, 0); + str_control->add_bigram_token("gone", "away", d, 0); + } + for (uint32_t d = 200; d < 220; ++d) { + pair_capped->add_token("gone", d, 0); + pair_capped->add_token("away", d, 1); + str_control->add_token("gone", d, 0); + str_control->add_token("away", d, 1); + } + // More hot occurrences after the pressure (docids keep ascending). + for (uint32_t d = 300; d < 310; ++d) { + both(d, "hot", "pair"); + } + // The sentinel, as the production writer feeds it at finish(). + pair_capped->add_token(format::make_phrase_bigram_sentinel_term(), 0, 0); + str_control->add_token(format::make_phrase_bigram_sentinel_term(), 0, 0); +} + +Status build_fixture(Fixture* f, writer::SpimiTermBuffer* buf, + const writer::BigramDropFilter* dropped) { + writer::SniiIndexInput input; + input.index_id = 22; + input.index_suffix = "Body"; + input.config = format::IndexConfig::kDocsPositions; + input.doc_count = kDocCount; + input.bigram_prune_min_df = kThreshold; + input.term_source = buf; + input.bigram_ever_dropped = dropped; + + writer::SniiCompoundWriter writer(&f->file); + RETURN_IF_ERROR(writer.add_logical_index(input)); + RETURN_IF_ERROR(writer.finish()); + RETURN_IF_ERROR(reader::SniiSegmentReader::open(&f->file, &f->segment_reader)); + return f->segment_reader.open_index(input.index_id, input.index_suffix, &f->index_reader); +} + +std::vector run_phrase(const reader::LogicalIndexReader& idx, + const std::vector& terms) { + std::vector docids; + assert_ok(phrase_query(idx, terms, &docids)); + return docids; +} + +std::vector run_phrase_prefix(const reader::LogicalIndexReader& idx, + const std::vector& terms) { + std::vector docids; + assert_ok(phrase_prefix_query(idx, terms, &docids)); + return docids; +} + +void reset_query_counters() { + qinternal::query_test_counters() = qinternal::QueryTestCounters {}; +} + +} // namespace + +TEST(SniiPhraseBigramPairKeyQuery, ResultsEqualStringKeyedControlAcrossRoutings) { + writer::SpimiTermBuffer pair_buf(/*has_positions=*/true); + writer::SpimiTermBuffer control_buf(/*has_positions=*/true); + pair_buf.configure_bigram_diet(/*vocab_cap_bytes=*/2 * 1024); + // Control: same diet mode (docs-only bigrams) but UNCAPPED string-keyed + // interning -- no eviction, complete bigram vocabulary (ground truth). + control_buf.configure_bigram_diet(/*vocab_cap_bytes=*/0); + + feed_stream(&pair_buf, &control_buf); + ASSERT_TRUE(pair_buf.status().ok()); + ASSERT_TRUE(control_buf.status().ok()); + // The victim was evicted from the PAIR-KEYED build (bloomed via the + // piecewise content hash -- probed here by the composed string); the hot + // pair was not. + ASSERT_NE(pair_buf.bigram_dropped_filter(), nullptr); + ASSERT_TRUE(pair_buf.bigram_dropped_filter()->maybe_contains(kVictimTerm)); + EXPECT_FALSE(pair_buf.bigram_dropped_filter()->maybe_contains(kHotTerm)); + + wtesting::reset_bigram_prune_counters(); + Fixture pair_fix; + assert_ok(build_fixture(&pair_fix, &pair_buf, pair_buf.bigram_dropped_filter())); + // Exactly ONE df-surviving bigram was dropped by the BLOOM in the pair + // build (the reappeared victim, df 20 >= threshold 2). + EXPECT_EQ(wtesting::bigram_bloom_dropped(), 1U); + + Fixture control_fix; + assert_ok(build_fixture(&control_fix, &control_buf, control_buf.bigram_dropped_filter())); + EXPECT_EQ(pair_fix.index_reader.bigram_prune_min_df(), kThreshold); + EXPECT_EQ(control_fix.index_reader.bigram_prune_min_df(), kThreshold); + + // HOT pair: direct bigram hit in BOTH builds, equal results. + const std::vector hot {"hot", "pair"}; + std::vector want_hot {0, 1}; + for (uint32_t d = 300; d < 310; ++d) { + want_hot.push_back(d); + } + EXPECT_EQ(run_phrase(control_fix.index_reader, hot), want_hot); + reset_query_counters(); + EXPECT_EQ(run_phrase(pair_fix.index_reader, hot), want_hot); + EXPECT_EQ(qinternal::query_test_counters().bigram_hits, 1U); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 0U); + + // PRUNED df==1 pair: no dict entry in EITHER build (df gate), fallback + // routing, equal results. + const std::vector rare {"rare", "once"}; + const std::vector want_rare {2}; + EXPECT_EQ(run_phrase(control_fix.index_reader, rare), want_rare); + reset_query_counters(); + EXPECT_EQ(run_phrase(pair_fix.index_reader, rare), want_rare); + EXPECT_EQ(qinternal::query_test_counters().bigram_hits, 0U); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 1U); + + // EVICTED-REAPPEARING pair: the uncapped string-keyed control answers + // DIRECTLY from its complete bigram posting {4, 200..219}; the capped pair + // build bloom-dropped the (incomplete) reappeared term and takes the COLD + // FALLBACK to unigram positions -- recovering the identical doc set, + // including doc 4, whose posting the eviction lost. + const std::vector victim {"gone", "away"}; + std::vector want_victim {4}; + for (uint32_t d = 200; d < 220; ++d) { + want_victim.push_back(d); + } + reset_query_counters(); + EXPECT_EQ(run_phrase(control_fix.index_reader, victim), want_victim); + EXPECT_EQ(qinternal::query_test_counters().bigram_hits, 1U); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 0U); + + reset_query_counters(); + EXPECT_EQ(run_phrase(pair_fix.index_reader, victim), want_victim); + EXPECT_EQ(qinternal::query_test_counters().bigram_hits, 0U); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 1U); + + // PHRASE_PREFIX over the same segments: identical results between the + // builds for a hot head ("hot pai" -> "hot pair") and the evicted victim + // head ("gone aw" -> "gone away"). + const std::vector hot_prefix {"hot", "pai"}; + EXPECT_EQ(run_phrase_prefix(control_fix.index_reader, hot_prefix), want_hot); + EXPECT_EQ(run_phrase_prefix(pair_fix.index_reader, hot_prefix), want_hot); + const std::vector victim_prefix {"gone", "aw"}; + EXPECT_EQ(run_phrase_prefix(control_fix.index_reader, victim_prefix), want_victim); + EXPECT_EQ(run_phrase_prefix(pair_fix.index_reader, victim_prefix), want_victim); +} diff --git a/be/test/storage/index/snii/query/phrase_bigram_prune_query_test.cpp b/be/test/storage/index/snii/query/phrase_bigram_prune_query_test.cpp new file mode 100644 index 00000000000000..72abf3d8c81404 --- /dev/null +++ b/be/test/storage/index/snii/query/phrase_bigram_prune_query_test.cpp @@ -0,0 +1,504 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" +#include "storage/index/snii_query_test_util.h" + +// G01 "bigram diet" reader/writer behavior over the shared 9000-doc fixture +// (snii_query_test_util.h), built in three layouts: +// control : include_phrase_bigrams=true, threshold 0 (legacy: every hand-fed +// bigram materialized WITH positions; miss == empty) +// pruned : include_phrase_bigrams=true, threshold 64 (low-df bigrams dropped, +// the surviving df-9000 bigram(repeat,repeat) written docs-only, meta +// records the threshold) +// nobigram: include_phrase_bigrams=false, threshold 0 (no bigram terms and no +// sentinel -> the generic positions-verification path == ground truth) +// Fixture bigram dfs: (failed,order)=3, (failed,ordinal)=1, (order,ordinal)=2, +// (repeat,repeat)=9000, all consistent with the unigram position layout, so every +// routing (bigram direct / pruned fallback / generic) must produce EQUAL results. +using namespace doris::snii; +using namespace doris::snii::snii_test; +using doris::snii::query::phrase_prefix_query; +using doris::snii::query::phrase_query; +namespace qinternal = doris::snii::query::internal; + +namespace { + +constexpr uint32_t kFixtureDocs = 9000; +constexpr uint32_t kPruneThreshold = 64; + +struct Fixture { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; +}; + +void build_fixture(Fixture* f, bool include_bigrams, uint32_t bigram_prune_min_df, + uint64_t bigram_prune_max_df = 0) { + assert_ok(build_reader(&f->file, &f->segment_reader, &f->index_reader, include_bigrams, + bigram_prune_min_df, bigram_prune_max_df)); +} + +std::vector run_phrase(const reader::LogicalIndexReader& idx, + const std::vector& terms) { + std::vector docids; + assert_ok(phrase_query(idx, terms, &docids)); + return docids; +} + +std::vector all_docids(uint32_t end_exclusive) { + std::vector docids(end_exclusive); + std::iota(docids.begin(), docids.end(), 0U); + return docids; +} + +void reset_query_counters() { + qinternal::query_test_counters() = qinternal::QueryTestCounters {}; +} + +} // namespace + +// The pruned segment's meta declares the applied threshold; legacy layouts stay 0. +TEST(SniiPhraseBigramPrune, MetaDeclaresThresholdOnlyOnPrunedSegments) { + Fixture control, pruned; + build_fixture(&control, /*include_bigrams=*/true, /*threshold=*/0); + build_fixture(&pruned, /*include_bigrams=*/true, kPruneThreshold); + + EXPECT_EQ(control.index_reader.bigram_prune_min_df(), 0U); + EXPECT_EQ(pruned.index_reader.bigram_prune_min_df(), kPruneThreshold); +} + +// Verification-gate case 1a: a HOT pair (df >= threshold) survives pruning and is +// answered DIRECTLY from the bigram posting (seam: one hit, zero fallbacks), with +// results equal to the unpruned control AND the pure-positions ground truth. +TEST(SniiPhraseBigramPrune, HotPairAnsweredViaBigramNoFallback) { + Fixture control, pruned, nobigram; + build_fixture(&control, true, 0); + build_fixture(&pruned, true, kPruneThreshold); + build_fixture(&nobigram, false, 0); + + const std::vector hot {"repeat", "repeat"}; + const std::vector truth = run_phrase(nobigram.index_reader, hot); + EXPECT_EQ(truth, all_docids(kFixtureDocs)); // repeat@{0,1,2} everywhere + + EXPECT_EQ(run_phrase(control.index_reader, hot), truth); + + reset_query_counters(); + EXPECT_EQ(run_phrase(pruned.index_reader, hot), truth); + EXPECT_EQ(qinternal::query_test_counters().bigram_hits, 1U); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 0U); +} + +// Verification-gate case 1b: COLD pairs (df < threshold) are pruned from the dict; +// their 2-term phrases take the fallback (seam) and still EQUAL the unpruned +// control and the ground truth. +TEST(SniiPhraseBigramPrune, ColdPairFallsBackAndEqualsControl) { + Fixture control, pruned, nobigram; + build_fixture(&control, true, 0); + build_fixture(&pruned, true, kPruneThreshold); + build_fixture(&nobigram, false, 0); + + const std::vector> cold_phrases { + {"failed", "order"}, // pruned df=3 -> {5000, 7000, 8000} + {"failed", "ordinal"}, // pruned df=1 -> {6000} + {"order", "ordinal"}, // pruned df=2 -> {5000, 7000} + }; + const std::vector> expected { + {5000, 7000, 8000}, + {6000}, + {5000, 7000}, + }; + + for (size_t i = 0; i < cold_phrases.size(); ++i) { + const std::vector truth = run_phrase(nobigram.index_reader, cold_phrases[i]); + EXPECT_EQ(truth, expected[i]); + EXPECT_EQ(run_phrase(control.index_reader, cold_phrases[i]), truth); + + // The pruned bigram term must be absent from the dict... + bool found = false; + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + assert_ok(pruned.index_reader.lookup( + format::make_phrase_bigram_term(cold_phrases[i][0], cold_phrases[i][1]), &found, + &entry, &frq_base, &prx_base)); + EXPECT_FALSE(found); + + // ... and the query must reroute through the fallback exactly once. + reset_query_counters(); + EXPECT_EQ(run_phrase(pruned.index_reader, cold_phrases[i]), truth); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 1U); + EXPECT_EQ(qinternal::query_test_counters().bigram_hits, 0U); + } +} + +// Verification-gate case 1c (G15): a near-ubiquitous pair (df above the max-df +// upper bound) is pruned from the dict even though it clears every min gate; +// its 2-term phrase reroutes through the fallback (seam: zero hits, one +// fallback) and still EQUALS the unpruned control and the ground truth. The +// meta declares ONLY the max -- that alone must arm the dict-miss fallback. +TEST(SniiPhraseBigramPrune, MaxDfPrunedHotPairFallsBackAndEqualsControl) { + Fixture control, maxpruned, nobigram; + build_fixture(&control, true, 0); + build_fixture(&maxpruned, true, /*bigram_prune_min_df=*/0, + /*bigram_prune_max_df=*/kFixtureDocs / 5); + build_fixture(&nobigram, false, 0); + + EXPECT_EQ(maxpruned.index_reader.bigram_prune_min_df(), 0U); + EXPECT_EQ(maxpruned.index_reader.bigram_prune_max_df(), kFixtureDocs / 5); + + // (repeat,repeat) df 9000 > 1800: pruned from the dict... + const std::vector hot {"repeat", "repeat"}; + bool found = false; + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + assert_ok(maxpruned.index_reader.lookup(format::make_phrase_bigram_term("repeat", "repeat"), + &found, &entry, &frq_base, &prx_base)); + EXPECT_FALSE(found); + + // ... while the low-df pairs the MIN gate would have dropped all survive. + assert_ok(maxpruned.index_reader.lookup(format::make_phrase_bigram_term("failed", "ordinal"), + &found, &entry, &frq_base, &prx_base)); + EXPECT_TRUE(found); + + const std::vector truth = run_phrase(nobigram.index_reader, hot); + EXPECT_EQ(truth, all_docids(kFixtureDocs)); + EXPECT_EQ(run_phrase(control.index_reader, hot), truth); + + reset_query_counters(); + EXPECT_EQ(run_phrase(maxpruned.index_reader, hot), truth); + EXPECT_EQ(qinternal::query_test_counters().bigram_hits, 0U); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 1U); +} + +// Verification-gate case 1d (G15): BOTH gates armed at once (min 2, max 1800) +// split the fixture's bigram df axis into three bands -- (failed,ordinal) df 1 +// min-pruned, (order,ordinal) df 2 == min and (failed,order) df 3 kept, +// (repeat,repeat) df 9000 max-pruned. Each pruned tail's 2-term phrase must +// fall back (and equal the unpruned control), while the middle band still +// answers DIRECTLY from its bigram posting (one hit, zero fallbacks). +TEST(SniiPhraseBigramPrune, MinAndMaxGatesTogetherPruneBothTailsKeepMiddle) { + constexpr uint32_t kMin = 2; + constexpr uint64_t kMax = 1800; + Fixture control, banded; + build_fixture(&control, true, 0); + build_fixture(&banded, true, kMin, kMax); + + EXPECT_EQ(banded.index_reader.bigram_prune_min_df(), kMin); + EXPECT_EQ(banded.index_reader.bigram_prune_max_df(), kMax); + + // Dict state: both tails miss, the middle band (min and max boundaries + // included) survives. + bool found = false; + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + assert_ok(banded.index_reader.lookup(format::make_phrase_bigram_term("failed", "ordinal"), + &found, &entry, &frq_base, &prx_base)); + EXPECT_FALSE(found); // df 1 < min + assert_ok(banded.index_reader.lookup(format::make_phrase_bigram_term("order", "ordinal"), + &found, &entry, &frq_base, &prx_base)); + EXPECT_TRUE(found); // df 2 == min: kept + assert_ok(banded.index_reader.lookup(format::make_phrase_bigram_term("failed", "order"), &found, + &entry, &frq_base, &prx_base)); + EXPECT_TRUE(found); // df 3 in (min, max): kept + assert_ok(banded.index_reader.lookup(format::make_phrase_bigram_term("repeat", "repeat"), + &found, &entry, &frq_base, &prx_base)); + EXPECT_FALSE(found); // df 9000 > max + + // LOW tail: fallback engages and the answer equals the unpruned control. + const std::vector low {"failed", "ordinal"}; + const std::vector low_truth = run_phrase(control.index_reader, low); + EXPECT_EQ(low_truth, (std::vector {6000})); + reset_query_counters(); + EXPECT_EQ(run_phrase(banded.index_reader, low), low_truth); + EXPECT_EQ(qinternal::query_test_counters().bigram_hits, 0U); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 1U); + + // HIGH tail: same fallback contract, full-cardinality answer intact. + const std::vector high {"repeat", "repeat"}; + const std::vector high_truth = run_phrase(control.index_reader, high); + EXPECT_EQ(high_truth, all_docids(kFixtureDocs)); + reset_query_counters(); + EXPECT_EQ(run_phrase(banded.index_reader, high), high_truth); + EXPECT_EQ(qinternal::query_test_counters().bigram_hits, 0U); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 1U); + + // MIDDLE band: still a direct bigram hit, no fallback, equal results. + const std::vector mid {"failed", "order"}; + const std::vector mid_truth = run_phrase(control.index_reader, mid); + EXPECT_EQ(mid_truth, (std::vector {5000, 7000, 8000})); + reset_query_counters(); + EXPECT_EQ(run_phrase(banded.index_reader, mid), mid_truth); + EXPECT_EQ(qinternal::query_test_counters().bigram_hits, 1U); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 0U); +} + +// Verification-gate case 2: the surviving bigram is written DOCS-ONLY on the +// pruned segment (prx_len == 0, no inline prx bytes) yet its 2-term phrase answer +// is identical -- the bigram hit path never reads bigram positions. +TEST(SniiPhraseBigramPrune, SurvivingBigramIsDocsOnlyAndAnswersEqually) { + Fixture control, pruned; + build_fixture(&control, true, 0); + build_fixture(&pruned, true, kPruneThreshold); + + const std::string bigram = format::make_phrase_bigram_term("repeat", "repeat"); + bool found = false; + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + + assert_ok(pruned.index_reader.lookup(bigram, &found, &entry, &frq_base, &prx_base)); + ASSERT_TRUE(found); + EXPECT_EQ(entry.df, kFixtureDocs); + EXPECT_EQ(entry.prx_len, 0U); // no .prx span streamed for the bigram + EXPECT_TRUE(entry.prx_bytes.empty()); + + // The legacy control keeps bigram positions (windowed df-9000 term -> a real + // prx span), pinning that the diet actually changed the pruned layout. + format::DictEntry control_entry; + assert_ok(control.index_reader.lookup(bigram, &found, &control_entry, &frq_base, &prx_base)); + ASSERT_TRUE(found); + EXPECT_GT(control_entry.prx_len, 0U); + + EXPECT_EQ(run_phrase(pruned.index_reader, {"repeat", "repeat"}), + run_phrase(control.index_reader, {"repeat", "repeat"})); +} + +// Verification-gate case 4: LEGACY segment (no meta threshold): a bigram dict miss +// keeps meaning EMPTY (the legacy writer materialized every adjacent pair, so a +// miss proves no adjacency) -- no fallback fires even though the unigram positions +// would match. The same query on a pruned-flag segment DOES fall back and returns +// the positions truth, which is exactly the semantic the meta flag gates. +TEST(SniiPhraseBigramPrune, LegacySegmentMissStaysEmptyPrunedSegmentFallsBack) { + Fixture control, pruned, nobigram; + build_fixture(&control, true, 0); + build_fixture(&pruned, true, kPruneThreshold); + build_fixture(&nobigram, false, 0); + + // "almost"@1 / "order"@2 are adjacent in most docs, but the fixture feeds NO + // bigram(almost, order) term -- simulating a pair the pruned writer dropped. + const std::vector phrase {"almost", "order"}; + const std::vector truth = run_phrase(nobigram.index_reader, phrase); + ASSERT_FALSE(truth.empty()); // docs minus {4000 (no almost), 5000/7000/8000 (order not @2)} + EXPECT_EQ(truth.size(), kFixtureDocs - 4); + + reset_query_counters(); + EXPECT_TRUE(run_phrase(control.index_reader, phrase).empty()); // legacy: miss == empty + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 0U); + EXPECT_EQ(qinternal::query_test_counters().bigram_hits, 0U); + + reset_query_counters(); + EXPECT_EQ(run_phrase(pruned.index_reader, phrase), truth); // pruned flag: fallback + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 1U); +} + +// Verification-gate case 5: phrase_prefix regression on the pruned segment. The +// multi-tail path filters bigram tails and verifies with UNIGRAM positions only +// (untouched by the diet), so pruned results equal the legacy control's. +TEST(SniiPhraseBigramPrune, PhrasePrefixUnaffectedByPruning) { + Fixture control, pruned; + build_fixture(&control, true, 0); + build_fixture(&pruned, true, kPruneThreshold); + + const std::vector expected {5000, 6000, 7000, 8000}; + + std::vector control_docs; + assert_ok(phrase_prefix_query(control.index_reader, {"failed", "ord"}, &control_docs, 10)); + EXPECT_EQ(control_docs, expected); + + std::vector pruned_docs; + assert_ok(phrase_prefix_query(pruned.index_reader, {"failed", "ord"}, &pruned_docs, 10)); + EXPECT_EQ(pruned_docs, expected); + + // A single-tail expansion exercises the ExecuteResolvedPhraseTerms path too. + std::vector single_tail; + assert_ok(phrase_prefix_query(pruned.index_reader, {"failed", "ordi"}, &single_tail, 10)); + EXPECT_EQ(single_tail, std::vector {6000}); +} + +namespace { + +// A pprefix-focused fixture (its own index; independent of the shared 9000-doc +// one): a HOT anchor whose tail-prefix family STRADDLES the prune threshold, +// plus one REAL term living inside the hidden-marker byte region. +// vulcan @0 in docs [0, 9000) hot anchor (df 9000) +// warpcore @1 in docs [0, 8000) bigram(vulcan,warpcore) df=8000 SURVIVES +// warpdrive @1 in docs [8000, 8003) bigram(vulcan,warpdrive) df=3 PRUNED +// warpgate @1 in docs [8003, 8010) bigram(vulcan,warpgate) df=7 PRUNED +// "\x1F" "zz_raw" @1 in docs {42, 43} a raw (untokenized-value style) +// term that sorts AFTER the marker +// cluster; the "\x1F" tail prefix +// range covers every hidden bigram +// dict term on bigram-bearing +// layouts. +// include_bigrams=false builds the pure-positions ground-truth layout. +Status build_warp_reader(Fixture* f, bool include_bigrams, uint32_t bigram_prune_min_df) { + constexpr uint32_t kDocCount = 9000; + writer::SniiIndexInput input; + input.index_id = 11; + input.index_suffix = "Warp"; + input.config = format::IndexConfig::kDocsPositions; + input.doc_count = kDocCount; + input.bigram_prune_min_df = bigram_prune_min_df; + input.terms = {make_term("vulcan", docs_with_one_position(0, kDocCount, 0)), + make_term("warpcore", docs_with_one_position(0, 8000, 1)), + make_term("warpdrive", docs_with_one_position(8000, 8003, 1)), + make_term("warpgate", docs_with_one_position(8003, 8010, 1)), + make_term("\x1F" + "zz_raw", + {{.docid = 42, .positions = {1}}, {.docid = 43, .positions = {1}}})}; + if (include_bigrams) { + input.terms.push_back(make_term(format::make_phrase_bigram_sentinel_term(), + {{.docid = 0, .positions = {0}}})); + input.terms.push_back(make_term(format::make_phrase_bigram_term("vulcan", "warpcore"), + docs_with_one_position(0, 8000, 0))); + input.terms.push_back(make_term(format::make_phrase_bigram_term("vulcan", "warpdrive"), + docs_with_one_position(8000, 8003, 0))); + input.terms.push_back(make_term(format::make_phrase_bigram_term("vulcan", "warpgate"), + docs_with_one_position(8003, 8010, 0))); + } + std::ranges::sort(input.terms, + [](const writer::TermPostings& lhs, const writer::TermPostings& rhs) { + return lhs.term < rhs.term; + }); + + writer::SniiCompoundWriter writer(&f->file); + RETURN_IF_ERROR(writer.add_logical_index(input)); + RETURN_IF_ERROR(writer.finish()); + RETURN_IF_ERROR(reader::SniiSegmentReader::open(&f->file, &f->segment_reader)); + return f->segment_reader.open_index(input.index_id, input.index_suffix, &f->index_reader); +} + +void build_warp_fixture(Fixture* f, bool include_bigrams, uint32_t bigram_prune_min_df) { + assert_ok(build_warp_reader(f, include_bigrams, bigram_prune_min_df)); +} + +std::vector run_phrase_prefix(const reader::LogicalIndexReader& idx, + const std::vector& terms, + int32_t max_expansions) { + std::vector docids; + assert_ok(phrase_prefix_query(idx, terms, &docids, max_expansions)); + return docids; +} + +} // namespace + +// G01 pprefix regression, the shape production hit: the anchor is HOT and its +// tail-prefix family MIXES a surviving (anchor,tail) bigram with pruned ones. +// The pprefix answer must come from UNIGRAM positions for EVERY expanded tail: +// a path that consulted the hidden (anchor,tail) bigram postings as +// authoritative would silently drop the pruned tails' docs (their pair left no +// dict trace) and could not position-verify the surviving pair (its posting is +// docs-only). Assert exact equality between the pruned segment, the unpruned +// control, and the no-bigram ground truth -- after pinning the dict-level +// premise (SOME pairs survive, SOME are pruned) so the fixture cannot rot. +TEST(SniiPhraseBigramPrune, PhrasePrefixMixedHotAndPrunedTailPairsEqualControl) { + Fixture control, pruned, nobigram; + build_warp_fixture(&control, /*include_bigrams=*/true, /*threshold=*/0); + build_warp_fixture(&pruned, /*include_bigrams=*/true, kPruneThreshold); + build_warp_fixture(&nobigram, /*include_bigrams=*/false, /*threshold=*/0); + + // Premise: the hot pair kept a docs-only dict entry on the pruned segment, + // both cold pairs left none; the control kept all three (with positions). + bool found = false; + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + assert_ok(pruned.index_reader.lookup(format::make_phrase_bigram_term("vulcan", "warpcore"), + &found, &entry, &frq_base, &prx_base)); + ASSERT_TRUE(found); + EXPECT_EQ(entry.df, 8000U); + EXPECT_EQ(entry.prx_len, 0U); // the survivor is docs-only (G01 part B) + assert_ok(pruned.index_reader.lookup(format::make_phrase_bigram_term("vulcan", "warpdrive"), + &found, &entry, &frq_base, &prx_base)); + EXPECT_FALSE(found); + assert_ok(pruned.index_reader.lookup(format::make_phrase_bigram_term("vulcan", "warpgate"), + &found, &entry, &frq_base, &prx_base)); + EXPECT_FALSE(found); + assert_ok(control.index_reader.lookup(format::make_phrase_bigram_term("vulcan", "warpdrive"), + &found, &entry, &frq_base, &prx_base)); + ASSERT_TRUE(found); // the legacy control materialized every pair + + // Multi-tail pprefix over the mixed family: every tail verified by unigram + // positions, so all three layouts agree with the ground truth. + const std::vector query {"vulcan", "warp"}; + const std::vector truth = run_phrase_prefix(nobigram.index_reader, query, 10); + EXPECT_EQ(truth, all_docids(8010)); + EXPECT_EQ(run_phrase_prefix(control.index_reader, query, 10), truth); + EXPECT_EQ(run_phrase_prefix(pruned.index_reader, query, 10), truth); + + // A capped expansion must budget REAL tails only, identically on every + // layout: the first two dict-order tails are warpcore + warpdrive. + const std::vector capped = run_phrase_prefix(nobigram.index_reader, query, 2); + EXPECT_EQ(capped, all_docids(8003)); + EXPECT_EQ(run_phrase_prefix(control.index_reader, query, 2), capped); + EXPECT_EQ(run_phrase_prefix(pruned.index_reader, query, 2), capped); + + // 2-term MATCH_PHRASE cross-checks over the same pairs: the hot pair is + // answered bigram-DIRECT from the docs-only posting, the pruned pair takes + // the positions fallback -- both equal to the pprefix per-tail evidence. + reset_query_counters(); + EXPECT_EQ(run_phrase(pruned.index_reader, {"vulcan", "warpcore"}), all_docids(8000)); + EXPECT_EQ(qinternal::query_test_counters().bigram_hits, 1U); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 0U); + reset_query_counters(); + EXPECT_EQ(run_phrase(pruned.index_reader, {"vulcan", "warpdrive"}), + (std::vector {8000, 8001, 8002})); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 1U); + EXPECT_EQ(qinternal::query_test_counters().bigram_hits, 0U); +} + +// G01 pprefix regression: hidden bigram dict terms that fall inside the tail +// prefix range must NOT consume max_expansions slots. Pre-fix, +// prefix_terms(max_expansions) filled the whole budget with hidden terms and +// only then erased them, silently dropping the REAL tail -- and because the +// pruned layout materializes FEWER hidden terms (sentinel + 1 survivor) than +// the legacy control (sentinel + 3 pairs), the same query returned DIFFERENT +// results per segment layout. The raw "\x1F"-lead term stands in for an +// untokenized value whose prefix range overlaps the hidden bigram cluster. +TEST(SniiPhraseBigramPrune, PhrasePrefixExpansionBudgetIgnoresHiddenBigramTerms) { + Fixture control, pruned, nobigram; + build_warp_fixture(&control, /*include_bigrams=*/true, /*threshold=*/0); + build_warp_fixture(&pruned, /*include_bigrams=*/true, kPruneThreshold); + build_warp_fixture(&nobigram, /*include_bigrams=*/false, /*threshold=*/0); + + const std::vector query {"vulcan", "\x1F"}; + const std::vector truth {42, 43}; // vulcan@0 followed by the raw term@1 + + EXPECT_EQ(run_phrase_prefix(nobigram.index_reader, query, 3), truth); + EXPECT_EQ(run_phrase_prefix(control.index_reader, query, 3), truth); + EXPECT_EQ(run_phrase_prefix(pruned.index_reader, query, 3), truth); +} diff --git a/be/test/storage/index/snii/query/phrase_bigram_vocab_cap_query_test.cpp b/be/test/storage/index/snii/query/phrase_bigram_vocab_cap_query_test.cpp new file mode 100644 index 00000000000000..99887c8c5e6896 --- /dev/null +++ b/be/test/storage/index/snii/query/phrase_bigram_vocab_cap_query_test.cpp @@ -0,0 +1,243 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/logical_index_writer.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" +#include "storage/index/snii_query_test_util.h" + +// G04 verification gate: phrase query RESULTS are identical between a +// vocab-capped build (df==1 bigram tail evicted + bloom-dropped at flush) and +// an uncapped control, across every routing: +// hot : pair never evicted (df>=2 before any cap pressure) -> answered +// DIRECTLY from its bigram posting in BOTH builds; +// warm : small survivor, direct in both; +// evicted-reappearing : evicted at df==1, reappears later; the control +// answers DIRECTLY (complete bigram posting incl. the pre-eviction +// doc), the capped build has NO dict entry (bloom drop) and takes the +// G01 COLD FALLBACK to unigram positions -- recovering the SAME doc +// set, including the docid the evicted posting lost. +// The same token stream (unigrams with real positions + zero-alloc bigram +// adds + the sentinel, mirroring the production column writer) feeds both +// builds through the streaming term_source path. +using namespace doris::snii; +using namespace doris::snii::snii_test; +using doris::snii::query::phrase_query; +namespace qinternal = doris::snii::query::internal; +namespace wtesting = doris::snii::writer::testing; + +namespace { + +constexpr uint32_t kDocCount = 400; +constexpr uint32_t kThreshold = 1; // >0: prune mode; every final df survives the + // df gate, isolating the BLOOM as the only + // dropper of the evicted-reappearing pair + +const std::string kHotTerm = format::make_phrase_bigram_term("hot", "pair"); +const std::string kVictimTerm = format::make_phrase_bigram_term("gone", "away"); + +struct Fixture { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; +}; + +// Feeds one document containing exactly the two adjacent words l r (unigram +// tokens at positions 0/1 + the hidden bigram), mirroring the production +// _add_value_tokens + _add_phrase_bigram_tokens split. +void feed_pair_doc(writer::SpimiTermBuffer* buf, uint32_t docid, std::string_view l, + std::string_view r) { + buf->add_token(l, docid, 0); + buf->add_token(r, docid, 1); + buf->add_bigram_token(l, r, docid, 0); +} + +std::pair filler_pair(uint32_t i) { + std::string l = "u"; + l += static_cast('a' + (i / 26) % 26); + l += static_cast('a' + i % 26); + l += static_cast('a' + (i / 676) % 26); + std::string r = "v"; + r += static_cast('a' + i % 26); + r += static_cast('a' + (i / 26) % 26); + r += static_cast('a' + (i / 676) % 26); + return {std::move(l), std::move(r)}; +} + +// Feeds the IDENTICAL token stream into both buffers. `capped` has the diet + +// cap configured; the filler phase keeps feeding unique tail pairs until the +// capped buffer provably evicted the victim (bounded; asserted by the caller). +void feed_stream(writer::SpimiTermBuffer* capped, writer::SpimiTermBuffer* control) { + auto both = [&](uint32_t docid, std::string_view l, std::string_view r) { + feed_pair_doc(capped, docid, l, r); + feed_pair_doc(control, docid, l, r); + }; + // Hot + warm pairs reach df==2 before any cap pressure -> never evictable. + both(0, "hot", "pair"); + both(1, "hot", "pair"); + both(2, "warm", "mild"); + both(3, "warm", "mild"); + // The victim: one doc -> df==1 when the tail pressure arrives. + both(4, "gone", "away"); + // Unique tail pairs until the victim is evicted (cursor-order sweep; the + // loop bound fails the test loudly if it never fires). Docids 5..154+. + uint32_t docid = 5; + for (uint32_t i = 0; i < 2000 && docid < 190; ++i) { + if (capped->bigram_dropped_filter() != nullptr && + capped->bigram_dropped_filter()->maybe_contains(kVictimTerm)) { + break; + } + const auto [l, r] = filler_pair(i); + both(docid++, l, r); + } + // The victim REAPPEARS: bigram tokens fed back-to-back first (the second + // add re-establishes df==2 immunity deterministically -- a term is never + // evicted by its own add's sweep), then the unigrams for those docs. + for (uint32_t d = 200; d < 220; ++d) { + capped->add_bigram_token("gone", "away", d, 0); + control->add_bigram_token("gone", "away", d, 0); + } + for (uint32_t d = 200; d < 220; ++d) { + for (writer::SpimiTermBuffer* buf : {capped, control}) { + buf->add_token("gone", d, 0); + buf->add_token("away", d, 1); + } + } + // More hot occurrences after the pressure (docids keep ascending). + for (uint32_t d = 300; d < 310; ++d) { + both(d, "hot", "pair"); + } + // The sentinel, as the production writer feeds it at finish(). + capped->add_token(format::make_phrase_bigram_sentinel_term(), 0, 0); + control->add_token(format::make_phrase_bigram_sentinel_term(), 0, 0); +} + +Status build_fixture(Fixture* f, writer::SpimiTermBuffer* buf, + const writer::BigramDropFilter* dropped) { + writer::SniiIndexInput input; + input.index_id = 21; + input.index_suffix = "Body"; + input.config = format::IndexConfig::kDocsPositions; + input.doc_count = kDocCount; + input.bigram_prune_min_df = kThreshold; + input.term_source = buf; + input.bigram_ever_dropped = dropped; + + writer::SniiCompoundWriter writer(&f->file); + RETURN_IF_ERROR(writer.add_logical_index(input)); + RETURN_IF_ERROR(writer.finish()); + RETURN_IF_ERROR(reader::SniiSegmentReader::open(&f->file, &f->segment_reader)); + return f->segment_reader.open_index(input.index_id, input.index_suffix, &f->index_reader); +} + +std::vector run_phrase(const reader::LogicalIndexReader& idx, + const std::vector& terms) { + std::vector docids; + assert_ok(phrase_query(idx, terms, &docids)); + return docids; +} + +void reset_query_counters() { + qinternal::query_test_counters() = qinternal::QueryTestCounters {}; +} + +} // namespace + +TEST(SniiPhraseBigramVocabCap, ResultsEqualControlAcrossHotAndEvictedRoutings) { + writer::SpimiTermBuffer capped_buf(/*has_positions=*/true); + writer::SpimiTermBuffer control_buf(/*has_positions=*/true); + capped_buf.configure_bigram_diet(/*vocab_cap_bytes=*/2 * 1024); + // Control: same diet mode (docs-only bigrams, as production prune mode + // implies) but UNCAPPED -- no eviction, complete bigram vocabulary. + control_buf.configure_bigram_diet(/*vocab_cap_bytes=*/0); + + feed_stream(&capped_buf, &control_buf); + ASSERT_TRUE(capped_buf.status().ok()); + ASSERT_TRUE(control_buf.status().ok()); + // The victim must have been evicted (and is therefore in the bloom); the + // hot pair must not be. + ASSERT_NE(capped_buf.bigram_dropped_filter(), nullptr); + ASSERT_TRUE(capped_buf.bigram_dropped_filter()->maybe_contains(kVictimTerm)); + EXPECT_FALSE(capped_buf.bigram_dropped_filter()->maybe_contains(kHotTerm)); + + wtesting::reset_bigram_prune_counters(); + Fixture capped; + assert_ok(build_fixture(&capped, &capped_buf, capped_buf.bigram_dropped_filter())); + // Exactly ONE df-surviving bigram was dropped by the BLOOM (the reappeared + // victim; threshold 1 means the df gate dropped nothing that reached it). + EXPECT_EQ(wtesting::bigram_bloom_dropped(), 1U); + + Fixture control; + assert_ok(build_fixture(&control, &control_buf, control_buf.bigram_dropped_filter())); + EXPECT_EQ(control.index_reader.bigram_prune_min_df(), kThreshold); + EXPECT_EQ(capped.index_reader.bigram_prune_min_df(), kThreshold); + + // HOT pair: direct bigram hit in BOTH builds, equal results. + const std::vector hot {"hot", "pair"}; + std::vector want_hot {0, 1}; + for (uint32_t d = 300; d < 310; ++d) { + want_hot.push_back(d); + } + EXPECT_EQ(run_phrase(control.index_reader, hot), want_hot); + reset_query_counters(); + EXPECT_EQ(run_phrase(capped.index_reader, hot), want_hot); + EXPECT_EQ(qinternal::query_test_counters().bigram_hits, 1U); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 0U); + + // WARM small survivor: direct in both, equal. + const std::vector warm {"warm", "mild"}; + const std::vector want_warm {2, 3}; + EXPECT_EQ(run_phrase(control.index_reader, warm), want_warm); + reset_query_counters(); + EXPECT_EQ(run_phrase(capped.index_reader, warm), want_warm); + EXPECT_EQ(qinternal::query_test_counters().bigram_hits, 1U); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 0U); + + // EVICTED-REAPPEARING pair: the control answers DIRECTLY from its complete + // bigram posting {4, 200..219}; the capped build's bigram was bloom-dropped + // (its surviving posting would have LOST doc 4), so it takes the COLD + // FALLBACK to unigram positions -- and recovers the identical doc set. + const std::vector victim {"gone", "away"}; + std::vector want_victim {4}; + for (uint32_t d = 200; d < 220; ++d) { + want_victim.push_back(d); + } + reset_query_counters(); + EXPECT_EQ(run_phrase(control.index_reader, victim), want_victim); + EXPECT_EQ(qinternal::query_test_counters().bigram_hits, 1U); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 0U); + + reset_query_counters(); + EXPECT_EQ(run_phrase(capped.index_reader, victim), want_victim); + EXPECT_EQ(qinternal::query_test_counters().bigram_hits, 0U); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 1U); +} diff --git a/be/test/storage/index/snii/query/phrase_prefix_query_test.cpp b/be/test/storage/index/snii/query/phrase_prefix_query_test.cpp new file mode 100644 index 00000000000000..8d7ca5de5c007e --- /dev/null +++ b/be/test/storage/index/snii/query/phrase_prefix_query_test.cpp @@ -0,0 +1,590 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/io/metered_file_reader.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" +#include "storage/index/snii_query_test_util.h" + +using namespace doris::snii; +using namespace doris::snii::reader; +using namespace doris::snii::writer; +using doris::Status; + +namespace { + +std::string TempPath() { + static int counter = 0; + return "/tmp/snii_phrase_prefix_query_" + std::to_string(getpid()) + "_" + + std::to_string(counter++) + ".idx"; +} + +bool HasPrefix(const std::string& term, const std::string& prefix) { + return term.size() >= prefix.size() && term.starts_with(prefix); +} + +struct Corpus { + std::vector> docs; + + std::vector phrase_prefix_docs(const std::vector& terms) const { + std::vector out; + if (terms.empty()) { + return out; + } + for (uint32_t d = 0; d < docs.size(); ++d) { + const std::vector& doc = docs[d]; + bool match = false; + if (terms.size() == 1) { + for (const std::string& term : doc) { + if (HasPrefix(term, terms.front())) { + match = true; + break; + } + } + } else if (doc.size() >= terms.size()) { + for (size_t start = 0; start + terms.size() <= doc.size(); ++start) { + bool exact = true; + for (size_t i = 0; i + 1 < terms.size(); ++i) { + if (doc[start + i] != terms[i]) { + exact = false; + break; + } + } + if (exact && HasPrefix(doc[start + terms.size() - 1], terms.back())) { + match = true; + break; + } + } + } + if (match) { + out.push_back(d); + } + } + return out; + } + + // Truncation-aware oracle: reproduces the byte-exact max_expansions semantics + // the query must honour -- enumerate the REAL tail terms sharing the prefix in + // lexicographic (dict) order, keep only the first `max_expansions`, then match + // exactly as phrase_prefix_docs but restricted to that surviving tail set. + // (These corpora carry no hidden phrase-bigram terms, so the corpus vocabulary + // with the prefix IS the index's real-term enumeration for it.) + std::vector phrase_prefix_docs_capped(const std::vector& terms, + int32_t max_expansions) const { + if (max_expansions <= 0 || terms.size() < 2) { + return phrase_prefix_docs(terms); + } + std::set vocab; + for (const std::vector& doc : docs) { + for (const std::string& t : doc) { + if (HasPrefix(t, terms.back())) { + vocab.insert(t); + } + } + } + std::set allowed; + int32_t taken = 0; + for (const std::string& t : vocab) { // std::set iterates ascending (dict order) + if (taken >= max_expansions) { + break; + } + allowed.insert(t); + ++taken; + } + std::vector out; + for (uint32_t d = 0; d < docs.size(); ++d) { + const std::vector& doc = docs[d]; + if (doc.size() < terms.size()) { + continue; + } + bool match = false; + for (size_t start = 0; start + terms.size() <= doc.size() && !match; ++start) { + bool exact = true; + for (size_t i = 0; i + 1 < terms.size(); ++i) { + if (doc[start + i] != terms[i]) { + exact = false; + break; + } + } + if (exact && allowed.contains(doc[start + terms.size() - 1])) { + match = true; + } + } + if (match) { + out.push_back(d); + } + } + return out; + } +}; + +Corpus BuildPhraseCorpus() { + Corpus c; + c.docs = {{"quick", "brown", "fox"}, {"quick", "blue", "fox"}, {"quick", "bronze", "fox"}, + {"slow", "brown", "fox"}, {"quick", "brownish"}, {"quick", "brown", "fossil"}, + {"quick", "brown", "fog"}, {"quick", "brown"}, {"brown", "fox", "quick"}}; + return c; +} + +Corpus BuildWideTailCorpus() { + Corpus c; + c.docs.resize(96); + for (uint32_t d = 0; d < c.docs.size(); ++d) { + c.docs[d].emplace_back(d == 0 ? "lead" : "other"); + char term[16]; + std::snprintf(term, sizeof(term), "aa_%03u", d); + c.docs[d].emplace_back(term); + } + return c; +} + +Corpus BuildRepeatedExactCorpus() { + Corpus c; + c.docs = {{"x", "x", "brown"}, + {"x", "y", "brown"}, + {"x", "brown", "x"}, + {"x", "x", "bronze"}, + {"x", "x", "blue"}}; + return c; +} + +Corpus BuildSharedExactWideTailCorpus() { + Corpus c; + c.docs.resize(768); + for (uint32_t d = 0; d < c.docs.size(); ++d) { + c.docs[d].emplace_back("lead"); + char term[16]; + std::snprintf(term, sizeof(term), "aa_%03u", d); + c.docs[d].emplace_back(term); + } + return c; +} + +// Large corpus whose leading exact term and every tail expansion have high df, so +// their postings span MULTIPLE windows and the merge advances its cursors across +// window boundaries. Most docs match ("lead" @0 then a "res_" tail @1); a slice +// injects a filler token so the tail is not adjacent (must NOT match), and a slice +// omits the leading term (must NOT match) -- forcing the cross-window sweep to +// reject as well as accept. +Corpus BuildCrossWindowTailCorpus() { + Corpus c; + c.docs.resize(5000); + const char* const tails[] = {"res_a", "res_b", "res_c"}; + for (uint32_t d = 0; d < c.docs.size(); ++d) { + if (d % 50 == 7) { + c.docs[d] = {"lead", "gap", tails[d % 3]}; // tail not adjacent to lead + } else if (d % 50 == 11) { + c.docs[d] = {"nolead", tails[d % 3]}; // leading term absent + } else { + c.docs[d] = {"lead", tails[d % 3]}; + } + } + return c; +} + +// CJK / multi-byte corpus. Leading term and tail prefix are UTF-8; prefix testing +// is byte-wise, matching the index's dict enumeration order. +Corpus BuildCjkTailCorpus() { + Corpus c; + const char* const tails[] = {"\xE7\xBB\x93\xE6\x9E\x9C\xE7\x94\xB2", // 结果甲 + "\xE7\xBB\x93\xE6\x9E\x9C\xE4\xB9\x99", // 结果乙 + "\xE7\xBB\x93\xE6\x9E\x9C\xE4\xB8\x99"}; // 结果丙 + const std::string lead = "\xE8\xBF\x9E\xE6\x8E\xA5"; // 连接 + c.docs.resize(120); + for (uint32_t d = 0; d < c.docs.size(); ++d) { + if (d % 20 == 3) { + c.docs[d] = {lead, "\xE9\x97\xB4\xE9\x9A\x94", + tails[d % 3]}; // 间隔 filler, not adjacent + } else { + c.docs[d] = {lead, tails[d % 3]}; + } + } + return c; +} + +// Two leading exact terms that both occur but are NEVER adjacent, so the leading +// phrase conjunction yields an empty expected-position set -- the multi-tail +// branch's `expected.docs.empty()` early return -- even though the tail prefix +// expands to several real terms. +Corpus BuildEmptyExpectedCorpus() { + Corpus c; + c.docs = {{"alpha", "x", "beta", "res_a"}, + {"alpha", "y", "beta", "res_b"}, + {"beta", "alpha", "res_c"}, + {"alpha", "z", "beta"}}; + return c; +} + +void WriteCorpus(const Corpus& c, const std::string& path) { + SpimiTermBuffer buf(/*has_positions=*/true); + for (uint32_t d = 0; d < c.docs.size(); ++d) { + const std::vector& terms = c.docs[d]; + for (uint32_t pos = 0; pos < terms.size(); ++pos) { + buf.add_token(terms[pos], d, pos); + } + } + + SniiIndexInput in; + in.index_id = 1; + in.index_suffix = "body"; + in.config = doris::snii::format::IndexConfig::kDocsPositionsScoring; + in.doc_count = static_cast(c.docs.size()); + in.encoded_norms.assign(c.docs.size(), 1); + in.terms = buf.finalize_sorted(); + in.target_dict_block_bytes = 2048; + + io::LocalFileWriter writer; + ASSERT_TRUE(writer.open(path).ok()); + SniiCompoundWriter compound(&writer); + ASSERT_TRUE(compound.add_logical_index(in).ok()); + ASSERT_TRUE(compound.finish().ok()); +} + +LogicalIndexReader OpenIndex(io::LocalFileReader* file, SniiSegmentReader* segment, + const std::string& path) { + EXPECT_TRUE(file->open(path).ok()); + EXPECT_TRUE(SniiSegmentReader::open(file, segment).ok()); + LogicalIndexReader idx; + EXPECT_TRUE(segment->open_index(1, "body", &idx).ok()); + return idx; +} + +LogicalIndexReader OpenMeteredIndex(io::MeteredFileReader* file, SniiSegmentReader* segment) { + EXPECT_TRUE(SniiSegmentReader::open(file, segment).ok()); + LogicalIndexReader idx; + EXPECT_TRUE(segment->open_index(1, "body", &idx).ok()); + return idx; +} + +} // namespace + +TEST(SniiPhrasePrefixQuery, MatchesPositionOracle) { + const Corpus corpus = BuildPhraseCorpus(); + const std::string path = TempPath(); + WriteCorpus(corpus, path); + + io::LocalFileReader file; + SniiSegmentReader segment; + LogicalIndexReader idx = OpenIndex(&file, &segment, path); + + const std::vector> cases = { + {"quick", "bro"}, {"quick", "brown", "fo"}, {"slow", "bro"}, + {"absent", "bro"}, {"quick", "missing"}, {"bro"}}; + for (const std::vector& terms : cases) { + std::vector got; + const Status st = query::phrase_prefix_query(idx, terms, &got); + ASSERT_TRUE(st.ok()) << st.to_string(); + EXPECT_TRUE(std::ranges::is_sorted(got)); + EXPECT_EQ(got, corpus.phrase_prefix_docs(terms)); + } + + std::remove(path.c_str()); +} + +TEST(SniiPhrasePrefixQuery, WideTailPrefixAvoidsPerExpansionLookup) { + const Corpus corpus = BuildWideTailCorpus(); + const std::string path = TempPath(); + WriteCorpus(corpus, path); + + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + io::MeteredFileReader metered(&local, /*block_size=*/4096); + SniiSegmentReader segment; + LogicalIndexReader idx = OpenMeteredIndex(&metered, &segment); + + metered.reset_metrics(); + const std::vector terms = {"lead", "aa_"}; + std::vector got; + const Status st = query::phrase_prefix_query(idx, terms, &got); + ASSERT_TRUE(st.ok()) << st.to_string(); + + EXPECT_EQ(got, corpus.phrase_prefix_docs(terms)); + EXPECT_LT(metered.metrics().read_at_calls, corpus.docs.size() / 3) + << "phrase_prefix_query must reuse PrefixHit entries, not lookup every tail term"; + + std::remove(path.c_str()); +} + +TEST(SniiPhrasePrefixQuery, RepeatedExactTermsMatchPositionOracle) { + const Corpus corpus = BuildRepeatedExactCorpus(); + const std::string path = TempPath(); + WriteCorpus(corpus, path); + + io::LocalFileReader file; + SniiSegmentReader segment; + LogicalIndexReader idx = OpenIndex(&file, &segment, path); + + const std::vector> cases = { + {"x", "x", "br"}, {"x", "brown", "x"}, {"x", "x", "missing"}}; + for (const std::vector& terms : cases) { + std::vector got; + const Status st = query::phrase_prefix_query(idx, terms, &got); + ASSERT_TRUE(st.ok()) << st.to_string(); + EXPECT_EQ(got, corpus.phrase_prefix_docs(terms)); + } + + std::remove(path.c_str()); +} + +TEST(SniiPhrasePrefixQuery, WideTailPrefixReusesExactTermPostingReads) { + const Corpus corpus = BuildSharedExactWideTailCorpus(); + const std::string path = TempPath(); + WriteCorpus(corpus, path); + + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + io::MeteredFileReader metered(&local, /*block_size=*/4096); + SniiSegmentReader segment; + LogicalIndexReader idx = OpenMeteredIndex(&metered, &segment); + + metered.reset_metrics(); + const std::vector terms = {"lead", "aa_"}; + std::vector got; + const Status st = query::phrase_prefix_query(idx, terms, &got); + ASSERT_TRUE(st.ok()) << st.to_string(); + + EXPECT_EQ(got, corpus.phrase_prefix_docs(terms)); + EXPECT_LT(metered.metrics().read_at_calls, corpus.docs.size() / 4) + << "wide phrase-prefix must not re-read exact term postings for every tail hit"; + + std::remove(path.c_str()); +} + +// --------------------------------------------------------------------------- +// Merged multi-tail path (CollectMergedTailMatches): the per-tail verify+union +// loop was replaced by a single batched, forward-merge sweep. Every case below +// asserts the merged result equals the independent position oracle -- i.e. is +// identical to the old per-tail semantics -- across the 0/1/many-expansion +// trichotomy, max_expansions truncation, an empty expected set, cross-window +// positions, CJK/unicode terms, and a df-pruned bigram segment. +// --------------------------------------------------------------------------- + +namespace qinternal = doris::snii::query::internal; + +// Zero expansions: the tail prefix matches no real term -> empty result. Also +// exercises the single-expansion (untouched) branch for comparison. +TEST(SniiPhrasePrefixMerge, ZeroAndSingleExpansionMatchOracle) { + const Corpus corpus = BuildSharedExactWideTailCorpus(); + const std::string path = TempPath(); + WriteCorpus(corpus, path); + + io::LocalFileReader file; + SniiSegmentReader segment; + LogicalIndexReader idx = OpenIndex(&file, &segment, path); + + const std::vector zero_terms = {"lead", "zzz"}; + std::vector zero_got; + ASSERT_TRUE(query::phrase_prefix_query(idx, zero_terms, &zero_got).ok()); + EXPECT_TRUE(zero_got.empty()); + EXPECT_EQ(zero_got, corpus.phrase_prefix_docs(zero_terms)); + + // Prefix "aa_000" matches exactly one full term -> single-tail branch. + const std::vector single_terms = {"lead", "aa_000"}; + std::vector single_got; + ASSERT_TRUE(query::phrase_prefix_query(idx, single_terms, &single_got).ok()); + EXPECT_TRUE(std::ranges::is_sorted(single_got)); + EXPECT_EQ(single_got, corpus.phrase_prefix_docs(single_terms)); + + std::remove(path.c_str()); +} + +// Many expansions spanning several resident-capped groups (768 tails, batch 32). +// The merged/grouped path must equal the full oracle and stay sorted. +TEST(SniiPhrasePrefixMerge, ManyExpansionGroupsMatchOracle) { + const Corpus corpus = BuildSharedExactWideTailCorpus(); + const std::string path = TempPath(); + WriteCorpus(corpus, path); + + io::LocalFileReader file; + SniiSegmentReader segment; + LogicalIndexReader idx = OpenIndex(&file, &segment, path); + + const std::vector terms = {"lead", "aa_"}; + std::vector got; + ASSERT_TRUE(query::phrase_prefix_query(idx, terms, &got).ok()); + EXPECT_TRUE(std::ranges::is_sorted(got)); + EXPECT_EQ(got, corpus.phrase_prefix_docs(terms)); + // The expected-docid projection must still be built exactly once per query + // (hoisted out of the per-tail loop), proving the multi-tail branch ran. + qinternal::query_test_counters() = qinternal::QueryTestCounters {}; + ASSERT_TRUE(query::phrase_prefix_query(idx, terms, &got).ok()); + EXPECT_EQ(qinternal::query_test_counters().expected_docids_build, 1U); + + std::remove(path.c_str()); +} + +// max_expansions truncation is byte-exact: lexicographic order, real unigrams +// only, and independent of how the tails are later grouped for the merge. +TEST(SniiPhrasePrefixMerge, MaxExpansionsTruncationMatchesOracle) { + const Corpus corpus = BuildSharedExactWideTailCorpus(); + const std::string path = TempPath(); + WriteCorpus(corpus, path); + + io::LocalFileReader file; + SniiSegmentReader segment; + LogicalIndexReader idx = OpenIndex(&file, &segment, path); + + const std::vector terms = {"lead", "aa_"}; + for (int32_t cap : {1, 10, 32, 50, 100, 768, 1000}) { + std::vector got; + ASSERT_TRUE(query::phrase_prefix_query(idx, terms, &got, cap).ok()) << "cap=" << cap; + EXPECT_TRUE(std::ranges::is_sorted(got)) << "cap=" << cap; + EXPECT_EQ(got, corpus.phrase_prefix_docs_capped(terms, cap)) << "cap=" << cap; + } + + std::remove(path.c_str()); +} + +// An empty expected set (two leading terms never adjacent) short-circuits the +// multi-tail branch to an empty result even though the tail prefix expands. +TEST(SniiPhrasePrefixMerge, EmptyExpectedSetReturnsEmpty) { + const Corpus corpus = BuildEmptyExpectedCorpus(); + const std::string path = TempPath(); + WriteCorpus(corpus, path); + + io::LocalFileReader file; + SniiSegmentReader segment; + LogicalIndexReader idx = OpenIndex(&file, &segment, path); + + const std::vector terms = {"alpha", "beta", "res_"}; + std::vector got; + ASSERT_TRUE(query::phrase_prefix_query(idx, terms, &got).ok()); + EXPECT_TRUE(got.empty()); + EXPECT_EQ(got, corpus.phrase_prefix_docs(terms)); + + std::remove(path.c_str()); +} + +// Cross-window positions: high-df leading + tails span multiple windows, and the +// forward merge must accept adjacent matches and reject the non-adjacent / +// leading-absent injections while sweeping across window boundaries. +TEST(SniiPhrasePrefixMerge, CrossWindowPositionsMatchOracle) { + const Corpus corpus = BuildCrossWindowTailCorpus(); + const std::string path = TempPath(); + WriteCorpus(corpus, path); + + io::LocalFileReader file; + SniiSegmentReader segment; + LogicalIndexReader idx = OpenIndex(&file, &segment, path); + + const std::vector terms = {"lead", "res_"}; + std::vector got; + ASSERT_TRUE(query::phrase_prefix_query(idx, terms, &got).ok()); + EXPECT_TRUE(std::ranges::is_sorted(got)); + EXPECT_EQ(got, corpus.phrase_prefix_docs(terms)); + + std::remove(path.c_str()); +} + +// CJK / multi-byte terms merge identically to the oracle. +TEST(SniiPhrasePrefixMerge, CjkUnicodeTailsMatchOracle) { + const Corpus corpus = BuildCjkTailCorpus(); + const std::string path = TempPath(); + WriteCorpus(corpus, path); + + io::LocalFileReader file; + SniiSegmentReader segment; + LogicalIndexReader idx = OpenIndex(&file, &segment, path); + + const std::vector terms = {"\xE8\xBF\x9E\xE6\x8E\xA5", // 连接 + "\xE7\xBB\x93\xE6\x9E\x9C"}; // 结果 + std::vector got; + ASSERT_TRUE(query::phrase_prefix_query(idx, terms, &got).ok()); + EXPECT_TRUE(std::ranges::is_sorted(got)); + EXPECT_EQ(got, corpus.phrase_prefix_docs(terms)); + + std::remove(path.c_str()); +} + +// A df-pruned bigram segment must yield the SAME multi-tail phrase-prefix result +// as the unpruned and bigram-free segments: the merged path verifies against +// unigram positions only and NEVER consults (possibly incomplete) bigram +// postings. Query {"failed","ord"} expands to the unigram tails order/ordinal. +TEST(SniiPhrasePrefixMerge, BigramPrunedSegmentMatchesUnigramPath) { + namespace snii_test = doris::snii::snii_test; + const std::vector terms = {"failed", "ord"}; + + auto run = [&](bool include_bigrams, uint32_t prune_min_df) { + snii_test::MemoryFile file; + SniiSegmentReader segment; + LogicalIndexReader idx; + EXPECT_TRUE( + snii_test::build_reader(&file, &segment, &idx, include_bigrams, prune_min_df).ok()); + std::vector got; + EXPECT_TRUE(query::phrase_prefix_query(idx, terms, &got).ok()); + EXPECT_TRUE(std::ranges::is_sorted(got)); + return got; + }; + + const std::vector baseline = run(/*include_bigrams=*/false, /*prune_min_df=*/0); + const std::vector unpruned = run(/*include_bigrams=*/true, /*prune_min_df=*/0); + const std::vector pruned = run(/*include_bigrams=*/true, /*prune_min_df=*/2); + + EXPECT_FALSE(baseline.empty()); // the query must actually match some docs + EXPECT_EQ(unpruned, baseline) << "bigram presence must not change phrase-prefix result"; + EXPECT_EQ(pruned, baseline) << "bigram df-pruning must not change phrase-prefix result"; +} + +// A MULTI-leading-term phrase-prefix ("failed order ord*") must NOT take the +// 2-term bigram fast path: the bigram pair (order, ord_expansion) only proves +// the LAST two terms are adjacent, not that "failed order" precedes them. Using +// it would over-match docs that have "order ord..." without the leading +// "failed". The single-leading guard routes this to full position verification; +// the result must be identical whether or not the segment carries bigrams (and +// must equal the brute-force position oracle would -- here anchored by the +// no-bigram baseline, which is pure position verification). +TEST(SniiPhrasePrefixMerge, MultiLeadingTermIgnoresBigramFastPath) { + namespace snii_test = doris::snii::snii_test; + const std::vector terms = {"failed", "order", "ord"}; + + auto run = [&](bool include_bigrams, uint32_t prune_min_df) { + snii_test::MemoryFile file; + SniiSegmentReader segment; + LogicalIndexReader idx; + EXPECT_TRUE( + snii_test::build_reader(&file, &segment, &idx, include_bigrams, prune_min_df).ok()); + std::vector got; + EXPECT_TRUE(query::phrase_prefix_query(idx, terms, &got).ok()); + EXPECT_TRUE(std::ranges::is_sorted(got)); + return got; + }; + + const std::vector baseline = run(/*include_bigrams=*/false, /*prune_min_df=*/0); + const std::vector unpruned = run(/*include_bigrams=*/true, /*prune_min_df=*/0); + const std::vector pruned = run(/*include_bigrams=*/true, /*prune_min_df=*/2); + + // "failed order ordinal" is adjacent in docs 5000/7000 (order@1 -> ordinal@2 + // there); the leading "failed order" must gate it -- bigram(order,ordinal) + // alone (present in docs 5000/7000) must not add a doc lacking "failed order". + EXPECT_EQ(unpruned, baseline) << "multi-leading phrase-prefix must not use the bigram pair"; + EXPECT_EQ(pruned, baseline) << "multi-leading phrase-prefix stays on the positions path"; +} diff --git a/be/test/storage/index/snii/query/phrase_skip_test.cpp b/be/test/storage/index/snii/query/phrase_skip_test.cpp new file mode 100644 index 00000000000000..b7b3299d31c3ab --- /dev/null +++ b/be/test/storage/index/snii/query/phrase_skip_test.cpp @@ -0,0 +1,579 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/io/metered_file_reader.h" +#include "storage/index/snii/query/boolean_query.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/reader/windowed_posting.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +// Differential test for phrase_query WINDOW SKIPPING (design spec section 6.2). +// +// Builds a corpus with a VERY high-df term ("aa_hi", df=2500) whose .frq +// posting spans MANY 256-doc windows, plus low/mid-df terms, and plants known +// phrases (including a 5-term phrase whose FIRST term is the high-df term). For +// every query the skipping phrase_query result must equal: +// (a) an in-memory brute-force ORACLE, and +// (b) an independent FULL-READ reference (decode every term's whole posting, +// intersect, positional check) -- the pre-skipping behavior. +// Finally it asserts the byte/round reduction: a phrase whose low-df lead term +// concentrates the candidates in a few windows reads FAR fewer bytes than the +// full-read path, and the high-df windows touched are NOT proportional to the +// term's total window count. +// +// NOTE on term naming: all real terms use an "aa_" prefix and a "zz_NNN" filler +// vocabulary fills the lexicographic tail, so every real term sorts within the +// SampledTermIndex's candidate range (the index samples per-block first terms). +using namespace doris::snii; +using namespace doris::snii::format; +using namespace doris::snii::reader; +using namespace doris::snii::writer; +using doris::Status; + +namespace { + +std::string TempPath() { + static int counter = 0; + return "/tmp/snii_phrase_skip_" + std::to_string(getpid()) + "_" + std::to_string(counter++) + + ".idx"; +} + +constexpr uint32_t kDocCount = 60000; +constexpr uint32_t kHiGap = 24; // aa_hi at 0, 24, 48 ... -> 2500 occurrences +constexpr uint32_t kHiCount = 2500; // df of aa_hi (>512 -> windowed, 10 windows) +// The 5-term phrase is planted only in the FIRST kPhraseSpan occurrences of +// aa_hi, concentrating its candidate docids into the first window(s). +constexpr uint32_t kPhraseSpan = 150; + +struct Corpus { + uint32_t doc_count = 0; + std::vector> docs; // docs[d] = ordered tokens + + bool phrase_in_doc(uint32_t d, const std::vector& phrase) const { + if (phrase.empty()) { + return false; + } + const auto& toks = docs[d]; + if (toks.size() < phrase.size()) { + return false; + } + for (size_t i = 0; i + phrase.size() <= toks.size(); ++i) { + bool match = true; + for (size_t k = 0; k < phrase.size(); ++k) { + if (toks[i + k] != phrase[k]) { + match = false; + break; + } + } + if (match) { + return true; + } + } + return false; + } + + std::vector oracle(const std::vector& phrase) const { + std::vector out; + for (uint32_t d = 0; d < doc_count; ++d) { + if (phrase_in_doc(d, phrase)) { + out.push_back(d); + } + } + return out; + } +}; + +Corpus BuildCorpus() { + Corpus c; + c.doc_count = kDocCount; + c.docs.resize(c.doc_count); + const std::vector vocab = {"alpha", "bravo", "charlie", "delta"}; + + for (uint32_t d = 0; d < c.doc_count; ++d) { + std::vector& toks = c.docs[d]; + const bool is_hi = (d % kHiGap == 0) && (d / kHiGap < kHiCount); + if (is_hi) { + toks.emplace_back("aa_hi"); // high-df, position 0 + } + const uint32_t occ = d / kHiGap; // aa_hi occurrence ordinal + + if (is_hi && occ < kPhraseSpan) { + // 5-term phrase led by the high-df term, concentrated in the first + // windows. + toks.emplace_back("aa_quick"); + toks.emplace_back("aa_brown"); + toks.emplace_back("aa_fox"); + toks.emplace_back("aa_jumps"); + } else if (d % 13 == 0) { + // A phrase NOT containing the high-df term, scattered across the corpus. + toks.emplace_back("aa_lazy"); + toks.emplace_back("aa_dog"); + } + for (uint32_t k = 0; k < 2; ++k) { + toks.push_back(vocab[(d + k) % vocab.size()]); + } + if (d % 101 == 0) { + toks.emplace_back("aa_rare"); + } + if (d % 997 == 0) { + toks.emplace_back("aa_echo"); + toks.emplace_back("aa_echo"); + } + // Larger filler vocabulary to occupy the lexicographic tail blocks. + char nm[16]; + std::snprintf(nm, sizeof(nm), "zz_%03u", d % 500); + toks.emplace_back(nm); + } + return c; +} + +Corpus BuildDensePhraseCorpus() { + Corpus c; + c.doc_count = 4096; + c.docs.resize(c.doc_count); + for (uint32_t d = 0; d < c.doc_count; ++d) { + std::vector& toks = c.docs[d]; + toks.emplace_back("aa_dense"); + if (d >= 16 && d < 96) { + toks.emplace_back("aa_rare_driver"); + } else { + toks.emplace_back("aa_gap"); + } + char nm[16]; + std::snprintf(nm, sizeof(nm), "zz_%03u", d % 257); + toks.emplace_back(nm); + } + return c; +} + +void WriteCorpus(const Corpus& c, const std::string& path) { + SpimiTermBuffer buf(/*has_positions=*/true); + for (uint32_t d = 0; d < c.doc_count; ++d) { + const auto& toks = c.docs[d]; + for (uint32_t pos = 0; pos < toks.size(); ++pos) { + buf.add_token(toks[pos], d, pos); + } + } + std::vector terms = buf.finalize_sorted(); + + SniiIndexInput in; + in.index_id = 1; + in.index_suffix = "body"; + in.config = IndexConfig::kDocsPositions; + in.doc_count = c.doc_count; + in.terms = std::move(terms); + in.target_dict_block_bytes = 256; + + io::LocalFileWriter w; + ASSERT_TRUE(w.open(path).ok()); + SniiCompoundWriter cw(&w); + ASSERT_TRUE(cw.add_logical_index(in).ok()); + ASSERT_TRUE(cw.finish().ok()); +} + +// Decodes one term's FULL posting (every window for windowed; the single window +// for slim/inline) -> sorted docids + aligned per-doc positions. This mirrors +// the pre-skipping reference path used to cross-check the skipping result. +struct FullPosting { + std::vector docids; + std::vector> positions; +}; + +bool DecodeFullTerm(const LogicalIndexReader& idx, const std::string& term, FullPosting* out) { + DictEntry entry; + uint64_t frq_base = 0, prx_base = 0; + bool found = false; + EXPECT_TRUE(idx.lookup(term, &found, &entry, &frq_base, &prx_base).ok()); + if (!found) { + return false; + } + + if (entry.kind == DictEntryKind::kPodRef && entry.enc == DictEntryEnc::kWindowed) { + DecodedPosting dp; + EXPECT_TRUE(read_windowed_posting(idx, entry, frq_base, prx_base, + /*want_positions=*/true, + /*want_freq=*/false, &dp) + .ok()); + out->docids = std::move(dp.docids); + out->positions = std::move(dp.positions); + return true; + } + std::vector frq, prx; + if (entry.kind == DictEntryKind::kPodRef) { + uint64_t foff = 0, flen = 0, poff = 0, plen = 0; + EXPECT_TRUE(idx.resolve_frq_window(entry, frq_base, &foff, &flen).ok()); + EXPECT_TRUE(idx.resolve_prx_window(entry, prx_base, &poff, &plen).ok()); + EXPECT_TRUE(idx.reader()->read_at(foff, flen, &frq).ok()); + EXPECT_TRUE(idx.reader()->read_at(poff, plen, &prx).ok()); + } else { + frq = entry.frq_bytes; + prx = entry.prx_bytes; + } + // Slim/inline window = [dd_region][freq_region]; the dd region is the docs + // prefix. + WindowMeta meta; + meta.win_base = 0; + meta.doc_count = entry.df; + meta.dd_zstd = entry.dd_meta.zstd; + meta.dd_uncomp_len = entry.dd_meta.uncomp_len; + meta.dd_disk_len = entry.dd_meta.disk_len; + meta.crc_dd = entry.dd_meta.crc; + // INLINE entries (format v2) carry no per-region crc -- their bytes are + // covered by the dict block crc32c -- so decode must skip the region crc + // check. + meta.verify_crc = entry.dd_meta.verify_crc; + EXPECT_LE(entry.dd_meta.disk_len, frq.size()); + Slice dd_region(frq.data(), static_cast(entry.dd_meta.disk_len)); + std::vector freqs; + Status st = decode_window_slices(meta, dd_region, Slice(), Slice(prx), + /*want_positions=*/true, /*want_freq=*/false, &out->docids, + &freqs, &out->positions); + EXPECT_TRUE(st.ok()) << st.msg(); + return true; +} + +// Independent FULL-READ phrase reference: intersect full postings, positional +// check. Mirrors the pre-skipping algorithm. +std::vector FullReadPhrase(const LogicalIndexReader& idx, + const std::vector& phrase) { + std::vector posts(phrase.size()); + for (size_t t = 0; t < phrase.size(); ++t) { + if (!DecodeFullTerm(idx, phrase[t], &posts[t])) { + return {}; + } + } + std::vector cand = posts[0].docids; + for (size_t t = 1; t < posts.size(); ++t) { + std::vector next; + std::set_intersection(cand.begin(), cand.end(), posts[t].docids.begin(), + posts[t].docids.end(), std::back_inserter(next)); + cand.swap(next); + } + auto positions_for = [](const FullPosting& p, uint32_t d) -> const std::vector& { + const auto it = std::ranges::lower_bound(p.docids, d); + return p.positions[static_cast(it - p.docids.begin())]; + }; + std::vector out; + for (uint32_t d : cand) { + const auto& first = positions_for(posts[0], d); + bool hit = false; + for (uint32_t start : first) { + bool ok = true; + for (size_t t = 1; t < posts.size(); ++t) { + const auto& ps = positions_for(posts[t], d); + if (!std::ranges::binary_search(ps, start + static_cast(t))) { + ok = false; + break; + } + } + if (ok) { + hit = true; + break; + } + } + if (hit) { + out.push_back(d); + } + } + return out; +} + +// Looks up the high-df term and reports its full .frq/.prx length + window +// count. +void HighDfStats(const LogicalIndexReader& idx, const std::string& term, uint64_t* frq_len, + uint64_t* prx_len, uint32_t* windows) { + DictEntry entry; + uint64_t frq_base = 0, prx_base = 0; + bool found = false; + ASSERT_TRUE(idx.lookup(term, &found, &entry, &frq_base, &prx_base).ok()); + ASSERT_TRUE(found); + ASSERT_EQ(entry.enc, DictEntryEnc::kWindowed); + FrqPreludeReader prelude; + ASSERT_TRUE(fetch_windowed_prelude(idx, entry, frq_base, &prelude).ok()); + *windows = prelude.n_windows(); + *frq_len = entry.frq_len; + *prx_len = entry.prx_len; +} + +} // namespace + +// boolean_and (MATCH all-terms) must equal the intersection of the per-term +// docid sets (term_query is the trusted oracle). Covers high+low df mix, +// all-present, an absent term (-> empty), and single-term (-> that term's +// docs). +TEST(SniiBooleanAnd, EqualsTermIntersection) { + Corpus c = BuildCorpus(); + const std::string path = TempPath(); + WriteCorpus(c, path); + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + io::MeteredFileReader metered(&local, /*block_size=*/4096); + SniiSegmentReader seg; + ASSERT_TRUE(SniiSegmentReader::open(&metered, &seg).ok()); + LogicalIndexReader idx; + ASSERT_TRUE(seg.open_index(1, "body", &idx).ok()); + + auto intersect_terms = [&](const std::vector& terms) { + std::vector acc; + bool first = true; + for (const auto& t : terms) { + std::vector d; + EXPECT_TRUE(query::term_query(idx, t, &d).ok()); + if (first) { + acc = d; + first = false; + } else { + std::vector out; + std::ranges::set_intersection(acc, d, std::back_inserter(out)); + acc = std::move(out); + } + } + return acc; + }; + + const std::vector> cases = { + {"aa_hi", "aa_quick"}, // high + low df + {"aa_quick", "aa_brown", "aa_fox"}, + {"aa_hi", "nope_absent"}, // absent term -> empty + {"aa_hi"}, // single term -> its docs + }; + for (const auto& terms : cases) { + std::string label; + for (const auto& t : terms) { + label += t + " "; + } + std::vector got; + ASSERT_TRUE(query::boolean_and(idx, terms, &got).ok()) << label; + EXPECT_TRUE(std::ranges::is_sorted(got)) << label; + EXPECT_EQ(got, intersect_terms(terms)) + << "boolean_and != term-intersection: [" << label << "]"; + } + std::remove(path.c_str()); +} + +// prefix_terms ordered enumeration: the full enumeration (empty prefix) must be +// strictly sorted; any prefix scan must equal the full enumeration filtered by +// that prefix; and every enumerated term must lookup() to the SAME DictEntry +// (df). +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiPrefixTerms, OrderedEnumerationMatchesFilterAndLookup) { + Corpus c = BuildCorpus(); + const std::string path = TempPath(); + WriteCorpus(c, path); + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + io::MeteredFileReader metered(&local, /*block_size=*/4096); + SniiSegmentReader seg; + ASSERT_TRUE(SniiSegmentReader::open(&metered, &seg).ok()); + LogicalIndexReader idx; + ASSERT_TRUE(seg.open_index(1, "body", &idx).ok()); + + std::vector all; + ASSERT_TRUE(idx.prefix_terms("", &all).ok()); + ASSERT_FALSE(all.empty()); + for (size_t i = 1; i < all.size(); ++i) { + EXPECT_LT(all[i - 1].term, all[i].term) << "at " << i; + } + for (const auto& h : all) { + bool found = false; + doris::snii::format::DictEntry e; + uint64_t fb = 0, pb = 0; + ASSERT_TRUE(idx.lookup(h.term, &found, &e, &fb, &pb).ok()); + EXPECT_TRUE(found) << h.term; + EXPECT_EQ(e.df, h.entry.df) << h.term; + } + auto filtered = [&](const std::string& pfx) { + std::vector v; + for (const auto& h : all) { + if (h.term.size() >= pfx.size() && h.term.starts_with(pfx)) { + v.push_back(h.term); + } + } + return v; + }; + auto scanned = [&](const std::string& pfx) { + std::vector hits; + EXPECT_TRUE(idx.prefix_terms(pfx, &hits).ok()); + std::vector v; + for (const auto& h : hits) { + v.push_back(h.term); + } + return v; + }; + for (const char* pfx : {"aa_", "aa_h", "zz_", "zz_0", "alpha"}) { + EXPECT_EQ(scanned(pfx), filtered(pfx)) << "prefix=" << pfx; + } + std::vector none; + ASSERT_TRUE(idx.prefix_terms("zzzznope", &none).ok()); + EXPECT_TRUE(none.empty()); + // Null output is rejected, not dereferenced. + EXPECT_FALSE(idx.prefix_terms("", nullptr).ok()); + std::remove(path.c_str()); +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiPhraseSkip, SkippingEqualsOracleAndFullRead) { + Corpus c = BuildCorpus(); + const std::string path = TempPath(); + WriteCorpus(c, path); + + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + // Small cache block so window-level skipping shows up in remote_bytes / GETs. + io::MeteredFileReader metered(&local, /*block_size=*/4096); + + SniiSegmentReader seg; + ASSERT_TRUE(SniiSegmentReader::open(&metered, &seg).ok()); + LogicalIndexReader idx; + ASSERT_TRUE(seg.open_index(1, "body", &idx).ok()); + ASSERT_EQ(idx.stats().doc_count, c.doc_count); + + uint64_t hi_frq_len = 0, hi_prx_len = 0; + uint32_t hi_windows = 0; + HighDfStats(idx, "aa_hi", &hi_frq_len, &hi_prx_len, &hi_windows); + ASSERT_GE(hi_windows, 8U) << "high-df term should span many windows"; + + // Phrases: present (incl. 5-term led by high-df term), absent, sub-phrases. + const std::vector> phrases = { + {"aa_hi", "aa_quick", "aa_brown", "aa_fox", "aa_jumps"}, // 5-term, hi lead + {"aa_hi", "aa_quick"}, // hi + mid + {"aa_quick", "aa_brown", "aa_fox", "aa_jumps"}, // no high-df term + {"aa_lazy", "aa_dog"}, // no high-df term + {"aa_brown", "aa_fox"}, // sub-phrase + {"aa_fox", "aa_brown"}, // reversed -> absent + {"aa_hi", "aa_lazy"}, // present terms, absent phrase + {"aa_quick", "aa_jumps"}, // non-consecutive -> absent + {"aa_echo", "aa_echo"}, // repeated term -> unique-term mapping + {"aa_hi"}, // single high-df term + {"nope", "missing"}, // absent terms -> empty + }; + + for (const auto& p : phrases) { + std::string label; + for (const auto& w : p) { + label += w + " "; + } + + std::vector got; + ASSERT_TRUE(query::phrase_query(idx, p, &got).ok()) << label; + EXPECT_TRUE(std::ranges::is_sorted(got)) << label; + + const std::vector want = c.oracle(p); + EXPECT_EQ(got, want) << "oracle mismatch: [" << label << "]"; + + const std::vector full = FullReadPhrase(idx, p); + EXPECT_EQ(got, full) << "full-read mismatch: [" << label << "]"; + } + + // ---- Byte/round reduction for a phrase CONTAINING the high-df term. ---- + // The low-df lead "aa_quick" concentrates candidates into the first window(s) + // of "aa_hi", so the skip path reads only those windows + the prelude, NOT + // the whole posting. Compare against a full-read of the same phrase. + const std::vector hi_phrase = {"aa_hi", "aa_quick", "aa_brown", "aa_fox", + "aa_jumps"}; + metered.reset_metrics(); + std::vector got; + ASSERT_TRUE(query::phrase_query(idx, hi_phrase, &got).ok()); + const io::IoMetrics skip = metered.metrics(); + + metered.reset_metrics(); + (void)FullReadPhrase(idx, hi_phrase); + const io::IoMetrics full = metered.metrics(); + + // 1. Skipping requests SUBSTANTIALLY fewer bytes than the full-read path: the + // high-df term contributes only its prelude + the candidate-covering + // windows (dd + prx sub-ranges), never its whole posting, so the 5-term + // skip query reads comfortably under 80% of the full-read bytes (a clear + // reduction even with the richer Phase-D prelude and 4 companion terms). + EXPECT_LT(skip.total_request_bytes * 10, full.total_request_bytes * 8) + << "skip=" << skip.total_request_bytes << " full=" << full.total_request_bytes; + // 2. Skipping requests strictly fewer bytes than the full-read path. + EXPECT_LT(skip.total_request_bytes, full.total_request_bytes) + << "skip=" << skip.total_request_bytes << " full=" << full.total_request_bytes; + // 3. Range GETs stay small and are NOT proportional to the term's window + // count + // (the high-df term contributes only its prelude + the covering windows). + EXPECT_LT(skip.range_gets, hi_windows) + << "range_gets=" << skip.range_gets << " windows=" << hi_windows; + // 4. Skipping issues no more range GETs than the full-read path, and stays in + // a small number of serial rounds (preludes + the selected-window batch). + EXPECT_LE(skip.range_gets, full.range_gets) + << "skip_gets=" << skip.range_gets << " full_gets=" << full.range_gets; + EXPECT_GE(skip.serial_rounds, 1U); + EXPECT_LE(skip.serial_rounds, 4U) << "skip did not stay in few serial rounds"; + + std::remove(path.c_str()); +} + +TEST(SniiPhraseSkip, DenseLeadingTermWithRareAnchorSkipsDocsButKeepsPositions) { + Corpus c = BuildDensePhraseCorpus(); + const std::string path = TempPath(); + WriteCorpus(c, path); + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + io::MeteredFileReader metered(&local, /*block_size=*/4096); + SniiSegmentReader seg; + ASSERT_TRUE(SniiSegmentReader::open(&metered, &seg).ok()); + LogicalIndexReader idx; + ASSERT_TRUE(seg.open_index(1, "body", &idx).ok()); + + uint64_t dense_frq = 0; + uint64_t dense_prx = 0; + uint32_t dense_windows = 0; + HighDfStats(idx, "aa_dense", &dense_frq, &dense_prx, &dense_windows); + ASSERT_GE(dense_windows, 8U); + + const std::vector phrase = {"aa_dense", "aa_rare_driver"}; + metered.reset_metrics(); + std::vector got; + ASSERT_TRUE(query::phrase_query(idx, phrase, &got).ok()); + const uint64_t skipped_bytes = metered.metrics().total_request_bytes; + + metered.reset_metrics(); + const std::vector full = FullReadPhrase(idx, phrase); + const uint64_t full_bytes = metered.metrics().total_request_bytes; + + EXPECT_EQ(got, c.oracle(phrase)); + EXPECT_EQ(got, full); + EXPECT_LT(skipped_bytes, full_bytes) + << "dense leading phrase should keep the rare-position anchor but avoid " + "reading dense docid regions"; + + std::remove(path.c_str()); +} diff --git a/be/test/storage/index/snii/query/position_math_test.cpp b/be/test/storage/index/snii/query/position_math_test.cpp new file mode 100644 index 00000000000000..365adfb3f8a24c --- /dev/null +++ b/be/test/storage/index/snii/query/position_math_test.cpp @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/internal/position_math.h" + +#include + +#include +#include +#include + +#include "common/status.h" + +using doris::Status; // RETURN_IF_ERROR expands to bare Status // NOLINT(misc-unused-using-decls) + +TEST(SniiPositionMath, AddsOffsetWhenRepresentable) { + uint32_t out = 0; + EXPECT_TRUE(doris::snii::query::internal::add_position_offset(41, 1, &out)); + EXPECT_EQ(out, 42U); + EXPECT_TRUE(doris::snii::query::internal::add_position_offset( + std::numeric_limits::max() - 2, 2, &out)); + EXPECT_EQ(out, std::numeric_limits::max()); +} + +TEST(SniiPositionMath, RejectsWraparound) { + uint32_t out = 7; + EXPECT_FALSE(doris::snii::query::internal::add_position_offset( + std::numeric_limits::max(), 1, &out)); + EXPECT_EQ(out, 7U); +} + +TEST(SniiPositionMath, BuildsDenseOffsets) { + std::vector offsets; + EXPECT_TRUE(doris::snii::query::internal::build_position_offsets(4, &offsets)); + EXPECT_EQ(offsets, (std::vector {0U, 1U, 2U, 3U})); +} + +TEST(SniiPositionMath, RejectsUnrepresentableOffsetCount) { + std::vector offsets = {9}; + EXPECT_FALSE(doris::snii::query::internal::build_position_offsets( + std::numeric_limits::max(), &offsets)); + EXPECT_EQ(offsets, (std::vector {9U})); +} diff --git a/be/test/storage/index/snii/query/posting_grouping_test.cpp b/be/test/storage/index/snii/query/posting_grouping_test.cpp new file mode 100644 index 00000000000000..563d0c01105bef --- /dev/null +++ b/be/test/storage/index/snii/query/posting_grouping_test.cpp @@ -0,0 +1,508 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/io/batch_range_fetcher.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/io/metered_file_reader.h" +#include "storage/index/snii/query/bm25_scorer.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/scoring_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/reader/windowed_posting.h" +#include "storage/index/snii/stats/snii_stats_provider.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +// PHASE D differential + contiguity test (design 1.6: posting-level dd/freq +// grouping). The windowed .frq payload is laid out +// [prelude][dd-block][freq-block] so a docid-only / phrase reader fetches the +// docs-only data ([prelude][dd-block]) as ONE CONTIGUOUS run. Over a ~5000-doc +// kDocsPositionsScoring index with a high-df term spanning MANY adaptive +// windows plus mid/low terms and a planted 5-term phrase, this asserts: +// (a) term_query + phrase_query (incl the 5-term phrase) docids == a +// brute-force +// ORACLE; scoring top-K (exhaustive == wand == selective) unchanged. +// (b) Through a MeteredFileReader the docid-only posting reader for the +// high-df +// term reads the dd-block as a CONTIGUOUS region: read_at is SMALL (a +// couple of ranges, NOT one-per-window / not thousands), range_gets is +// small, AND request_bytes is strictly LESS than fetching the full +// posting (the freq-block is skipped). All three (read_at, range_gets, +// request_bytes) drop vs the full-posting read, with identical docids. +using namespace doris::snii; +using namespace doris::snii::format; +using namespace doris::snii::reader; +using namespace doris::snii::writer; +using doris::snii::query::Bm25Params; +using doris::snii::query::ScoredDoc; +using doris::snii::stats::SniiStatsProvider; + +namespace { + +std::string TempPath() { + static int counter = 0; + return "/tmp/snii_posting_grouping_" + std::to_string(getpid()) + "_" + + std::to_string(counter++) + ".idx"; +} + +constexpr uint32_t kDocCount = 5000; +constexpr uint32_t kHiCount = 4800; // df of aa_hi (>> 256 -> many windows) +constexpr uint32_t kPhraseSpan = 150; // 5-term phrase planted in early aa_hi docs + +// aa_hi appears with varied, large frequency so its per-window freq region is +// big in absolute terms (the freq-block the docs-only path never fetches). +uint32_t HiFreq(uint32_t occ) { + return 50U + (occ * 17U + 11U) % 240U; +} + +struct Corpus { + uint32_t doc_count = 0; + std::vector> docs; + std::vector doc_len; + + bool phrase_in_doc(uint32_t d, const std::vector& phrase) const { + if (phrase.empty()) { + return false; + } + const auto& toks = docs[d]; + if (toks.size() < phrase.size()) { + return false; + } + for (size_t i = 0; i + phrase.size() <= toks.size(); ++i) { + bool match = true; + for (size_t k = 0; k < phrase.size(); ++k) { + if (toks[i + k] != phrase[k]) { + match = false; + break; + } + } + if (match) { + return true; + } + } + return false; + } + + std::vector term_oracle(const std::string& term) const { + std::vector out; + for (uint32_t d = 0; d < doc_count; ++d) { + const auto& toks = docs[d]; + if (std::ranges::find(toks, term) != toks.end()) { + out.push_back(d); + } + } + return out; + } + + std::vector phrase_oracle(const std::vector& phrase) const { + std::vector out; + for (uint32_t d = 0; d < doc_count; ++d) { + if (phrase_in_doc(d, phrase)) { + out.push_back(d); + } + } + return out; + } +}; + +Corpus BuildCorpus() { + Corpus c; + c.doc_count = kDocCount; + c.docs.resize(c.doc_count); + c.doc_len.assign(c.doc_count, 0); + const std::vector vocab = {"alpha", "bravo", "charlie", "delta"}; + + for (uint32_t d = 0; d < c.doc_count; ++d) { + std::vector& toks = c.docs[d]; + const bool is_hi = d < kHiCount; + if (is_hi) { + toks.emplace_back("aa_hi"); // high-df, position 0 + } + const uint32_t occ = d; + + if (is_hi && occ < kPhraseSpan) { + toks.emplace_back("aa_quick"); + toks.emplace_back("aa_brown"); + toks.emplace_back("aa_fox"); + toks.emplace_back("aa_jumps"); + } else if (d % 13 == 0) { + toks.emplace_back("aa_lazy"); + toks.emplace_back("aa_dog"); + } + if (d % 5 == 0) { + toks.emplace_back("aa_mid"); + if (d % 15 == 0) { + toks.emplace_back("aa_mid"); + } + } + for (uint32_t k = 0; k < 2; ++k) { + toks.push_back(vocab[(d + k) % vocab.size()]); + } + if (d % 101 == 0) { + toks.emplace_back("aa_rare"); + } + char nm[16]; + std::snprintf(nm, sizeof(nm), "zz_%03u", d % 400); + toks.emplace_back(nm); + if (is_hi) { + const uint32_t extra = HiFreq(occ) - 1U; // total freq = HiFreq(occ) + for (uint32_t k = 0; k < extra; ++k) { + toks.emplace_back("aa_hi"); + } + } + c.doc_len[d] = toks.size(); + } + return c; +} + +void WriteCorpus(const Corpus& c, const std::string& path) { + SpimiTermBuffer buf(/*has_positions=*/true); + for (uint32_t d = 0; d < c.doc_count; ++d) { + const auto& toks = c.docs[d]; + for (uint32_t pos = 0; pos < toks.size(); ++pos) { + buf.add_token(toks[pos], d, pos); + } + } + std::vector terms = buf.finalize_sorted(); + + SniiIndexInput in; + in.index_id = 1; + in.index_suffix = "body"; + in.config = IndexConfig::kDocsPositionsScoring; + in.doc_count = c.doc_count; + in.terms = std::move(terms); + in.target_dict_block_bytes = 256; + in.encoded_norms.resize(c.doc_count); + for (uint32_t d = 0; d < c.doc_count; ++d) { + in.encoded_norms[d] = doris::snii::query::encode_norm(c.doc_len[d]); + } + + io::LocalFileWriter w; + ASSERT_TRUE(w.open(path).ok()); + SniiCompoundWriter cw(&w); + ASSERT_TRUE(cw.add_logical_index(in).ok()); + ASSERT_TRUE(cw.finish().ok()); +} + +// dd-block / freq-block byte sizes of a windowed term (the contiguous runs). +struct Blocks { + uint64_t prelude_len = 0; + uint64_t dd_block = 0; + uint64_t freq_block = 0; + uint32_t windows = 0; +}; + +Blocks MeasureBlocks(const LogicalIndexReader& idx, const std::string& term) { + Blocks b; + DictEntry entry; + uint64_t frq_base = 0, prx_base = 0; + bool found = false; + EXPECT_TRUE(idx.lookup(term, &found, &entry, &frq_base, &prx_base).ok()); + EXPECT_TRUE(found); + EXPECT_EQ(entry.enc, DictEntryEnc::kWindowed); + b.prelude_len = entry.prelude_len; + FrqPreludeReader prelude; + EXPECT_TRUE(fetch_windowed_prelude(idx, entry, frq_base, &prelude).ok()); + b.windows = prelude.n_windows(); + b.dd_block = prelude.dd_block_len(); + b.freq_block = prelude.freq_block_len(); + return b; +} + +std::vector ReferenceRanking(const Corpus& c, const std::vector& terms, + uint32_t k, const Bm25Params& params) { + uint64_t sum_ttf = 0; + for (uint64_t dl : c.doc_len) { + sum_ttf += dl; + } + const double avgdl = static_cast(sum_ttf) / std::max(1, c.doc_count); + + std::unordered_map scores; + for (const auto& term : terms) { + std::map plist; + for (uint32_t d = 0; d < c.doc_count; ++d) { + uint32_t f = 0; + for (const auto& t : c.docs[d]) { + if (t == term) { + ++f; + } + } + if (f > 0) { + plist[d] = f; + } + } + if (plist.empty()) { + continue; + } + const uint64_t df = plist.size(); + const double idf = + std::log(1.0 + (static_cast(c.doc_count) - df + 0.5) / (df + 0.5)); + for (const auto& [docid, freq] : plist) { + const double dl = doris::snii::query::decode_norm( + doris::snii::query::encode_norm(c.doc_len[docid])); + const double denom = freq + params.k1 * (1.0 - params.b + params.b * dl / avgdl); + scores[docid] += idf * (freq * (params.k1 + 1.0)) / denom; + } + } + + std::vector all; + all.reserve(scores.size()); + for (const auto& [docid, s] : scores) { + all.push_back({docid, s}); + } + std::ranges::sort(all, [](const ScoredDoc& a, const ScoredDoc& b) { + if (a.score != b.score) { + return a.score > b.score; + } + return a.docid < b.docid; + }); + if (all.size() > k) { + all.resize(k); + } + return all; +} + +void ExpectRankingEqual(const std::vector& got, const std::vector& ref, + const char* label) { + ASSERT_EQ(got.size(), ref.size()) << label; + for (size_t i = 0; i < ref.size(); ++i) { + EXPECT_EQ(got[i].docid, ref[i].docid) << label << " docid i=" << i; + EXPECT_NEAR(got[i].score, ref[i].score, 1e-9) << label << " score i=" << i; + } +} + +// PRE-GROUPING SIMULATION: decode a windowed term's docids by fetching EACH +// window's dd region in its OWN read round (one BatchRangeFetcher per window), +// as the prior per-window [dd][freq] layout forced -- each window's docs prefix +// had to be fetched on its own because the next window's dd was separated by +// the current window's (skipped) freq region. This reproduces the read_at / +// range_get explosion (one per window) that the grouped, contiguous dd-block +// eliminates. Same docids. +std::vector DecodePerWindow(const LogicalIndexReader& idx, const DictEntry& entry, + uint64_t frq_base, uint64_t prx_base) { + FrqPreludeReader prelude; + EXPECT_TRUE(fetch_windowed_prelude(idx, entry, frq_base, &prelude).ok()); + std::vector docids; + for (uint32_t w = 0; w < prelude.n_windows(); ++w) { + WindowAbsRange r; + EXPECT_TRUE(windowed_window_range(idx, entry, frq_base, prx_base, prelude, w, + /*want_positions=*/false, /*want_freq=*/false, &r) + .ok()); + WindowMeta m; + EXPECT_TRUE(prelude.window(w, &m).ok()); + // One read round per window (the un-grouped reader could not coalesce + // across the interleaved freq regions of the old layout). + doris::snii::io::BatchRangeFetcher fetcher(idx.reader(), /*coalesce_gap=*/0); + const size_t h = fetcher.add(r.dd_off, r.dd_len); + EXPECT_TRUE(fetcher.fetch().ok()); + std::vector wd, wf; + std::vector> wp; + EXPECT_TRUE(decode_window_slices(m, fetcher.get(h), Slice(), Slice(), + /*want_positions=*/false, + /*want_freq=*/false, &wd, &wf, &wp) + .ok()); + docids.insert(docids.end(), wd.begin(), wd.end()); + } + return docids; +} + +} // namespace + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiPostingGrouping, ContiguousDdBlockSavesAllThreeMetrics) { + Corpus c = BuildCorpus(); + const std::string path = TempPath(); + WriteCorpus(c, path); + + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + // Fine-grained FileCache block (< a window's freq region) so the grouped + // docs-only run occupies strictly fewer cache blocks than (i) the full + // posting and (ii) a per-window dd fetch whose gaps span the skipped freq + // regions. + io::MeteredFileReader metered(&local, /*block_size=*/64); + + SniiSegmentReader seg; + ASSERT_TRUE(SniiSegmentReader::open(&metered, &seg).ok()); + LogicalIndexReader idx; + ASSERT_TRUE(seg.open_index(1, "body", &idx).ok()); + ASSERT_EQ(idx.stats().doc_count, c.doc_count); + + // The high-df term must span many windows (the contiguity win is meaningful). + const Blocks hi = MeasureBlocks(idx, "aa_hi"); + ASSERT_GE(hi.windows, 8U) << "high-df term should span many windows"; + ASSERT_GT(hi.freq_block, 0U) << "freq-block must carry real bytes"; + + // ---- (a) term_query / phrase_query docids == ORACLE ----------------------- + const std::vector terms_to_check = {"aa_hi", "aa_mid", "aa_quick", "aa_lazy", + "aa_rare"}; + for (const auto& term : terms_to_check) { + std::vector got; + ASSERT_TRUE(query::term_query(idx, term, &got).ok()) << term; + EXPECT_TRUE(std::ranges::is_sorted(got)) << term; + EXPECT_EQ(got, c.term_oracle(term)) << "term_query oracle mismatch: " << term; + } + + const std::vector> phrases = { + {"aa_hi", "aa_quick", "aa_brown", "aa_fox", "aa_jumps"}, // 5-term, hi lead + {"aa_hi", "aa_quick"}, // hi + mid + {"aa_quick", "aa_brown", "aa_fox", "aa_jumps"}, // no high-df term + {"aa_lazy", "aa_dog"}, // scattered phrase + {"aa_brown", "aa_fox"}, // sub-phrase + {"aa_fox", "aa_brown"}, // reversed -> absent + {"aa_hi", "aa_lazy"}, // present terms, absent phrase + }; + for (const auto& p : phrases) { + std::string label; + for (const auto& w : p) { + label += w + " "; + } + std::vector got; + ASSERT_TRUE(query::phrase_query(idx, p, &got).ok()) << label; + EXPECT_TRUE(std::ranges::is_sorted(got)) << label; + EXPECT_EQ(got, c.phrase_oracle(p)) << "phrase oracle mismatch: [" << label << "]"; + } + + // ---- scoring: exhaustive == wand == selective == reference ---------------- + SniiStatsProvider stats; + ASSERT_TRUE(SniiStatsProvider::open(&idx, &stats).ok()); + const Bm25Params params; + const std::vector score_terms = {"aa_hi", "aa_mid", "aa_rare"}; + for (uint32_t k : {1U, 10U, 100U}) { + std::vector ex, wa, sel; + ASSERT_TRUE(query::scoring_query_exhaustive(idx, stats, score_terms, k, params, &ex).ok()); + ASSERT_TRUE(query::scoring_query_wand(idx, stats, score_terms, k, params, &wa).ok()); + ASSERT_TRUE( + query::scoring_query_wand_selective(idx, stats, score_terms, k, params, &sel).ok()); + const std::vector ref = ReferenceRanking(c, score_terms, k, params); + ExpectRankingEqual(ex, ref, "exhaustive"); + ExpectRankingEqual(wa, ex, "wand"); + ExpectRankingEqual(sel, ex, "selective"); + } + + DictEntry hi_entry; + uint64_t hi_frq_base = 0, hi_prx_base = 0; + bool hi_found = false; + ASSERT_TRUE(idx.lookup("aa_hi", &hi_found, &hi_entry, &hi_frq_base, &hi_prx_base).ok()); + ASSERT_TRUE(hi_found); + ASSERT_EQ(hi_entry.frq_docs_len, hi.prelude_len + hi.dd_block); + ASSERT_LT(hi_entry.frq_docs_len, hi_entry.frq_len); + + DictEntry oversized_docs_prefix = hi_entry; + ++oversized_docs_prefix.frq_docs_len; + std::vector corrupt_docs; + // Integrated SNII reports posting corruption with the inverted-index-file + // corruption code (the "docs prefix length mismatch" guard), not the generic + // CORRUPTION code the standalone test used. + EXPECT_TRUE(query::internal::read_docid_posting(idx, oversized_docs_prefix, hi_frq_base, + hi_prx_base, &corrupt_docs) + .is()); + + // ---- (b) docid-only posting read fetches the dd-block CONTIGUOUSLY -------- + // GROUPED = the Phase-D grouped docid-only path: read [prelude][dd-block] as + // one contiguous prefix; the freq-block is skipped on the wire. The lookup is + // deliberately outside this metrics window so the assertion covers posting + // I/O. + metered.reset_metrics(); + std::vector grouped_docs; + ASSERT_TRUE(query::internal::read_docid_posting(idx, hi_entry, hi_frq_base, hi_prx_base, + &grouped_docs) + .ok()); + const io::IoMetrics grouped = metered.metrics(); + ASSERT_EQ(grouped_docs, c.term_oracle("aa_hi")); + EXPECT_EQ(grouped.serial_rounds, 1U) + << "docid-only windowed term_query should fetch [prelude][dd-block] in " + "one " + "batched round"; + EXPECT_EQ(grouped.range_gets, 1U) << "docid-only windowed term_query should " + "issue one contiguous prefix range"; + + // PER-WINDOW = the pre-grouping layout: fetch EACH window's dd region as its + // own physical range (gaps = the skipped freq regions). Same resolved term. + metered.reset_metrics(); + const std::vector perwin_docs = + DecodePerWindow(idx, hi_entry, hi_frq_base, hi_prx_base); + const io::IoMetrics perwin = metered.metrics(); + ASSERT_EQ(perwin_docs, c.term_oracle("aa_hi")); + + // FULL = fetch the whole posting (prelude + dd-block + freq-block). Same + // resolved term. + metered.reset_metrics(); + { + DecodedPosting full_posting; + ASSERT_TRUE(read_windowed_posting(idx, hi_entry, hi_frq_base, hi_prx_base, + /*want_positions=*/false, + /*want_freq=*/true, &full_posting) + .ok()); + EXPECT_EQ(full_posting.docids, c.term_oracle("aa_hi")); + EXPECT_EQ(full_posting.freqs.size(), full_posting.docids.size()); + } + const io::IoMetrics full = metered.metrics(); + + // The grouped docid-only path issues only a HANDFUL of logical reads (lookup + // + prelude + ONE dd-block range), NOT one-per-window and not thousands. + EXPECT_LE(grouped.read_at_calls, 4U) + << "grouped docid-only read_at not contiguous-small: " << grouped.read_at_calls; + EXPECT_LT(grouped.read_at_calls, hi.windows) + << "grouped must not read per-window: read_at=" << grouped.read_at_calls + << " windows=" << hi.windows; + + // (1) vs PER-WINDOW: all three drop because the dd-block is ONE contiguous + // range instead of one fetch per window separated by the skipped freq + // regions. + EXPECT_LT(grouped.read_at_calls, perwin.read_at_calls) + << "read_at vs per-window did not drop: grouped=" << grouped.read_at_calls + << " perwin=" << perwin.read_at_calls; + EXPECT_LT(grouped.range_gets, perwin.range_gets) + << "range_gets vs per-window did not drop: grouped=" << grouped.range_gets + << " perwin=" << perwin.range_gets; + EXPECT_LE(grouped.total_request_bytes, perwin.total_request_bytes) + << "request_bytes vs per-window grew: grouped=" << grouped.total_request_bytes + << " perwin=" << perwin.total_request_bytes; + + // (2) vs FULL posting: request_bytes and remote_bytes strictly drop + // (freq-block skipped on the wire and its cache blocks never touched). + EXPECT_LT(grouped.total_request_bytes, full.total_request_bytes) + << "request_bytes vs full did not drop: grouped=" << grouped.total_request_bytes + << " full=" << full.total_request_bytes; + EXPECT_EQ(full.total_request_bytes - grouped.total_request_bytes, hi.freq_block) + << "skipped bytes != freq-block size"; + EXPECT_LT(grouped.remote_bytes, full.remote_bytes) + << "remote_bytes vs full did not drop: grouped=" << grouped.remote_bytes + << " full=" << full.remote_bytes; + + std::remove(path.c_str()); +} diff --git a/be/test/storage/index/snii/query/prefix_query_test.cpp b/be/test/storage/index/snii/query/prefix_query_test.cpp new file mode 100644 index 00000000000000..3651c264cc7dfa --- /dev/null +++ b/be/test/storage/index/snii/query/prefix_query_test.cpp @@ -0,0 +1,185 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/prefix_query.h" + +#include +#include + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/io/metered_file_reader.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +using namespace doris::snii; +using namespace doris::snii::reader; +using namespace doris::snii::writer; + +namespace { + +std::string TempPath() { + static int counter = 0; + return "/tmp/snii_prefix_query_" + std::to_string(getpid()) + "_" + std::to_string(counter++) + + ".idx"; +} + +struct Corpus { + uint32_t doc_count = 0; + std::vector> docs; + + std::vector prefix_docs(const std::string& prefix) const { + std::set ids; + for (uint32_t d = 0; d < docs.size(); ++d) { + for (const std::string& term : docs[d]) { + if (term.size() >= prefix.size() && term.starts_with(prefix)) { + ids.insert(d); + } + } + } + return {ids.begin(), ids.end()}; + } +}; + +Corpus BuildMixedCorpus() { + Corpus c; + c.doc_count = 120; + c.docs.resize(c.doc_count); + for (uint32_t d = 0; d < c.doc_count; ++d) { + std::vector& terms = c.docs[d]; + if (d < 80) { + char term[16]; + std::snprintf(term, sizeof(term), "aa_%03u", d); + terms.emplace_back(term); + } + if (d < 80 && d % 2 == 0) { + terms.emplace_back("aa_even"); + } + if (d < 50) { + char term[16]; + std::snprintf(term, sizeof(term), "ab_%03u", d); + terms.emplace_back(term); + } + char filler[16]; + std::snprintf(filler, sizeof(filler), "zz_%03u", d); + terms.emplace_back(filler); + } + return c; +} + +Corpus BuildLowDfPrefixCorpus() { + Corpus c; + c.doc_count = 96; + c.docs.resize(c.doc_count); + for (uint32_t d = 0; d < c.doc_count; ++d) { + char term[16]; + std::snprintf(term, sizeof(term), "aa_%03u", d); + c.docs[d].emplace_back(term); + } + return c; +} + +void WriteCorpus(const Corpus& c, const std::string& path) { + SpimiTermBuffer buf(/*has_positions=*/false); + for (uint32_t d = 0; d < c.docs.size(); ++d) { + const std::vector& terms = c.docs[d]; + for (uint32_t pos = 0; pos < terms.size(); ++pos) { + buf.add_token(terms[pos], d, pos); + } + } + + SniiIndexInput in; + in.index_id = 1; + in.index_suffix = "body"; + in.config = doris::snii::format::IndexConfig::kDocsOnly; + in.doc_count = c.doc_count; + in.terms = buf.finalize_sorted(); + in.target_dict_block_bytes = 2048; + + io::LocalFileWriter writer; + ASSERT_TRUE(writer.open(path).ok()); + SniiCompoundWriter compound(&writer); + ASSERT_TRUE(compound.add_logical_index(in).ok()); + ASSERT_TRUE(compound.finish().ok()); +} + +LogicalIndexReader OpenIndex(io::LocalFileReader* file, SniiSegmentReader* segment, + const std::string& path) { + EXPECT_TRUE(file->open(path).ok()); + EXPECT_TRUE(SniiSegmentReader::open(file, segment).ok()); + LogicalIndexReader idx; + EXPECT_TRUE(segment->open_index(1, "body", &idx).ok()); + return idx; +} + +LogicalIndexReader OpenMeteredIndex(io::MeteredFileReader* file, SniiSegmentReader* segment) { + EXPECT_TRUE(SniiSegmentReader::open(file, segment).ok()); + LogicalIndexReader idx; + EXPECT_TRUE(segment->open_index(1, "body", &idx).ok()); + return idx; +} + +} // namespace + +TEST(SniiPrefixQuery, MatchesPrefixOracle) { + const Corpus corpus = BuildMixedCorpus(); + const std::string path = TempPath(); + WriteCorpus(corpus, path); + + io::LocalFileReader file; + SniiSegmentReader segment; + LogicalIndexReader idx = OpenIndex(&file, &segment, path); + + for (const char* prefix : {"aa_", "aa_e", "ab_0", "zz_", "missing"}) { + std::vector got; + ASSERT_TRUE(query::prefix_query(idx, prefix, &got).ok()) << prefix; + EXPECT_TRUE(std::ranges::is_sorted(got)) << prefix; + EXPECT_EQ(got, corpus.prefix_docs(prefix)) << prefix; + } + + std::remove(path.c_str()); +} + +TEST(SniiPrefixQuery, UsesEnumeratedEntriesWithoutPerTermLookup) { + const Corpus corpus = BuildLowDfPrefixCorpus(); + const std::string path = TempPath(); + WriteCorpus(corpus, path); + + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + io::MeteredFileReader metered(&local, /*block_size=*/4096); + SniiSegmentReader segment; + LogicalIndexReader idx = OpenMeteredIndex(&metered, &segment); + + metered.reset_metrics(); + std::vector got; + ASSERT_TRUE(query::prefix_query(idx, "aa_", &got).ok()); + + EXPECT_EQ(got, corpus.prefix_docs("aa_")); + EXPECT_LT(metered.metrics().read_at_calls, corpus.doc_count / 3) + << "prefix_query must reuse PrefixHit entries, not lookup every term again"; + + std::remove(path.c_str()); +} diff --git a/be/test/storage/index/snii/query/query_operator_error_test.cpp b/be/test/storage/index/snii/query/query_operator_error_test.cpp new file mode 100644 index 00000000000000..e27031114cc705 --- /dev/null +++ b/be/test/storage/index/snii/query/query_operator_error_test.cpp @@ -0,0 +1,268 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/io/file_reader.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/query/boolean_query.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/regexp_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/query/wildcard_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +using namespace doris::snii; +using namespace doris::snii::reader; +using namespace doris::snii::writer; +using doris::Status; + +namespace { + +std::string TempPath() { + static int counter = 0; + return "/tmp/snii_query_operator_error_" + std::to_string(getpid()) + "_" + + std::to_string(counter++) + ".idx"; +} + +// In-memory FileReader (backed by a flat byte buffer) whose reads can be made +// to fail on demand, so queries can be driven over the I/O-error propagation +// paths after the index has been opened successfully. +class FaultInjectingReader : public io::FileReader { +public: + explicit FaultInjectingReader(std::vector bytes) : bytes_(std::move(bytes)) {} + + void set_fail_reads(bool v) { fail_reads_ = v; } + + Status read_at(uint64_t offset, size_t len, std::vector* out) override { + if (fail_reads_) { + return Status::IOError("injected read failure"); + } + if (out == nullptr) { + return Status::InvalidArgument("memory reader null out"); + } + if (offset > bytes_.size() || len > bytes_.size() - offset) { + return Status::Corruption("memory reader read past end"); + } + out->assign(bytes_.begin() + static_cast(offset), + bytes_.begin() + static_cast(offset + len)); + return Status::OK(); + } + + uint64_t size() const override { return bytes_.size(); } + +private: + std::vector bytes_; + bool fail_reads_ = false; +}; + +struct Corpus { + std::vector> docs = { + {"alpha", "beta", "gamma"}, {"alpha", "beta", "delta"}, {"alpha", "bravo", "gamma"}, + {"beta", "gamma"}, {"alphabeta", "beta"}, + }; +}; + +struct EnvGuard { + std::string name; + bool had = false; + std::string old; + + explicit EnvGuard(const char* env_name) : name(env_name) { + const char* value = std::getenv(env_name); + had = value != nullptr; + if (had) { + old = value; + } + } + + ~EnvGuard() { + if (had) { + ::setenv(name.c_str(), old.c_str(), 1); + } else { + ::unsetenv(name.c_str()); + } + } +}; + +void BuildIndexBytes(const Corpus& corpus, doris::snii::format::IndexConfig config, + std::vector* bytes) { + SpimiTermBuffer buf(/*has_positions=*/config != doris::snii::format::IndexConfig::kDocsOnly); + for (uint32_t d = 0; d < corpus.docs.size(); ++d) { + const std::vector& terms = corpus.docs[d]; + for (uint32_t pos = 0; pos < terms.size(); ++pos) { + buf.add_token(terms[pos], d, pos); + } + } + + SniiIndexInput in; + in.index_id = 1; + in.index_suffix = "body"; + in.config = config; + in.doc_count = static_cast(corpus.docs.size()); + if (config == doris::snii::format::IndexConfig::kDocsPositionsScoring) { + in.encoded_norms.assign(corpus.docs.size(), 1); + } + in.terms = buf.finalize_sorted(); + in.target_dict_block_bytes = 512; + + const std::string path = TempPath(); + { + io::LocalFileWriter writer; + ASSERT_TRUE(writer.open(path).ok()); + SniiCompoundWriter compound(&writer); + ASSERT_TRUE(compound.add_logical_index(in).ok()); + ASSERT_TRUE(compound.finish().ok()); + } + { + io::LocalFileReader file; + ASSERT_TRUE(file.open(path).ok()); + ASSERT_TRUE(file.read_at(0, file.size(), bytes).ok()); + } + std::remove(path.c_str()); +} + +LogicalIndexReader OpenIndex(FaultInjectingReader* file, SniiSegmentReader* segment) { + EXPECT_TRUE(SniiSegmentReader::open(file, segment).ok()); + LogicalIndexReader idx; + EXPECT_TRUE(segment->open_index(1, "body", &idx).ok()); + return idx; +} + +void ExpectIoError(const Status& status) { + EXPECT_TRUE(status.is()) << status.to_string(); +} + +} // namespace + +TEST(SniiQueryOperatorBoundaries, RejectNullOutputPointers) { + LogicalIndexReader idx; + // A typed null output pointer selects each operator's std::vector* + // overload unambiguously (the integrated API also exposes DocIdSink* + // overloads, so a bare nullptr would be ambiguous). + auto* null_docs = static_cast*>(nullptr); + EXPECT_TRUE( + query::term_query(idx, "alpha", null_docs).is()); + EXPECT_TRUE( + query::boolean_or(idx, {"alpha"}, null_docs).is()); + EXPECT_TRUE( + query::boolean_and(idx, {"alpha"}, null_docs).is()); + EXPECT_TRUE(query::prefix_query(idx, "al", null_docs).is()); + EXPECT_TRUE( + query::wildcard_query(idx, "a*", null_docs).is()); + EXPECT_TRUE( + query::regexp_query(idx, "a.*", null_docs).is()); + EXPECT_TRUE(query::phrase_query(idx, {"alpha"}, null_docs) + .is()); + EXPECT_TRUE(query::phrase_prefix_query(idx, {"alpha"}, null_docs) + .is()); +} + +TEST(SniiQueryOperatorBoundaries, EmptyMissingAndSingleTermCasesAreWellDefined) { + std::vector bytes; + BuildIndexBytes(Corpus {}, doris::snii::format::IndexConfig::kDocsPositionsScoring, &bytes); + FaultInjectingReader file(std::move(bytes)); + SniiSegmentReader segment; + LogicalIndexReader idx = OpenIndex(&file, &segment); + + std::vector got = {99}; + ASSERT_TRUE(query::term_query(idx, "missing", &got).ok()); + EXPECT_TRUE(got.empty()); + + got = {99}; + ASSERT_TRUE(query::boolean_or(idx, {}, &got).ok()); + EXPECT_TRUE(got.empty()); + + got = {99}; + ASSERT_TRUE(query::boolean_and(idx, {}, &got).ok()); + EXPECT_TRUE(got.empty()); + + got = {99}; + ASSERT_TRUE(query::boolean_and(idx, {"alpha", "missing"}, &got).ok()); + EXPECT_TRUE(got.empty()); + + got = {99}; + ASSERT_TRUE(query::phrase_query(idx, {}, &got).ok()); + EXPECT_TRUE(got.empty()); + + got = {99}; + ASSERT_TRUE(query::phrase_prefix_query(idx, {}, &got).ok()); + EXPECT_TRUE(got.empty()); + + ASSERT_TRUE(query::prefix_query(idx, "", &got).ok()); + EXPECT_EQ(got, (std::vector {0, 1, 2, 3, 4})); + + ASSERT_TRUE(query::phrase_query(idx, {"alpha"}, &got).ok()); + EXPECT_EQ(got, (std::vector {0, 1, 2})); + + ASSERT_TRUE(query::phrase_prefix_query(idx, {"alpha"}, &got).ok()); + EXPECT_EQ(got, (std::vector {0, 1, 2, 4})); + + ASSERT_TRUE(query::wildcard_query(idx, "", &got).ok()); + EXPECT_TRUE(got.empty()); + + EXPECT_TRUE(query::regexp_query(idx, "[", &got).is()); +} + +TEST(SniiQueryOperatorBoundaries, PositionQueriesRejectDocsOnlyIndex) { + std::vector bytes; + BuildIndexBytes(Corpus {}, doris::snii::format::IndexConfig::kDocsOnly, &bytes); + FaultInjectingReader file(std::move(bytes)); + SniiSegmentReader segment; + LogicalIndexReader idx = OpenIndex(&file, &segment); + + std::vector got; + EXPECT_TRUE(query::phrase_query(idx, {"alpha", "beta"}, &got) + .is()); + EXPECT_TRUE(query::phrase_prefix_query(idx, {"alpha", "b"}, &got) + .is()); +} + +TEST(SniiQueryOperatorIoErrors, PropagateUnderlyingReadFailures) { + EnvGuard dict_resident("SNII_DICT_RESIDENT_MAX"); + ::setenv("SNII_DICT_RESIDENT_MAX", "0", 1); + + std::vector bytes; + BuildIndexBytes(Corpus {}, doris::snii::format::IndexConfig::kDocsPositionsScoring, &bytes); + FaultInjectingReader file(std::move(bytes)); + SniiSegmentReader segment; + LogicalIndexReader idx = OpenIndex(&file, &segment); + file.set_fail_reads(true); + + std::vector got; + ExpectIoError(query::term_query(idx, "alpha", &got)); + ExpectIoError(query::boolean_or(idx, {"alpha", "beta"}, &got)); + ExpectIoError(query::boolean_and(idx, {"alpha", "beta"}, &got)); + ExpectIoError(query::prefix_query(idx, "al", &got)); + ExpectIoError(query::wildcard_query(idx, "alpha*", &got)); + ExpectIoError(query::regexp_query(idx, "alpha.*", &got)); + ExpectIoError(query::phrase_query(idx, {"alpha", "beta"}, &got)); + ExpectIoError(query::phrase_prefix_query(idx, {"alpha", "b"}, &got)); +} diff --git a/be/test/storage/index/snii/query/query_profile_test.cpp b/be/test/storage/index/snii/query/query_profile_test.cpp new file mode 100644 index 00000000000000..c65a632df9172d --- /dev/null +++ b/be/test/storage/index/snii/query/query_profile_test.cpp @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/query_profile.h" + +#include +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/io/metered_file_reader.h" +#include "storage/index/snii/query/boolean_query.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/regexp_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/query/wildcard_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +using namespace doris::snii; +using namespace doris::snii::reader; +using namespace doris::snii::writer; +using doris::Status; + +namespace { + +std::string TempPath() { + static int counter = 0; + return "/tmp/snii_query_profile_" + std::to_string(getpid()) + "_" + std::to_string(counter++) + + ".idx"; +} + +struct Corpus { + std::vector> docs; +}; + +Corpus BuildCorpus() { + Corpus c; + c.docs.resize(128); + for (uint32_t d = 0; d < c.docs.size(); ++d) { + std::vector& doc = c.docs[d]; + doc.emplace_back("lead"); + doc.emplace_back("quick"); + doc.emplace_back(d % 2 == 0 ? "brown" : "bronze"); + char term[16]; + std::snprintf(term, sizeof(term), "aa_%03u", d); + doc.emplace_back(term); + } + return c; +} + +void WriteCorpus(const Corpus& c, const std::string& path) { + SpimiTermBuffer buf(/*has_positions=*/true); + for (uint32_t d = 0; d < c.docs.size(); ++d) { + const std::vector& terms = c.docs[d]; + for (uint32_t pos = 0; pos < terms.size(); ++pos) { + buf.add_token(terms[pos], d, pos); + } + } + + SniiIndexInput in; + in.index_id = 1; + in.index_suffix = "body"; + in.config = doris::snii::format::IndexConfig::kDocsPositionsScoring; + in.doc_count = static_cast(c.docs.size()); + in.encoded_norms.assign(c.docs.size(), 1); + in.terms = buf.finalize_sorted(); + in.target_dict_block_bytes = 512; + + io::LocalFileWriter writer; + ASSERT_TRUE(writer.open(path).ok()); + SniiCompoundWriter compound(&writer); + ASSERT_TRUE(compound.add_logical_index(in).ok()); + ASSERT_TRUE(compound.finish().ok()); +} + +LogicalIndexReader OpenMeteredIndex(io::MeteredFileReader* file, SniiSegmentReader* segment) { + EXPECT_TRUE(SniiSegmentReader::open(file, segment).ok()); + LogicalIndexReader idx; + EXPECT_TRUE(segment->open_index(1, "body", &idx).ok()); + return idx; +} + +void ExpectProfileMatchesMeteredDelta(io::MeteredFileReader* metered, + const std::function& run) { + metered->reset_metrics(); + query::QueryProfile profile; + const Status st = run(&profile); + ASSERT_TRUE(st.ok()) << st.to_string(); + + EXPECT_GT(profile.elapsed_ns, 0U); + ASSERT_TRUE(profile.has_io_metrics); + EXPECT_EQ(profile.io_delta.read_at_calls, metered->metrics().read_at_calls); + EXPECT_EQ(profile.io_delta.serial_rounds, metered->metrics().serial_rounds); + EXPECT_EQ(profile.io_delta.range_gets, metered->metrics().range_gets); + EXPECT_EQ(profile.io_delta.remote_bytes, metered->metrics().remote_bytes); + EXPECT_EQ(profile.io_delta.total_request_bytes, metered->metrics().total_request_bytes); +} + +} // namespace + +TEST(SniiQueryProfile, ReportsElapsedTimeAndMeteredIoForNativeOperators) { + const Corpus corpus = BuildCorpus(); + const std::string path = TempPath(); + WriteCorpus(corpus, path); + + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + io::MeteredFileReader metered(&local, /*block_size=*/512); + SniiSegmentReader segment; + LogicalIndexReader idx = OpenMeteredIndex(&metered, &segment); + + std::vector docs; + ExpectProfileMatchesMeteredDelta(&metered, [&](query::QueryProfile* profile) { + return query::term_query(idx, "lead", &docs, profile); + }); + ExpectProfileMatchesMeteredDelta(&metered, [&](query::QueryProfile* profile) { + return query::boolean_or(idx, {"lead", "missing"}, &docs, profile); + }); + ExpectProfileMatchesMeteredDelta(&metered, [&](query::QueryProfile* profile) { + return query::boolean_and(idx, {"quick", "brown"}, &docs, profile); + }); + ExpectProfileMatchesMeteredDelta(&metered, [&](query::QueryProfile* profile) { + return query::prefix_query(idx, "aa_", &docs, profile); + }); + ExpectProfileMatchesMeteredDelta(&metered, [&](query::QueryProfile* profile) { + return query::wildcard_query(idx, "aa_0??", &docs, profile); + }); + ExpectProfileMatchesMeteredDelta(&metered, [&](query::QueryProfile* profile) { + return query::regexp_query(idx, "aa_00[0-9]", &docs, profile); + }); + ExpectProfileMatchesMeteredDelta(&metered, [&](query::QueryProfile* profile) { + return query::phrase_query(idx, {"quick", "brown"}, &docs, profile); + }); + ExpectProfileMatchesMeteredDelta(&metered, [&](query::QueryProfile* profile) { + return query::phrase_prefix_query(idx, {"quick", "bro"}, &docs, profile); + }); + + std::remove(path.c_str()); +} + +TEST(SniiQueryProfile, ReportsElapsedTimeForOperatorErrorPath) { + const Corpus corpus = BuildCorpus(); + const std::string path = TempPath(); + WriteCorpus(corpus, path); + + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + io::MeteredFileReader metered(&local, /*block_size=*/512); + SniiSegmentReader segment; + LogicalIndexReader idx = OpenMeteredIndex(&metered, &segment); + + metered.reset_metrics(); + std::vector docs; + query::QueryProfile profile; + const Status st = query::regexp_query(idx, "(", &docs, &profile); + + EXPECT_FALSE(st.ok()); + EXPECT_GT(profile.elapsed_ns, 0U); + ASSERT_TRUE(profile.has_io_metrics); + EXPECT_EQ(profile.io_delta.read_at_calls, 0U); + EXPECT_EQ(profile.io_delta.serial_rounds, 0U); + EXPECT_EQ(profile.io_delta.range_gets, 0U); + EXPECT_EQ(profile.io_delta.remote_bytes, 0U); + EXPECT_EQ(profile.io_delta.total_request_bytes, 0U); + + std::remove(path.c_str()); +} + +TEST(SniiQueryProfile, ScopeFinalizesProfileOnEarlyReturn) { + query::QueryProfile profile; + { query::QueryProfileScope scope(/*reader=*/nullptr, &profile); } + + EXPECT_GT(profile.elapsed_ns, 0U); + EXPECT_FALSE(profile.has_io_metrics); +} diff --git a/be/test/storage/index/snii/query/scoring_query_test.cpp b/be/test/storage/index/snii/query/scoring_query_test.cpp new file mode 100644 index 00000000000000..a3225961b5d277 --- /dev/null +++ b/be/test/storage/index/snii/query/scoring_query_test.cpp @@ -0,0 +1,346 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/query/scoring_query.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/io/metered_file_reader.h" +#include "storage/index/snii/query/bm25_scorer.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/stats/snii_stats_provider.h" +#include "storage/index/snii/writer/logical_index_writer.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +using namespace doris::snii; +using namespace doris::snii::format; +using namespace doris::snii::writer; +using doris::snii::query::Bm25Params; +using doris::snii::query::ScoredDoc; +using doris::snii::stats::SniiStatsProvider; + +namespace { + +std::string TempPath() { + static int counter = 0; + return "/tmp/snii_score_test_" + std::to_string(getpid()) + "_" + std::to_string(counter++) + + ".idx"; +} + +// A small in-memory corpus: each doc is a bag of (term -> freq). Doc lengths vary +// so length normalization matters. "common" is a high-df term (~half the docs), +// "rare" is a low-df term. +struct Corpus { + uint32_t doc_count = 0; + // term -> (docid -> freq), docids ascending. + std::map> postings; + std::vector doc_len; // per-doc total token count +}; + +// Builds ~60 docs with varied lengths and a high-df + low-df term. +Corpus MakeCorpus() { + Corpus c; + c.doc_count = 60; + c.doc_len.assign(c.doc_count, 0); + + auto add = [&](const std::string& term, uint32_t doc, uint32_t freq) { + c.postings[term][doc] += freq; + c.doc_len[doc] += freq; + }; + + for (uint32_t d = 0; d < c.doc_count; ++d) { + // "common": appears in even docs, freq varies 1..4. + if (d % 2 == 0) { + add("common", d, 1 + (d % 4)); + } + // "rare": appears in only a few docs. + if (d == 3 || d == 17 || d == 42) { + add("rare", d, 2); + } + // "filler": gives docs varied lengths so dl differs widely. + add("filler", d, 1 + (d % 7) * 3); + // a unique padding token per doc to spread lengths further. + add("pad" + std::to_string(d % 11), d, (d % 5) + 1); + } + return c; +} + +// Converts the corpus into a sorted SniiIndexInput with encoded norms. +SniiIndexInput ToInput(const Corpus& c) { + SniiIndexInput in; + in.index_id = 1; + in.index_suffix = "body"; + in.config = IndexConfig::kDocsPositionsScoring; + in.doc_count = c.doc_count; + in.target_dict_block_bytes = 1; // one block per term + + in.encoded_norms.resize(c.doc_count); + for (uint32_t d = 0; d < c.doc_count; ++d) { + in.encoded_norms[d] = doris::snii::query::encode_norm(c.doc_len[d]); + } + + for (const auto& [term, plist] : c.postings) { + TermPostings tp; + tp.term = term; + for (const auto& [docid, freq] : plist) { + tp.docids.push_back(docid); + tp.freqs.push_back(freq); + for (uint32_t k = 0; k < freq; ++k) { + tp.positions_flat.push_back(k); // flat + } + } + in.terms.push_back(std::move(tp)); + } + return in; +} + +// Reference BM25 ranking computed directly from the corpus (same encode/decode). +std::vector ReferenceRanking(const Corpus& c, const std::vector& norms, + const std::vector& terms, uint32_t k, + const Bm25Params& params) { + uint64_t sum_ttf = 0; + for (const auto& dl : c.doc_len) { + sum_ttf += dl; + } + const double avgdl = static_cast(sum_ttf) / std::max(1, c.doc_count); + + std::unordered_map scores; + for (const auto& term : terms) { + auto it = c.postings.find(term); + if (it == c.postings.end()) { + continue; + } + const uint64_t df = it->second.size(); + const double idf = + std::log(1.0 + (static_cast(c.doc_count) - df + 0.5) / (df + 0.5)); + for (const auto& [docid, freq] : it->second) { + const double dl = doris::snii::query::decode_norm(norms[docid]); + const double denom = freq + params.k1 * (1.0 - params.b + params.b * dl / avgdl); + scores[docid] += idf * (freq * (params.k1 + 1.0)) / denom; + } + } + + std::vector all; + all.reserve(scores.size()); + for (const auto& [docid, s] : scores) { + all.push_back({docid, s}); + } + std::ranges::sort(all, [](const ScoredDoc& a, const ScoredDoc& b) { + if (a.score != b.score) { + return a.score > b.score; + } + return a.docid < b.docid; + }); + if (all.size() > k) { + all.resize(k); + } + return all; +} + +} // namespace + +// Helper that mirrors ToInput's encoding so the reference path can decode norms. +namespace { +std::vector EncodeNorms(const Corpus& c) { + std::vector v(c.doc_count); + for (uint32_t d = 0; d < c.doc_count; ++d) { + v[d] = doris::snii::query::encode_norm(c.doc_len[d]); + } + return v; +} +} // namespace + +// Fixture-free test: build, open, and compare. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiScoringQuery, ReferenceOracleAndWandEqualsExhaustive) { + const Corpus corpus = MakeCorpus(); + const std::vector norms = EncodeNorms(corpus); + const std::string path = TempPath(); + + // --- build the scoring index --- + { + io::LocalFileWriter w; + ASSERT_TRUE(w.open(path).ok()); + SniiCompoundWriter cw(&w); + ASSERT_TRUE(cw.add_logical_index(ToInput(corpus)).ok()); + ASSERT_TRUE(cw.finish().ok()); + } + + // --- open via SniiSegmentReader over a MeteredFileReader --- + io::LocalFileReader inner; + ASSERT_TRUE(inner.open(path).ok()); + io::MeteredFileReader metered(&inner); + reader::SniiSegmentReader seg; + ASSERT_TRUE(reader::SniiSegmentReader::open(&metered, &seg).ok()); + reader::LogicalIndexReader idx; + ASSERT_TRUE(seg.open_index(1, "body", &idx).ok()); + + SniiStatsProvider stats; + ASSERT_TRUE(SniiStatsProvider::open(&idx, &stats).ok()); + + // (c) SniiStatsProvider df / ttf / avgdl / encoded_norm match brute force. + uint64_t sum_ttf = 0; + for (const auto& dl : corpus.doc_len) { + sum_ttf += dl; + } + EXPECT_EQ(stats.indexed_doc_count(), corpus.doc_count); + EXPECT_EQ(stats.sum_total_term_freq(), sum_ttf); + EXPECT_NEAR(stats.avgdl(), static_cast(sum_ttf) / corpus.doc_count, 1e-9); + + for (const auto& [term, plist] : corpus.postings) { + uint64_t df = 0, ttf = 0; + ASSERT_TRUE(stats.doc_freq(term, &df).ok()); + ASSERT_TRUE(stats.total_term_freq(term, &ttf).ok()); + uint64_t exp_ttf = 0; + for (const auto& [d, f] : plist) { + exp_ttf += f; + } + EXPECT_EQ(df, plist.size()) << term; + EXPECT_EQ(ttf, exp_ttf) << term; + } + for (uint32_t d = 0; d < corpus.doc_count; ++d) { + uint8_t got = 0; + ASSERT_TRUE(stats.encoded_norm(d, &got).ok()); + EXPECT_EQ(got, norms[d]) << "docid " << d; + } + + // (a) single-term scoring_query top-K matches the reference. + const Bm25Params params; // defaults k1=1.2, b=0.75 + const uint32_t k = 10; + + auto run_and_check = [&](const std::vector& terms) { + std::vector reference = ReferenceRanking(corpus, norms, terms, k, params); + std::vector exhaustive; + ASSERT_TRUE(doris::snii::query::scoring_query_exhaustive(idx, stats, terms, k, params, + &exhaustive) + .ok()); + std::vector wand; + ASSERT_TRUE( + doris::snii::query::scoring_query_wand(idx, stats, terms, k, params, &wand).ok()); + + ASSERT_EQ(exhaustive.size(), reference.size()); + for (size_t i = 0; i < reference.size(); ++i) { + EXPECT_EQ(exhaustive[i].docid, reference[i].docid) << "rank " << i; + EXPECT_NEAR(exhaustive[i].score, reference[i].score, 1e-6) << "rank " << i; + } + // (b) WAND-pruned top-K equals the exhaustive top-K. + ASSERT_EQ(wand.size(), exhaustive.size()); + for (size_t i = 0; i < wand.size(); ++i) { + EXPECT_EQ(wand[i].docid, exhaustive[i].docid) << "wand rank " << i; + EXPECT_NEAR(wand[i].score, exhaustive[i].score, 1e-6) << "wand rank " << i; + } + }; + + run_and_check({"common"}); + run_and_check({"rare"}); + run_and_check({"common", "rare"}); + run_and_check({"common", "rare", "filler"}); + + std::remove(path.c_str()); +} + +namespace { + +// A corpus engineered to produce SCORE TIES at the top-k boundary and to drive +// the WINDOWED posting path: uniform doc length (so length-norm is constant) and +// high-df terms (df >= kSlimDfThreshold = 512 -> windowed pod_ref + frq_prelude). +// Every doc has the same length L=8, so docs sharing a term/freq score identically. +Corpus MakeWindowedTieCorpus() { + Corpus c; + c.doc_count = 700; // >= 512 so "anchor" becomes a windowed term + c.doc_len.assign(c.doc_count, 0); + auto add = [&](const std::string& term, uint32_t doc, uint32_t freq) { + c.postings[term][doc] += freq; + c.doc_len[doc] += freq; + }; + for (uint32_t d = 0; d < c.doc_count; ++d) { + add("anchor", d, 1); // df=700 (windowed), freq=1 everywhere -> ties + if (d % 2 == 0) { + add("evenz", d, 1); // df=350 (windowed), another high-df term + } + add("u" + std::to_string(d), d, 6); // unique pad: keeps every dl == 8 exactly + } + return c; +} + +} // namespace + +// Differential: WAND top-k MUST equal exhaustive top-k EVEN with boundary ties and +// windowed (block-max) terms, across many k. Strict-'>' pruning would drop ties. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiScoringQuery, WandEqualsExhaustiveWithTiesAndWindowedTerms) { + const Corpus corpus = MakeWindowedTieCorpus(); + const std::string path = TempPath(); + { + io::LocalFileWriter w; + ASSERT_TRUE(w.open(path).ok()); + SniiCompoundWriter cw(&w); + ASSERT_TRUE(cw.add_logical_index(ToInput(corpus)).ok()); + ASSERT_TRUE(cw.finish().ok()); + } + io::LocalFileReader inner; + ASSERT_TRUE(inner.open(path).ok()); + io::MeteredFileReader metered(&inner); + reader::SniiSegmentReader seg; + ASSERT_TRUE(reader::SniiSegmentReader::open(&metered, &seg).ok()); + reader::LogicalIndexReader idx; + ASSERT_TRUE(seg.open_index(1, "body", &idx).ok()); + SniiStatsProvider stats; + ASSERT_TRUE(SniiStatsProvider::open(&idx, &stats).ok()); + + const Bm25Params params; + const std::vector norms = EncodeNorms(corpus); + auto check = [&](const std::vector& terms, uint32_t k) { + std::vector ex, wa; + ASSERT_TRUE(scoring_query_exhaustive(idx, stats, terms, k, params, &ex).ok()); + ASSERT_TRUE(scoring_query_wand(idx, stats, terms, k, params, &wa).ok()); + const std::vector ref = ReferenceRanking(corpus, norms, terms, k, params); + ASSERT_EQ(wa.size(), ex.size()); + ASSERT_EQ(ex.size(), ref.size()); + for (size_t i = 0; i < ex.size(); ++i) { + EXPECT_EQ(wa[i].docid, ex[i].docid) + << "terms[0]=" << terms[0] << " k=" << k << " i=" << i; + EXPECT_EQ(ex[i].docid, ref[i].docid) << "ref k=" << k << " i=" << i; + EXPECT_NEAR(wa[i].score, ex[i].score, 1e-9); + } + }; + // Single high-df term: all 700 docs tie -> top-k must be the k smallest docids. + for (uint32_t k : {1U, 3U, 5U, 50U, 200U}) { + check({"anchor"}, k); + } + // Two windowed terms: even docs score higher (two terms) -> ties within each tier. + for (uint32_t k : {1U, 4U, 10U, 100U}) { + check({"anchor", "evenz"}, k); + } + + std::remove(path.c_str()); +} diff --git a/be/test/storage/index/snii/query/scoring_wand_selective_test.cpp b/be/test/storage/index/snii/query/scoring_wand_selective_test.cpp new file mode 100644 index 00000000000000..fe66f9fbe2f0d9 --- /dev/null +++ b/be/test/storage/index/snii/query/scoring_wand_selective_test.cpp @@ -0,0 +1,514 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/io/metered_file_reader.h" +#include "storage/index/snii/query/bm25_scorer.h" +#include "storage/index/snii/query/scoring_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/reader/windowed_posting.h" +#include "storage/index/snii/stats/snii_stats_provider.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +// Phase C differential test: scoring_query_wand_selective (block-max SELECTIVE +// FETCH) MUST return top-K (docid sequence AND scores within 1e-9) byte-identical +// to scoring_query_exhaustive AND scoring_query_wand AND an in-memory brute-force +// reference -- for MANY random queries, varied k (incl k=1), varied term sets +// (incl a high-df windowed term spanning many .frq windows), AND a corpus crafted +// to force SCORE TIES (uniform doc length). It additionally asserts the selective +// path fetches FEWER .frq windows / bytes than reading all windows for a small-k +// high-df query, WITHOUT ever changing the result. +using namespace doris::snii; +using namespace doris::snii::format; +using namespace doris::snii::reader; +using namespace doris::snii::writer; +using doris::snii::query::Bm25Params; +using doris::snii::query::ScoredDoc; +using doris::snii::stats::SniiStatsProvider; + +namespace { + +std::string TempPath() { + static int counter = 0; + return "/tmp/snii_wand_sel_" + std::to_string(getpid()) + "_" + std::to_string(counter++) + + ".idx"; +} + +// term -> (docid -> freq); plus per-doc length (token count) for norms. +struct Corpus { + uint32_t doc_count = 0; + std::map> postings; + std::vector doc_len; +}; + +// A scoring corpus with a HIGH-DF term ("hi", df=doc_count -> windowed, spans +// many .frq windows), two MID-df windowed terms, and several low-df terms. Doc +// lengths VARY (so length-norm matters and scores spread out for the random +// queries), but every doc that carries "hi" carries exactly freq=1 of it. +Corpus MakeVariedCorpus(uint32_t doc_count) { + Corpus c; + c.doc_count = doc_count; + c.doc_len.assign(doc_count, 0); + auto add = [&](const std::string& t, uint32_t d, uint32_t f) { + c.postings[t][d] += f; + c.doc_len[d] += f; + }; + // All queryable terms use an "aa_" prefix and a "zz_NNN" filler vocabulary fills + // the lexicographic tail, so every real term sorts within the SampledTermIndex's + // candidate range (the index samples per-block first terms; without tail filler + // a term sorting last can fall outside the sampled range and miss on lookup). + for (uint32_t d = 0; d < doc_count; ++d) { + add("aa_hi", d, 1 + (d % 5)); // df=N, windowed, varied freq + if (d % 2 == 0) { + add("aa_evenmid", d, 1 + (d % 3)); + } + if (d % 3 == 0) { + add("aa_thirdmid", d, 1 + (d % 4)); + } + if (d % 37 == 0) { + add("aa_rarea", d, 2); + } + if (d % 53 == 0) { + add("aa_rareb", d, 3); + } + if (d % 101 == 0) { + add("aa_rarec", d, 1); + } + add("aa_pad" + std::to_string(d % 13), d, 1 + (d % 7)); // spreads dl widely + char nm[16]; + std::snprintf(nm, sizeof(nm), "zz_%03u", d % 257); // tail filler vocabulary + add(nm, d, 1); + } + return c; +} + +// A TIE corpus: uniform doc length L=8 for every doc (so equal-freq docs of a +// term score identically -> ties at the top-K boundary). "hi"/"evenz" are high-df +// windowed terms with freq=1 everywhere -> massive ties broken only by docid. +Corpus MakeTieCorpus(uint32_t doc_count) { + Corpus c; + c.doc_count = doc_count; + c.doc_len.assign(doc_count, 0); + auto add = [&](const std::string& t, uint32_t d, uint32_t f) { + c.postings[t][d] += f; + c.doc_len[d] += f; + }; + for (uint32_t d = 0; d < doc_count; ++d) { + add("aa_hi", d, 1); // df=N, freq=1 everywhere -> ties + if (d % 2 == 0) { + add("aa_evenz", d, 1); // another high-df windowed term + } + if (d % 41 == 0) { + add("aa_rarez", d, 1); // low-df, still freq=1 + } + // unique pad keeps every dl == 8 exactly; named in the "m_" mid range so the + // queryable "aa_" terms still sort within the sampled candidate range. + add("mm_u" + std::to_string(d), d, 6); + } + return c; +} + +// A DECAYING corpus engineered so block-max WAND genuinely SKIPS later windows of +// the high-df term "hi": "hi"'s in-doc freq decreases with docid (early docs have +// the highest tf) and doc length increases with docid (early docs are shortest, +// so least length-penalized). Thus EARLY windows hold the top scorers and fill the +// heap with a high theta, after which LATE windows' block-max (small tf, long dl) +// falls below theta and is provably skippable -- selective never fetches them. +Corpus MakeDecayCorpus(uint32_t doc_count) { + Corpus c; + c.doc_count = doc_count; + c.doc_len.assign(doc_count, 0); + auto add = [&](const std::string& t, uint32_t d, uint32_t f) { + c.postings[t][d] += f; + c.doc_len[d] += f; + }; + for (uint32_t d = 0; d < doc_count; ++d) { + // tf decays from ~20 (docid 0) down to 1 (last docids). + const uint32_t band = d / 256; // one step per window-sized band + const uint32_t tf = band < 20 ? (20 - band) : 1; + add("aa_hi", d, tf); + // Padding grows with docid so later docs are longer (higher length penalty). + add("mm_pad" + std::to_string(d % 7), d, 1 + band); + } + return c; +} + +SniiIndexInput ToInput(const Corpus& c) { + SniiIndexInput in; + in.index_id = 1; + in.index_suffix = "body"; + in.config = IndexConfig::kDocsPositionsScoring; + in.doc_count = c.doc_count; + in.target_dict_block_bytes = 256; + in.encoded_norms.resize(c.doc_count); + for (uint32_t d = 0; d < c.doc_count; ++d) { + in.encoded_norms[d] = doris::snii::query::encode_norm(c.doc_len[d]); + } + for (const auto& [term, plist] : c.postings) { + TermPostings tp; + tp.term = term; + for (const auto& [docid, freq] : plist) { + tp.docids.push_back(docid); + tp.freqs.push_back(freq); + for (uint32_t k = 0; k < freq; ++k) { + tp.positions_flat.push_back(k); // flat + } + } + in.terms.push_back(std::move(tp)); + } + return in; +} + +std::vector EncodeNorms(const Corpus& c) { + std::vector v(c.doc_count); + for (uint32_t d = 0; d < c.doc_count; ++d) { + v[d] = doris::snii::query::encode_norm(c.doc_len[d]); + } + return v; +} + +// Independent brute-force BM25 ranking straight from the corpus. +std::vector ReferenceRanking(const Corpus& c, const std::vector& norms, + const std::vector& terms, uint32_t k, + const Bm25Params& params) { + uint64_t sum_ttf = 0; + for (const auto& dl : c.doc_len) { + sum_ttf += dl; + } + const double avgdl = static_cast(sum_ttf) / std::max(1, c.doc_count); + std::unordered_map scores; + for (const auto& term : terms) { + auto it = c.postings.find(term); + if (it == c.postings.end()) { + continue; + } + const uint64_t df = it->second.size(); + const double idf = + std::log(1.0 + (static_cast(c.doc_count) - df + 0.5) / (df + 0.5)); + for (const auto& [docid, freq] : it->second) { + const double dl = doris::snii::query::decode_norm(norms[docid]); + const double denom = freq + params.k1 * (1.0 - params.b + params.b * dl / avgdl); + scores[docid] += idf * (freq * (params.k1 + 1.0)) / denom; + } + } + std::vector all; + all.reserve(scores.size()); + for (const auto& [docid, s] : scores) { + all.push_back({docid, s}); + } + std::ranges::sort(all, [](const ScoredDoc& a, const ScoredDoc& b) { + if (a.score != b.score) { + return a.score > b.score; + } + return a.docid < b.docid; + }); + if (all.size() > k) { + all.resize(k); + } + return all; +} + +// Opens a built index file over a metered reader. +struct OpenedIndex { + io::LocalFileReader inner; + io::MeteredFileReader metered; + SniiSegmentReader seg; + LogicalIndexReader idx; + SniiStatsProvider stats; + explicit OpenedIndex(const std::string& path, size_t block_size) : metered(&inner, block_size) { + EXPECT_TRUE(inner.open(path).ok()); + } +}; + +void BuildAndOpen(const Corpus& c, const std::string& path) { + io::LocalFileWriter w; + ASSERT_TRUE(w.open(path).ok()); + SniiCompoundWriter cw(&w); + ASSERT_TRUE(cw.add_logical_index(ToInput(c)).ok()); + ASSERT_TRUE(cw.finish().ok()); +} + +// Counts the .frq window count of a windowed term (its full posting window span). +uint32_t WindowCount(const LogicalIndexReader& idx, const std::string& term) { + DictEntry entry; + uint64_t frq_base = 0, prx_base = 0; + bool found = false; + EXPECT_TRUE(idx.lookup(term, &found, &entry, &frq_base, &prx_base).ok()); + if (!found || entry.enc != DictEntryEnc::kWindowed) { + return 0; + } + FrqPreludeReader prelude; + EXPECT_TRUE(fetch_windowed_prelude(idx, entry, frq_base, &prelude).ok()); + return prelude.n_windows(); +} + +// Asserts two top-K vectors are BYTE-IDENTICAL: same length, same docid sequence, +// scores equal within tol. This is the hard soundness gate used for selective vs +// the eager WAND (both implement the documented (score desc, docid asc) order and +// the >= theta tie rule, so they must agree exactly, ties included). +void ExpectIdentical(const std::vector& a, const std::vector& b, + const std::string& label, double tol) { + ASSERT_EQ(a.size(), b.size()) << label << " size"; + for (size_t i = 0; i < a.size(); ++i) { + EXPECT_EQ(a[i].docid, b[i].docid) << label << " docid rank " << i; + EXPECT_NEAR(a[i].score, b[i].score, tol) << label << " score rank " << i; + } +} + +// Asserts a is a VALID top-K relative to oracle b, tolerant ONLY of the floating- +// point ULP tie-order ambiguity inherent to BM25 (two docs whose summed scores +// differ by < tol may legitimately swap order across summation orders; the +// exhaustive path sums per doc in unordered_map order, the WAND path in cursor +// order). It groups each vector into maximal tie bands (consecutive scores within +// tol) and requires: +// - identical length and position-wise scores within tol (any real score / +// ranking error shows here -- positions only slide WITHIN an equal band), +// - the docid MULTISET of each tie band is identical (catches wrong / dropped +// docids, including at the top-K boundary when the boundary is not in a band), +// - a's docids are ascending within each band (a is the canonical winner). +void ExpectValidTopK(const std::vector& a, const std::vector& b, + const std::string& label, double tol) { + ASSERT_EQ(a.size(), b.size()) << label << " size"; + size_t i = 0; + while (i < a.size()) { + size_t j = i + 1; + while (j < a.size() && std::abs(a[j].score - a[i].score) <= tol) { + ++j; + } + std::vector ad, bd; + for (size_t t = i; t < j; ++t) { + EXPECT_NEAR(a[t].score, b[t].score, tol) << label << " band score " << t; + ad.push_back(a[t].docid); + bd.push_back(b[t].docid); + if (t > i) { + EXPECT_LT(a[t - 1].docid, a[t].docid) + << label << " a not docid-ascending in tie band at " << t; + } + } + std::ranges::sort(ad); + std::ranges::sort(bd); + EXPECT_EQ(ad, bd) << label << " tie-band docid set differs near rank " << i; + i = j; + } +} + +} // namespace + +// Selective vs exhaustive / eager-WAND / brute-force over MANY random queries on a +// varied-length scoring corpus with a high-df windowed term spanning many windows. +// The hard gate: selective is BYTE-IDENTICAL to the eager WAND (same docids, same +// scores, ties included) -- selective only changes the bytes read. Against the +// exhaustive oracle and the brute-force reference it must be a VALID top-K (same +// scores, same docid sets per tie band); those two paths sum per-doc in a +// different order, so they may legitimately swap docids only WITHIN an equal-score +// tie band (BM25 floating-point ULP), which ExpectValidTopK tolerates while still +// catching any real ranking divergence. +TEST(SniiScoringWandSelective, MatchesExhaustiveOverRandomQueries) { + const Corpus corpus = MakeVariedCorpus(4000); + const std::vector norms = EncodeNorms(corpus); + const std::string path = TempPath(); + BuildAndOpen(corpus, path); + + OpenedIndex oi(path, /*block_size=*/4096); + ASSERT_TRUE(SniiSegmentReader::open(&oi.metered, &oi.seg).ok()); + ASSERT_TRUE(oi.seg.open_index(1, "body", &oi.idx).ok()); + ASSERT_TRUE(SniiStatsProvider::open(&oi.idx, &oi.stats).ok()); + + ASSERT_GE(WindowCount(oi.idx, "aa_hi"), 8U) << "aa_hi must span many windows"; + + const Bm25Params params; + const std::vector vocab = {"aa_hi", "aa_evenmid", "aa_thirdmid", + "aa_rarea", "aa_rareb", "aa_rarec", + "aa_pad0", "aa_pad7", "aa_missing"}; + std::mt19937 rng(0xC0FFEEU); + std::uniform_int_distribution n_terms(1, 4); + std::uniform_int_distribution pick(0, vocab.size() - 1); + const std::vector ks = {1, 2, 3, 5, 10, 25, 100, 500}; + std::uniform_int_distribution pick_k(0, ks.size() - 1); + + for (int iter = 0; iter < 300; ++iter) { + // Distinct query terms (a query never repeats a term in practice). + std::vector terms; + const uint32_t nt = n_terms(rng); + for (uint32_t t = 0; t < nt; ++t) { + const std::string cand = vocab[pick(rng)]; + if (std::ranges::find(terms, cand) == terms.end()) { + terms.push_back(cand); + } + } + if (terms.empty()) { + continue; + } + const uint32_t k = ks[pick_k(rng)]; + + std::vector sel, ex, wa; + ASSERT_TRUE(doris::snii::query::scoring_query_wand_selective(oi.idx, oi.stats, terms, k, + params, &sel) + .ok()); + ASSERT_TRUE(doris::snii::query::scoring_query_exhaustive(oi.idx, oi.stats, terms, k, params, + &ex) + .ok()); + ASSERT_TRUE(doris::snii::query::scoring_query_wand(oi.idx, oi.stats, terms, k, params, &wa) + .ok()); + const std::vector ref = ReferenceRanking(corpus, norms, terms, k, params); + + std::string label = "iter " + std::to_string(iter) + " k=" + std::to_string(k) + " terms:"; + for (const auto& t : terms) { + label += " " + t; + } + + // HARD GATE: selective == eager WAND exactly (docids + scores, ties included). + ExpectIdentical(sel, wa, label + " [sel==wand]", 1e-9); + // Selective is a valid top-K vs the exhaustive oracle and brute-force ref. + ExpectValidTopK(sel, ex, label + " [sel~ex]", 1e-9); + ExpectValidTopK(sel, ref, label + " [sel~ref]", 1e-9); + } + std::remove(path.c_str()); +} + +// Selective == exhaustive == wand == brute-force WITH SCORE TIES (uniform doc +// length) across many k, including k=1. Strict pruning that dropped ties would +// fail here; the >= theta tie rule must be preserved end to end. +TEST(SniiScoringWandSelective, MatchesExhaustiveWithTies) { + const Corpus corpus = MakeTieCorpus(2400); + const std::vector norms = EncodeNorms(corpus); + const std::string path = TempPath(); + BuildAndOpen(corpus, path); + + OpenedIndex oi(path, /*block_size=*/4096); + ASSERT_TRUE(SniiSegmentReader::open(&oi.metered, &oi.seg).ok()); + ASSERT_TRUE(oi.seg.open_index(1, "body", &oi.idx).ok()); + ASSERT_TRUE(SniiStatsProvider::open(&oi.idx, &oi.stats).ok()); + ASSERT_GE(WindowCount(oi.idx, "aa_hi"), 4U); + + const Bm25Params params; + auto check = [&](const std::vector& terms, uint32_t k) { + std::vector sel, ex, wa; + ASSERT_TRUE(doris::snii::query::scoring_query_wand_selective(oi.idx, oi.stats, terms, k, + params, &sel) + .ok()); + ASSERT_TRUE(doris::snii::query::scoring_query_exhaustive(oi.idx, oi.stats, terms, k, params, + &ex) + .ok()); + ASSERT_TRUE(doris::snii::query::scoring_query_wand(oi.idx, oi.stats, terms, k, params, &wa) + .ok()); + const std::vector ref = ReferenceRanking(corpus, norms, terms, k, params); + std::string label = "ties terms[0]=" + terms[0] + " k=" + std::to_string(k); + // Uniform doc length => EXACT ties (bitwise-equal scores): every path must + // agree exactly, ordering equal-score docs by ascending docid (the >= theta + // tie rule). This is the strongest possible selective==exhaustive gate. + ExpectIdentical(sel, ex, label + " [sel==ex]", 1e-9); + ExpectIdentical(sel, wa, label + " [sel==wand]", 1e-9); + ExpectIdentical(sel, ref, label + " [sel==ref]", 1e-9); + }; + for (uint32_t k : {1U, 2U, 3U, 5U, 25U, 200U, 1000U}) { + check({"aa_hi"}, k); + } + for (uint32_t k : {1U, 4U, 10U, 100U, 800U}) { + check({"aa_hi", "aa_evenz"}, k); + } + for (uint32_t k : {1U, 5U, 50U}) { + check({"aa_hi", "aa_evenz", "aa_rarez"}, k); + } + std::remove(path.c_str()); +} + +// Selective fetch reads FEWER .frq windows / bytes than reading EVERY window of +// the high-df term, for a small-k single-high-df-term query -- with NO change in +// the result. The decay corpus puts the top scorers in the EARLY windows so the +// block-max of later windows provably falls below theta and they are skipped. We +// measure the selective path's remote_bytes / requested bytes against the eager +// full-posting read of "hi" through the same metered reader. +TEST(SniiScoringWandSelective, FetchesFewerWindowsForSmallKHighDf) { + const Corpus corpus = MakeDecayCorpus(8000); + const std::string path = TempPath(); + BuildAndOpen(corpus, path); + + // Fine-grained cache block: the high-df frq span is only a few KiB, so a coarse + // 4096-byte block can round BOTH the selective and full reads up to the same + // aligned block set even though their raw request sizes differ. A 512-byte block + // keeps the block-aligned remote_bytes measure faithful to the genuine per-window + // savings (independent of where the interleaved posting region happens to land). + OpenedIndex oi(path, /*block_size=*/512); + ASSERT_TRUE(SniiSegmentReader::open(&oi.metered, &oi.seg).ok()); + ASSERT_TRUE(oi.seg.open_index(1, "body", &oi.idx).ok()); + ASSERT_TRUE(SniiStatsProvider::open(&oi.idx, &oi.stats).ok()); + const uint32_t hi_windows = WindowCount(oi.idx, "aa_hi"); + ASSERT_GE(hi_windows, 16U) << "aa_hi must span many windows"; + + const Bm25Params params; + const std::vector terms = {"aa_hi"}; + const uint32_t k = 5; + + // Selective path metrics (prelude + only the surviving windows fetched). + oi.metered.reset_metrics(); + std::vector sel; + ASSERT_TRUE(doris::snii::query::scoring_query_wand_selective(oi.idx, oi.stats, terms, k, params, + &sel) + .ok()); + const io::IoMetrics sel_m = oi.metered.metrics(); + + // Full-posting read of the same term (every window's full .frq), as the + // "read all windows" baseline through the same metered reader. + oi.metered.reset_metrics(); + DictEntry entry; + uint64_t frq_base = 0, prx_base = 0; + bool found = false; + ASSERT_TRUE(oi.idx.lookup("aa_hi", &found, &entry, &frq_base, &prx_base).ok()); + ASSERT_TRUE(found); + DecodedPosting dp; + ASSERT_TRUE(read_windowed_posting(oi.idx, entry, frq_base, prx_base, + /*want_positions=*/false, /*want_freq=*/true, &dp) + .ok()); + const io::IoMetrics full_m = oi.metered.metrics(); + + // Result must be unchanged vs exhaustive (byte savings never change the answer). + std::vector ex; + ASSERT_TRUE( + doris::snii::query::scoring_query_exhaustive(oi.idx, oi.stats, terms, k, params, &ex) + .ok()); + ExpectValidTopK(sel, ex, "decay small-k [sel~ex]", 1e-9); + + // Selective requests strictly fewer raw bytes than reading all windows: later + // windows are skipped, not merely cached. + EXPECT_LT(sel_m.total_request_bytes, full_m.total_request_bytes) + << "sel=" << sel_m.total_request_bytes << " full=" << full_m.total_request_bytes; + // And fewer remote bytes leave the wire. + EXPECT_LT(sel_m.remote_bytes, full_m.remote_bytes) + << "sel=" << sel_m.remote_bytes << " full=" << full_m.remote_bytes; + + std::remove(path.c_str()); +} diff --git a/be/test/storage/index/snii/reader/snii_end_to_end_test.cpp b/be/test/storage/index/snii/reader/snii_end_to_end_test.cpp new file mode 100644 index 00000000000000..e5f2a2c7576dd8 --- /dev/null +++ b/be/test/storage/index/snii/reader/snii_end_to_end_test.cpp @@ -0,0 +1,295 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/io/metered_file_reader.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +using namespace doris::snii; +using namespace doris::snii::format; +using namespace doris::snii::reader; +using namespace doris::snii::writer; + +namespace { + +std::string TempPath() { + static int counter = 0; + return "/tmp/snii_e2e_test_" + std::to_string(getpid()) + "_" + std::to_string(counter++) + + ".idx"; +} + +// Deterministic corpus oracle: term -> set, and doc -> token sequence. +struct Corpus { + uint32_t doc_count = 0; + std::vector> docs; // docs[d] = ordered token list + std::map> term_docs; + + // Does the phrase occur consecutively in doc d? + bool phrase_in_doc(uint32_t d, const std::vector& phrase) const { + if (phrase.empty()) { + return false; + } + const auto& toks = docs[d]; + if (toks.size() < phrase.size()) { + return false; + } + for (size_t i = 0; i + phrase.size() <= toks.size(); ++i) { + bool match = true; + for (size_t k = 0; k < phrase.size(); ++k) { + if (toks[i + k] != phrase[k]) { + match = false; + break; + } + } + if (match) { + return true; + } + } + return false; + } + + std::vector phrase_oracle(const std::vector& phrase) const { + std::vector out; + for (uint32_t d = 0; d < doc_count; ++d) { + if (phrase_in_doc(d, phrase)) { + out.push_back(d); + } + } + return out; + } +}; + +// Builds a deterministic corpus of 60 docs. Plants a HIGH-frequency term +// "the" in every doc (df=60, but we also force a high-df term via padding to +// exercise the windowed path), several low-df terms, and known phrases. +Corpus BuildCorpus() { + Corpus c; + c.doc_count = 60; + c.docs.resize(c.doc_count); + + // Common base vocabulary cycled deterministically. + const std::vector vocab = {"alpha", "bravo", "charlie", "delta", + "echo", "foxtrot", "golf", "hotel"}; + + for (uint32_t d = 0; d < c.doc_count; ++d) { + std::vector& toks = c.docs[d]; + // "the" in every doc -> high df. + toks.emplace_back("the"); + // A planted 5-term phrase in docs where d % 7 == 0. + if (d % 7 == 0) { + toks.emplace_back("quick"); + toks.emplace_back("brown"); + toks.emplace_back("fox"); + toks.emplace_back("jumps"); + toks.emplace_back("over"); + } + // A planted 2-term phrase "lazy dog" in docs where d % 5 == 0. + if (d % 5 == 0) { + toks.emplace_back("lazy"); + toks.emplace_back("dog"); + } + // Repeated-term phrase to verify execution de-duplicates reads without + // weakening positional semantics. + if (d % 13 == 0) { + toks.emplace_back("repeat"); + toks.emplace_back("repeat"); + } + // Cycle through vocab to give terms varied dfs. + for (uint32_t k = 0; k < 4; ++k) { + toks.push_back(vocab[(d + k) % vocab.size()]); + } + // A unique low-df marker per doc bucket. + if (d % 11 == 0) { + toks.emplace_back("rare"); + } + } + + for (uint32_t d = 0; d < c.doc_count; ++d) { + for (const auto& t : c.docs[d]) { + c.term_docs[t].insert(d); + } + } + return c; +} + +// Adds a synthetic high-df term spanning > kSlimDfThreshold docs to force the +// windowed pod_ref path. Uses an expanded doc space; the term appears in docs +// [0, df) at position 0. Updates the oracle. +void PlantHighDfTerm(Corpus* c, const std::string& term, uint32_t df) { + // Expand the corpus to hold df docs if needed. + if (df > c->doc_count) { + uint32_t old = c->doc_count; + c->doc_count = df; + c->docs.resize(df); + (void)old; + } + for (uint32_t d = 0; d < df; ++d) { + c->docs[d].insert(c->docs[d].begin(), term); // position 0 + c->term_docs[term].insert(d); + } + // Recompute term_docs fully to keep positions/oracle consistent. + c->term_docs.clear(); + for (uint32_t d = 0; d < c->doc_count; ++d) { + for (const auto& t : c->docs[d]) { + c->term_docs[t].insert(d); + } + } +} + +// Feeds the corpus into a SpimiTermBuffer and writes a single-index container. +void WriteCorpus(const Corpus& c, const std::string& path) { + SpimiTermBuffer buf(/*has_positions=*/true); + for (uint32_t d = 0; d < c.doc_count; ++d) { + const auto& toks = c.docs[d]; + for (uint32_t pos = 0; pos < toks.size(); ++pos) { + buf.add_token(toks[pos], d, pos); + } + } + std::vector terms = buf.finalize_sorted(); + + SniiIndexInput in; + in.index_id = 1; + in.index_suffix = "body"; + in.config = IndexConfig::kDocsPositions; + in.doc_count = c.doc_count; + in.terms = std::move(terms); + // Small block target so we get multiple DICT blocks (exercises sampling). + in.target_dict_block_bytes = 256; + + io::LocalFileWriter w; + ASSERT_TRUE(w.open(path).ok()); + SniiCompoundWriter cw(&w); + ASSERT_TRUE(cw.add_logical_index(in).ok()); + ASSERT_TRUE(cw.finish().ok()); +} + +std::vector SetToVec(const std::set& s) { + // NOLINTNEXTLINE(modernize-return-braced-init-list) + return std::vector(s.begin(), s.end()); +} + +} // namespace + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiEndToEnd, TermAndPhraseAgainstOracle) { + Corpus c = BuildCorpus(); + // Force a windowed (df >= 512) term to exercise the windowed pod_ref path. + PlantHighDfTerm(&c, "ubiquitous", 600); + + const std::string path = TempPath(); + WriteCorpus(c, path); + + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + io::MeteredFileReader metered(&local); + + SniiSegmentReader seg; + ASSERT_TRUE(SniiSegmentReader::open(&metered, &seg).ok()); + EXPECT_EQ(seg.n_logical_indexes(), 1U); + + LogicalIndexReader idx; + ASSERT_TRUE(seg.open_index(1, "body", &idx).ok()); + EXPECT_EQ(idx.stats().doc_count, c.doc_count); + + // Small DICT blocks are resident after open: exact lookup should not add + // query path I/O once the index is open. + metered.reset_metrics(); + { + DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + bool found = false; + ASSERT_TRUE(idx.lookup("quick", &found, &entry, &frq_base, &prx_base).ok()); + ASSERT_TRUE(found); + EXPECT_EQ(metered.metrics().read_at_calls, 0U); + } + + // ---- term_query: present terms match the oracle docid set exactly. ---- + std::vector present_terms = {"the", "quick", "brown", "fox", + "jumps", "over", "lazy", "dog", + "rare", "alpha", "repeat", "ubiquitous"}; + for (const auto& t : present_terms) { + std::vector got; + ASSERT_TRUE(query::term_query(idx, t, &got).ok()) << t; + std::vector want = SetToVec(c.term_docs[t]); + EXPECT_EQ(got, want) << "term=" << t; + } + + // ---- term_query: absent terms -> empty. ---- + std::vector absent_terms = {"nonexistent", "zzzz", "qqqq-absent"}; + for (const auto& t : absent_terms) { + std::vector got; + ASSERT_TRUE(query::term_query(idx, t, &got).ok()) << t; + EXPECT_TRUE(got.empty()) << "term=" << t; + } + + // ---- phrase_query: present and absent phrases vs oracle. ---- + std::vector> phrases = { + {"quick", "brown", "fox", "jumps", "over"}, // 5-term planted phrase + {"lazy", "dog"}, // 2-term planted phrase + {"repeat", "repeat"}, // repeated-term phrase + {"brown", "fox"}, // sub-phrase + {"fox", "brown"}, // reversed -> absent + {"quick", "fox"}, // non-consecutive -> absent + {"the"}, // single-term phrase + {"nope", "missing"}, // absent terms -> empty + }; + for (const auto& p : phrases) { + std::vector got; + ASSERT_TRUE(query::phrase_query(idx, p, &got).ok()); + std::vector want = c.phrase_oracle(p); + std::string label; + for (const auto& w : p) { + label += w + " "; + } + EXPECT_EQ(got, want) << "phrase=[" << label << "]"; + } + + // ---- metrics sanity: a fresh phrase query over windowed postings + // plans/batches its reads. Small DICT blocks are resident and small postings + // may be inline, so use the synthetic high-df term to force external posting + // I/O. + metered.reset_metrics(); + std::vector got; + ASSERT_TRUE(query::phrase_query(idx, {"ubiquitous", "the"}, &got).ok()); + const auto& m = metered.metrics(); + EXPECT_GE(m.serial_rounds, 1U); + EXPECT_GE(m.range_gets, 1U); + // The planned batch should keep rounds small (resident DICT + batched + // postings). + EXPECT_LE(m.serial_rounds, 8U) << "phrase query did not batch its I/O"; + + std::remove(path.c_str()); +} diff --git a/be/test/storage/index/snii/reader/snii_segment_reader_open_test.cpp b/be/test/storage/index/snii/reader/snii_segment_reader_open_test.cpp new file mode 100644 index 00000000000000..846a60bbbceff5 --- /dev/null +++ b/be/test/storage/index/snii/reader/snii_segment_reader_open_test.cpp @@ -0,0 +1,228 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Focused open()-path tests for SniiSegmentReader. These pin two guarantees of +// the offset-0 bootstrap-read removal: +// 1. open() issues NO read intersecting the bootstrap header region +// [0, kBootstrapHeaderSize) -- the redundant offset-0 cache block / remote +// round-trip is gone. +// 2. The container version gate is preserved by the tail pointer: a corrupt +// offset-0 bootstrap header no longer fails open(), but a corrupt tail +// pointer format_version still does. + +#include +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/bootstrap_header.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/tail_pointer.h" +#include "storage/index/snii/io/file_reader.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +using namespace doris::snii; +using namespace doris::snii::format; +using namespace doris::snii::reader; +using namespace doris::snii::writer; +namespace ErrorCode = doris::ErrorCode; +using doris::Status; + +namespace { + +std::string TempPath() { + static int counter = 0; + return "/tmp/snii_seg_open_test_" + std::to_string(getpid()) + "_" + std::to_string(counter++) + + ".idx"; +} + +// An in-memory FileReader over an owned byte buffer that RECORDS every read +// range. The buffer is mutable so a test can corrupt specific on-disk bytes +// before re-opening. read_batch is overridden so batched reads are recorded too +// (open() currently uses only read_at, but recording both keeps the assertion +// honest if that ever changes). +class RecordingFileReader : public io::FileReader { +public: + explicit RecordingFileReader(std::vector bytes) : bytes_(std::move(bytes)) {} + + Status read_at(uint64_t offset, size_t len, std::vector* out) override { + reads_.push_back(io::Range {offset, len}); + if (offset > bytes_.size() || len > bytes_.size() - offset) { + return Status::Error( + "recording reader: read past EOF"); + } + out->assign(bytes_.begin() + static_cast(offset), + bytes_.begin() + static_cast(offset + len)); + return Status::OK(); + } + + Status read_batch(const std::vector& ranges, + std::vector>* outs) override { + outs->resize(ranges.size()); + for (size_t i = 0; i < ranges.size(); ++i) { + RETURN_IF_ERROR(read_at(ranges[i].offset, ranges[i].len, &(*outs)[i])); + } + return Status::OK(); + } + + uint64_t size() const override { return bytes_.size(); } + + const std::vector& reads() const { return reads_; } + std::vector& bytes() { return bytes_; } + + // True iff any recorded read overlaps [lo, hi). + bool any_read_intersects(uint64_t lo, uint64_t hi) const { + for (const auto& r : reads_) { + const uint64_t r_lo = r.offset; + const uint64_t r_hi = r.offset + r.len; + if (r_lo < hi && lo < r_hi) { + return true; + } + } + return false; + } + +private: + std::vector bytes_; + std::vector reads_; +}; + +// Writes a minimal single-index docs+positions container and returns its bytes. +std::vector BuildContainerBytes() { + SpimiTermBuffer buf(/*has_positions=*/true); + // A tiny deterministic corpus: a couple of terms across a few docs. + const char* docs[] = {"alpha bravo", "bravo charlie", "alpha charlie delta"}; + for (uint32_t d = 0; d < 3; ++d) { + std::string s = docs[d]; + uint32_t pos = 0; + size_t start = 0; + while (start <= s.size()) { + size_t sp = s.find(' ', start); + std::string tok = + s.substr(start, sp == std::string::npos ? std::string::npos : sp - start); + if (!tok.empty()) { + buf.add_token(tok, d, pos++); + } + if (sp == std::string::npos) { + break; + } + start = sp + 1; + } + } + std::vector terms = buf.finalize_sorted(); + + SniiIndexInput in; + in.index_id = 1; + in.index_suffix = "body"; + in.config = IndexConfig::kDocsPositions; + in.doc_count = 3; + in.terms = std::move(terms); + in.target_dict_block_bytes = 256; + + const std::string path = TempPath(); + { + io::LocalFileWriter w; + EXPECT_TRUE(w.open(path).ok()); + SniiCompoundWriter cw(&w); + EXPECT_TRUE(cw.add_logical_index(in).ok()); + EXPECT_TRUE(cw.finish().ok()); + } + + io::LocalFileReader r; + EXPECT_TRUE(r.open(path).ok()); + std::vector bytes; + EXPECT_TRUE(r.read_at(0, r.size(), &bytes).ok()); + std::remove(path.c_str()); + return bytes; +} + +} // namespace + +// open() must succeed and must NOT read any byte in [0, kBootstrapHeaderSize). +TEST(SniiSegmentReaderOpen, IssuesNoReadAtBootstrapRegion) { + std::vector bytes = BuildContainerBytes(); + ASSERT_GT(bytes.size(), kBootstrapHeaderSize + tail_pointer_size()); + + RecordingFileReader reader(std::move(bytes)); + SniiSegmentReader seg; + ASSERT_TRUE(SniiSegmentReader::open(&reader, &seg).ok()); + EXPECT_EQ(seg.n_logical_indexes(), 1U); + + // The container must still be usable (real, not vacuous): the logical index + // opens and reports the corpus doc count. + LogicalIndexReader idx; + ASSERT_TRUE(seg.open_index(1, "body", &idx).ok()); + EXPECT_EQ(idx.stats().doc_count, 3U); + + // Core assertion: open() + the read_index_meta above touched the tail, never + // the bootstrap header at the front of the file. + EXPECT_FALSE(reader.any_read_intersects(0, kBootstrapHeaderSize)) + << "open path read the offset-0 bootstrap region"; + // And it issued at least one read (otherwise the assertion above is vacuous). + EXPECT_GE(reader.reads().size(), 1U); +} + +// A corrupt offset-0 bootstrap header no longer fails open(): nothing reads it. +TEST(SniiSegmentReaderOpen, IgnoresCorruptBootstrapHeader) { + std::vector bytes = BuildContainerBytes(); + ASSERT_GE(bytes.size(), kBootstrapHeaderSize); + + RecordingFileReader reader(std::move(bytes)); + // Smash the entire bootstrap header region. + for (uint32_t i = 0; i < kBootstrapHeaderSize; ++i) { + reader.bytes()[i] = 0xFFu; + } + + SniiSegmentReader seg; + EXPECT_TRUE(SniiSegmentReader::open(&reader, &seg).ok()) + << "open() must not depend on the offset-0 bootstrap header"; +} + +// The container version gate is preserved by the tail pointer: corrupting the +// tail pointer's format_version makes open() fail. +TEST(SniiSegmentReaderOpen, RejectsCorruptTailFormatVersion) { + std::vector good = BuildContainerBytes(); + ASSERT_GE(good.size(), tail_pointer_size()); + + // Sanity: the unmodified container opens. + { + RecordingFileReader ok_reader(good); + SniiSegmentReader seg; + ASSERT_TRUE(SniiSegmentReader::open(&ok_reader, &seg).ok()); + } + + // The tail pointer is the last tail_pointer_size() bytes. Its layout is + // u32 magic, u16 format_version, ... so format_version sits at + // (size - tail_pointer_size) + 4. + RecordingFileReader bad_reader(good); + const uint64_t tp_start = bad_reader.size() - tail_pointer_size(); + const uint64_t fv_off = tp_start + 4; // skip the u32 magic + // Write a wrong, never-valid format_version (kFormatVersion is small). + bad_reader.bytes()[fv_off] = 0xFFu; + bad_reader.bytes()[fv_off + 1] = 0xFFu; + + SniiSegmentReader seg; + EXPECT_FALSE(SniiSegmentReader::open(&bad_reader, &seg).ok()) + << "open() must reject a container whose tail format_version is wrong"; +} diff --git a/be/test/storage/index/snii/snii_bigram_defer_e2e_test.cpp b/be/test/storage/index/snii/snii_bigram_defer_e2e_test.cpp new file mode 100644 index 00000000000000..4e12a4b1fb74e6 --- /dev/null +++ b/be/test/storage/index/snii/snii_bigram_defer_e2e_test.cpp @@ -0,0 +1,344 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// SNII bigram-defer END-TO-END wiring tests (see +// config::snii_bigram_defer_build_to_compaction): real tablet, real rowset +// writers, real compaction -- the full `_opts.write_type == TYPE_DIRECT` -> +// `opts.is_direct_load` -> `set_direct_load(true)` chain that the writer-level +// tests cannot see. Written segments are probed with the raw SNII segment +// reader (sentinel / pair / unigram dict lookups), so the assertions pin the +// WRITE-side wiring without depending on the query read path: +// 1. Direct-load rowsets defer: no bigram sentinel, no pair terms -- through +// BOTH production segment writers (VerticalSegmentWriter, the +// enable_vertical_segment_writer default, AND the horizontal +// SegmentWriter), covering both `write_type == TYPE_DIRECT` assignments. +// 2. Compaction rebuilds: the FULL-compaction output segment (written with +// DataWriteType::TYPE_COMPACTION) carries the sentinel AND materializes +// the high-df pair again -- the "compaction re-feeds every token" premise +// the whole deferral rests on. An inverted wiring (`!=`, deferring +// compaction forever) fails here. +// 3. With the config OFF a direct load still full-builds (guards an +// accidentally-always-on deferral and proves the probes see sentinels on +// exactly this path). +// 4. Variant subcolumn index writers inherit the marking through +// `opts.is_direct_load` propagation in variant_column_writer_impl.cpp: +// a direct-load variant rowset's subcolumn index defers, a config-off one +// full-builds. + +#include + +#include +#include +#include +#include + +#include "common/config.h" +#include "common/status.h" +#include "storage/index/inverted/inverted_index_desc.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/olap_define.h" +#include "storage/rowset/rowset.h" +#include "testutil/index_storage_test_util.h" + +namespace doris::index_storage_test { +namespace { + +constexpr int32_t kTextColumnUid = 2; +constexpr int64_t kTextIndexId = 230001; +constexpr int32_t kVariantColumnUid = 2; +constexpr int64_t kVariantIndexId = 230002; + +// What the raw SNII dict of one segment index says about the bigram build, +// plus the resident kPhraseBigramsDeferred capability flag: the flag and the +// postings must stay COHERENT across the segment lifecycle (deferred load: +// flag set + no sentinel; compacted / full build: flag clear + sentinel). A +// stale flag on a compacted segment would silently disable the bigram fast +// path forever with correct results -- only this assertion catches it. +struct SniiBigramProbe { + bool sentinel = false; + bool pair = false; + bool hello = false; + bool deferred_flag = false; +}; + +std::map phrase_index_properties() { + return {{"parser", "english"}, {"lower_case", "true"}, {"support_phrase", "true"}}; +} + +// 40 rows with the adjacent (hello, world) pair + 40 without: after compacting +// two such rowsets the pair's df is 80, inside the default prune survival +// window [auto floor 64, 2 x floor 128] for the 160-doc segment -- so the +// compacted full build PROVABLY materializes the pair (each 80-doc load +// segment alone stays under the floor, which is fine: deferred segments feed +// no pairs at all and the control test only asserts the sentinel). +std::vector half_pair_rows(const std::string& tag) { + std::vector rows; + rows.reserve(80); + for (uint32_t i = 0; i < 40; ++i) { + rows.push_back("hello world " + tag + std::to_string(i)); + } + for (uint32_t i = 0; i < 40; ++i) { + rows.push_back("alpha beta " + tag + std::to_string(i)); + } + return rows; +} + +class SniiBigramDeferE2eTest : public IndexStorageTestFixture { +protected: + void SetUp() override { + IndexStorageTestFixture::SetUp(); + _saved_defer = config::snii_bigram_defer_build_to_compaction; + _saved_vertical_writer = config::enable_vertical_segment_writer; + } + + void TearDown() override { + config::snii_bigram_defer_build_to_compaction = _saved_defer; + config::enable_vertical_segment_writer = _saved_vertical_writer; + IndexStorageTestFixture::TearDown(); + } + + // The single-file SNII index sits next to the segment file: + // {tablet_path}/{rowset_id}_{seg_id}.idx -- the same path derivation the + // production writers used (get_index_file_path_prefix + _v2). + std::string segment_index_path(const RowsetSharedPtr& rowset, int seg_id) const { + const std::string seg_path = local_segment_path(tablet()->tablet_path(), + rowset->rowset_id().to_string(), seg_id); + return segment_v2::InvertedIndexDescriptor::get_index_file_path_v2( + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix(seg_path)); + } + + // Opens segment 0's logical index (index_id, suffix) with the raw SNII + // reader and probes the dict for the bigram sentinel, the (hello, world) + // pair, and the "hello" unigram. On any open failure the all-false probe is + // returned with the failure recorded -- callers assert probe.hello == true + // on every path, so a missing file / wrong suffix can never pass as "the + // sentinel is absent". + SniiBigramProbe probe_segment_index(const RowsetSharedPtr& rowset, int64_t index_id, + const std::string& suffix) const { + SniiBigramProbe probe; + ::doris::snii::io::LocalFileReader file; + ::doris::snii::reader::SniiSegmentReader segment; + ::doris::snii::reader::LogicalIndexReader idx; + const std::string path = segment_index_path(rowset, 0); + if (Status st = file.open(path); !st.ok()) { + ADD_FAILURE() << "open " << path << ": " << st.to_string(); + return probe; + } + if (Status st = ::doris::snii::reader::SniiSegmentReader::open(&file, &segment); !st.ok()) { + ADD_FAILURE() << "segment open " << path << ": " << st.to_string(); + return probe; + } + if (Status st = segment.open_index(static_cast(index_id), suffix, &idx); + !st.ok()) { + ADD_FAILURE() << "index (" << index_id << ", '" << suffix << "') in " << path << ": " + << st.to_string(); + return probe; + } + probe.sentinel = + lookup_found(idx, ::doris::snii::format::make_phrase_bigram_sentinel_term()); + probe.pair = + lookup_found(idx, ::doris::snii::format::make_phrase_bigram_term("hello", "world")); + probe.hello = lookup_found(idx, "hello"); + probe.deferred_flag = idx.phrase_bigrams_deferred(); + return probe; + } + +private: + static bool lookup_found(const ::doris::snii::reader::LogicalIndexReader& idx, + const std::string& term) { + bool found = false; + ::doris::snii::format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + EXPECT_TRUE(idx.lookup(term, &found, &entry, &frq_base, &prx_base).ok()); + return found; + } + + bool _saved_defer = false; + bool _saved_vertical_writer = true; +}; + +// The core wiring round trip: two direct-load rowsets (one through the default +// VerticalSegmentWriter, one through the horizontal SegmentWriter) defer the +// bigram build; FULL compaction of the pair -- whose output segment is written +// under DataWriteType::TYPE_COMPACTION -- rebuilds sentinel and pairs. +TEST_F(SniiBigramDeferE2eTest, DirectLoadDefersBigramAndFullCompactionRebuilds) { + config::snii_bigram_defer_build_to_compaction = true; + + IndexTabletOptions options; + options.tablet_id = 130001; + options.index_storage_format = InvertedIndexStorageFormatPB::SNII; + options.text_columns = {TextColumnSpec {.unique_id = kTextColumnUid, .name = "title"}}; + options.inverted_indexes.push_back(IndexSpec::column_index( + kTextIndexId, "idx_title", kTextColumnUid, phrase_index_properties())); + ASSERT_TRUE(create_tablet(options).ok()); + + // Rowset v0: FORCE the vertical writer (do not trust the ambient default: + // if the environment started with it false, both rowsets would take the + // horizontal path and the VerticalSegmentWriter assignment went untested). + config::enable_vertical_segment_writer = true; + IndexRowsetSpec rowset0; + rowset0.version = 0; + rowset0.batches.push_back(IndexBatch::single_text(half_pair_rows("wiki"), 0)); + auto rowset0_result = write_rowset(rowset0); + ASSERT_TRUE(rowset0_result.has_value()) << rowset0_result.error(); + + // Rowset v1: the horizontal SegmentWriter path (segment_writer.cpp + // assignment) -- some deployments run with the vertical writer disabled. + config::enable_vertical_segment_writer = false; + IndexRowsetSpec rowset1; + rowset1.version = 1; + rowset1.batches.push_back(IndexBatch::single_text(half_pair_rows("doc"), 100)); + auto rowset1_result = write_rowset(rowset1); + config::enable_vertical_segment_writer = true; + ASSERT_TRUE(rowset1_result.has_value()) << rowset1_result.error(); + + // Both load segments deferred: no sentinel, no pair, unigrams intact, and + // the resident capability flag SET (flag/postings coherence, load side). + for (const auto& rowset : {rowset0_result.value(), rowset1_result.value()}) { + ASSERT_EQ(rowset->num_segments(), 1); + const SniiBigramProbe probe = probe_segment_index(rowset, kTextIndexId, ""); + EXPECT_FALSE(probe.sentinel) << rowset->rowset_id().to_string(); + EXPECT_FALSE(probe.pair) << rowset->rowset_id().to_string(); + EXPECT_TRUE(probe.hello) << rowset->rowset_id().to_string(); + EXPECT_TRUE(probe.deferred_flag) << rowset->rowset_id().to_string(); + } + + // FULL compaction re-feeds every token under TYPE_COMPACTION: the output + // segment must carry the sentinel again AND materialize the now-df-80 + // pair -- the deferred work actually got done, not just re-declared. + auto compacted = compact_rowsets(IndexCompactionKind::FULL, + {rowset0_result.value(), rowset1_result.value()}); + ASSERT_TRUE(compacted.has_value()) << compacted.error(); + ASSERT_NE(compacted.value(), nullptr); + ASSERT_EQ(compacted.value()->num_rows(), 160); + ASSERT_EQ(compacted.value()->num_segments(), 1); + const SniiBigramProbe compacted_probe = + probe_segment_index(compacted.value(), kTextIndexId, ""); + EXPECT_TRUE(compacted_probe.sentinel); + EXPECT_TRUE(compacted_probe.pair); + EXPECT_TRUE(compacted_probe.hello); + // Flag/postings coherence, compaction side: a stale deferred flag next to + // rebuilt pairs would permanently disable the bigram fast path (correct + // results, invisible perf regression) -- it must be CLEARED here. + EXPECT_FALSE(compacted_probe.deferred_flag); +} + +// Config off: a direct load keeps the full bigram build (sentinel present). +// Guards an accidentally-always-on deferral and proves the probe machinery +// sees sentinels on exactly the load path the test above asserts them absent. +TEST_F(SniiBigramDeferE2eTest, ConfigOffDirectLoadKeepsFullBigramBuild) { + config::snii_bigram_defer_build_to_compaction = false; + + IndexTabletOptions options; + options.tablet_id = 130002; + options.index_storage_format = InvertedIndexStorageFormatPB::SNII; + options.text_columns = {TextColumnSpec {.unique_id = kTextColumnUid, .name = "title"}}; + options.inverted_indexes.push_back(IndexSpec::column_index( + kTextIndexId, "idx_title", kTextColumnUid, phrase_index_properties())); + ASSERT_TRUE(create_tablet(options).ok()); + + IndexRowsetSpec rowset0; + rowset0.version = 0; + rowset0.batches.push_back(IndexBatch::single_text(half_pair_rows("base"), 0)); + auto rowset_result = write_rowset(rowset0); + ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error(); + ASSERT_EQ(rowset_result.value()->num_segments(), 1); + + const SniiBigramProbe probe = probe_segment_index(rowset_result.value(), kTextIndexId, ""); + EXPECT_TRUE(probe.sentinel); + EXPECT_TRUE(probe.hello); + EXPECT_FALSE(probe.deferred_flag); +} + +// Variant subcolumn wiring: the direct-load marking reaches subcolumn index +// writers only through the `opts.is_direct_load` propagation in +// variant_column_writer_impl.cpp -- a dropped copy there loses the deferral +// for every variant load while nothing else fails. The predefined path +// "content" carries a field-pattern phrase index; its logical index is keyed +// by the escaped full path ("v.content" with '.' as %2E). +TEST_F(SniiBigramDeferE2eTest, VariantSubcolumnIndexFollowsDirectLoadDeferral) { + config::snii_bigram_defer_build_to_compaction = true; + + VariantColumnSpec variant; + variant.unique_id = kVariantColumnUid; + variant.name = "v"; + variant.predefined_paths = {VariantPathSpec {.path = "content", + .type = FieldType::OLAP_FIELD_TYPE_STRING, + .nullable = true, + .pattern_type = PatternTypePB::MATCH_NAME, + .array_item_type = {}, + .array_item_nullable = true}}; + IndexSpec index_spec = IndexSpec::field_pattern_index(kVariantIndexId, "idx_v_content", + kVariantColumnUid, "content"); + for (const auto& [key, value] : phrase_index_properties()) { + index_spec.properties[key] = value; + } + + IndexTabletOptions options; + options.tablet_id = 130003; + options.index_storage_format = InvertedIndexStorageFormatPB::SNII; + options.variant_columns = {variant}; + options.inverted_indexes.push_back(index_spec); + ASSERT_TRUE(create_tablet(options).ok()); + + std::vector jsons; + jsons.reserve(80); + for (uint32_t i = 0; i < 80; ++i) { + jsons.push_back(R"({"content": "hello world item)" + std::to_string(i) + R"("})"); + } + + // Escaped-by-production-rules suffix of the subcolumn's logical index: + // escape_for_path_name("v.content") -- '.' escapes to %2E. + const std::string suffix = "v%2Econtent"; + + // Direct load with the config on: the subcolumn index defers. + IndexRowsetSpec rowset0; + rowset0.version = 0; + rowset0.batches.push_back(IndexBatch::single_variant(jsons, 0)); + auto defer_rowset = write_rowset(rowset0); + ASSERT_TRUE(defer_rowset.has_value()) << defer_rowset.error(); + ASSERT_EQ(defer_rowset.value()->num_segments(), 1); + const SniiBigramProbe defer_probe = + probe_segment_index(defer_rowset.value(), kVariantIndexId, suffix); + EXPECT_FALSE(defer_probe.sentinel); + EXPECT_FALSE(defer_probe.pair); + EXPECT_TRUE(defer_probe.hello); + EXPECT_TRUE(defer_probe.deferred_flag); + + // Config off, same data: the subcolumn index full-builds (positive control + // for the probe and for the propagation's false direction). + config::snii_bigram_defer_build_to_compaction = false; + IndexRowsetSpec rowset1; + rowset1.version = 1; + rowset1.batches.push_back(IndexBatch::single_variant(jsons, 100)); + auto full_rowset = write_rowset(rowset1); + ASSERT_TRUE(full_rowset.has_value()) << full_rowset.error(); + ASSERT_EQ(full_rowset.value()->num_segments(), 1); + const SniiBigramProbe full_probe = + probe_segment_index(full_rowset.value(), kVariantIndexId, suffix); + EXPECT_TRUE(full_probe.sentinel); + EXPECT_TRUE(full_probe.hello); + EXPECT_FALSE(full_probe.deferred_flag); +} + +} // namespace +} // namespace doris::index_storage_test diff --git a/be/test/storage/index/snii/snii_test_env.cpp b/be/test/storage/index/snii/snii_test_env.cpp new file mode 100644 index 00000000000000..c1b8de6728f762 --- /dev/null +++ b/be/test/storage/index/snii/snii_test_env.cpp @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include + +#include "io/fs/local_file_system.h" +#include "runtime/exec_env.h" +#include "storage/index/index_writer.h" // segment_v2::TmpFileDirs +#include "storage/options.h" // StorePath + +namespace doris { +namespace { + +// Initializes ExecEnv's tmp_file_dirs ONCE for the whole doris_be_test binary so the +// SNII writer's resolve_temp_dir() (ExecEnv::get_tmp_file_dirs()->get_tmp_file_dir()) +// works in unit tests, which otherwise leave tmp_file_dirs null. Mirrors the pattern +// used by ann_index_writer_test / segcompaction_test. +class SniiTmpDirEnvironment : public ::testing::Environment { +public: + void SetUp() override { + if (ExecEnv::GetInstance()->get_tmp_file_dirs() != nullptr) { + return; // another test environment already configured it + } + const std::string tmp_dir = "./ut_dir/snii_tmp"; + static_cast(io::global_local_filesystem()->delete_directory(tmp_dir)); + static_cast(io::global_local_filesystem()->create_directory(tmp_dir)); + std::vector paths; + paths.emplace_back(tmp_dir, -1); + auto dirs = std::make_unique(paths); + static_cast(dirs->init()); + ExecEnv::GetInstance()->set_tmp_file_dir(std::move(dirs)); + } +}; + +[[maybe_unused]] ::testing::Environment* const g_snii_tmp_dir_env = + ::testing::AddGlobalTestEnvironment(new SniiTmpDirEnvironment); + +} // namespace +} // namespace doris diff --git a/be/test/storage/index/snii/writer/bigram_prune_writer_test.cpp b/be/test/storage/index/snii/writer/bigram_prune_writer_test.cpp new file mode 100644 index 00000000000000..bd83c347c3a1c9 --- /dev/null +++ b/be/test/storage/index/snii/writer/bigram_prune_writer_test.cpp @@ -0,0 +1,535 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/status.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/query/internal/docid_posting_reader.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/reader/windowed_posting.h" +#include "storage/index/snii/writer/logical_index_writer.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii_query_test_util.h" + +// G01 part B writer gate: threshold-boundary DETERMINISM of the flush-time bigram +// df-prune, observed through the always-on writer seams +// (bigram_terms_materialized / bigram_terms_pruned), the dict (lookup hit/miss), +// the docs-only survivor layout (prx_len == 0), and the per-index meta threshold +// declaration. df == threshold-1 must prune; df == threshold must materialize -- +// exactly, every build. +using namespace doris::snii; +using namespace doris::snii::snii_test; +namespace wtesting = doris::snii::writer::testing; + +namespace { + +constexpr uint32_t kDocCount = 100; +constexpr uint32_t kThreshold = 4; + +std::vector one_position_docs(std::vector docids) { + std::vector docs; + docs.reserve(docids.size()); + for (uint32_t docid : docids) { + docs.push_back({docid, {0}}); + } + return docs; +} + +// A tiny kDocsPositions index: two unigrams, the sentinel, and two bigram terms +// straddling the threshold -- (aa,bb) at df == kThreshold - 1 (prune side) and +// (bb,cc) at df == kThreshold (materialize side). +Status build_boundary_index(MemoryFile* file, uint32_t bigram_prune_min_df, + reader::SniiSegmentReader* segment_reader, + reader::LogicalIndexReader* index_reader, + uint64_t bigram_prune_max_df = 0) { + writer::SniiIndexInput input; + input.index_id = 3; + input.index_suffix = "Body"; + input.config = format::IndexConfig::kDocsPositions; + input.doc_count = kDocCount; + input.bigram_prune_min_df = bigram_prune_min_df; + input.bigram_prune_max_df = bigram_prune_max_df; + input.terms = { + make_term("aa", one_position_docs({1, 2, 3})), + make_term("bb", one_position_docs({1, 2, 3, 4, 5, 6})), + make_term(format::make_phrase_bigram_sentinel_term(), one_position_docs({0})), + make_term(format::make_phrase_bigram_term("aa", "bb"), + one_position_docs({1, 2, 3})), // df == kThreshold - 1 + make_term(format::make_phrase_bigram_term("bb", "cc"), + one_position_docs({4, 5, 6, 7})), // df == kThreshold + }; + std::ranges::sort(input.terms, + [](const writer::TermPostings& lhs, const writer::TermPostings& rhs) { + return lhs.term < rhs.term; + }); + + writer::SniiCompoundWriter writer(file); + RETURN_IF_ERROR(writer.add_logical_index(input)); + RETURN_IF_ERROR(writer.finish()); + RETURN_IF_ERROR(reader::SniiSegmentReader::open(file, segment_reader)); + return segment_reader->open_index(input.index_id, input.index_suffix, index_reader); +} + +bool lookup_term(const reader::LogicalIndexReader& idx, const std::string& term, + format::DictEntry* entry) { + bool found = false; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + assert_ok(idx.lookup(term, &found, entry, &frq_base, &prx_base)); + return found; +} + +} // namespace + +TEST(SniiBigramPruneWriter, ThresholdBoundaryIsDeterministic) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + + wtesting::reset_bigram_prune_counters(); + assert_ok(build_boundary_index(&file, kThreshold, &segment_reader, &index_reader)); + + // Exactly one prune decision on each side of the boundary; the sentinel is + // counted by NEITHER seam (it is never prunable). + EXPECT_EQ(wtesting::bigram_terms_pruned(), 1U); // (aa,bb) df 3 < 4 + EXPECT_EQ(wtesting::bigram_terms_materialized(), 1U); // (bb,cc) df 4 >= 4 + + // Dict state matches the seams: the pruned bigram has NO entry, the survivor + // and the sentinel do. + format::DictEntry entry; + EXPECT_FALSE(lookup_term(index_reader, format::make_phrase_bigram_term("aa", "bb"), &entry)); + ASSERT_TRUE(lookup_term(index_reader, format::make_phrase_bigram_term("bb", "cc"), &entry)); + EXPECT_EQ(entry.df, kThreshold); + // Docs-only survivor: no positions anywhere (inline prx bytes empty, no span). + EXPECT_EQ(entry.prx_len, 0U); + EXPECT_TRUE(entry.prx_bytes.empty()); + ASSERT_TRUE(lookup_term(index_reader, format::make_phrase_bigram_sentinel_term(), &entry)); + + // Unigrams are untouched by the diet: positions still written. + ASSERT_TRUE(lookup_term(index_reader, "aa", &entry)); + EXPECT_TRUE(entry.prx_len > 0 || !entry.prx_bytes.empty()); + + // The applied threshold is declared in the per-index meta. + EXPECT_EQ(index_reader.bigram_prune_min_df(), kThreshold); +} + +TEST(SniiBigramPruneWriter, MaxDfUpperBoundaryIsDeterministic) { + // G15 upper gate, min gate off (0) to isolate it: df == max must + // materialize (the gate drops strictly-above), df == max + 1 must be + // pruned -- exactly, every build -- and the meta must declare the applied + // max so the reader still falls back on the resulting dict miss. + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + + wtesting::reset_bigram_prune_counters(); + assert_ok(build_boundary_index(&file, /*bigram_prune_min_df=*/0, &segment_reader, &index_reader, + /*bigram_prune_max_df=*/3)); + + // (aa,bb) df 3 == max -> materialize; (bb,cc) df 4 > max -> pruned. The + // upper drop is counted by its OWN seam (never by the min-gate one); the + // sentinel (df 1) counts nowhere, as always. + EXPECT_EQ(wtesting::bigram_terms_max_pruned(), 1U); + EXPECT_EQ(wtesting::bigram_terms_pruned(), 0U); + EXPECT_EQ(wtesting::bigram_terms_materialized(), 1U); + + format::DictEntry entry; + ASSERT_TRUE(lookup_term(index_reader, format::make_phrase_bigram_term("aa", "bb"), &entry)); + EXPECT_EQ(entry.df, 3U); + EXPECT_FALSE(lookup_term(index_reader, format::make_phrase_bigram_term("bb", "cc"), &entry)); + ASSERT_TRUE(lookup_term(index_reader, format::make_phrase_bigram_sentinel_term(), &entry)); + + // Max-only declaration: min stays 0, max lands in the meta (this is what + // keeps the reader's dict-miss fallback armed without the min gate). + EXPECT_EQ(index_reader.bigram_prune_min_df(), 0U); + EXPECT_EQ(index_reader.bigram_prune_max_df(), 3U); +} + +TEST(SniiBigramPruneWriter, MinAndMaxGatesTogetherKeepOnlyMiddleBand) { + // Both gates armed at once (min == 4, max == 6): the df axis must split into + // exactly three bands -- df < 4 min-pruned, 4 <= df <= 6 materialized + // (BOTH boundaries inclusive-keep), df > 6 max-pruned -- each drop counted + // by its OWN seam, and the meta declaring BOTH applied thresholds. + constexpr uint32_t kMin = 4; + constexpr uint64_t kMax = 6; + writer::SniiIndexInput input; + input.index_id = 3; + input.index_suffix = "Body"; + input.config = format::IndexConfig::kDocsPositions; + input.doc_count = kDocCount; + input.bigram_prune_min_df = kMin; + input.bigram_prune_max_df = kMax; + input.terms = { + make_term("aa", one_position_docs({1, 2, 3})), + make_term(format::make_phrase_bigram_sentinel_term(), one_position_docs({0})), + make_term(format::make_phrase_bigram_term("aa", "bb"), + one_position_docs({1, 2, 3})), // df 3 < min: LOW tail, pruned + make_term(format::make_phrase_bigram_term("bb", "cc"), + one_position_docs({1, 2, 3, 4})), // df 4 == min: kept + make_term(format::make_phrase_bigram_term("cc", "dd"), + one_position_docs({1, 2, 3, 4, 5, 6})), // df 6 == max: kept + make_term(format::make_phrase_bigram_term("dd", "ee"), + one_position_docs({1, 2, 3, 4, 5, 6, 7})), // df 7 > max: HIGH tail, pruned + }; + std::ranges::sort(input.terms, + [](const writer::TermPostings& lhs, const writer::TermPostings& rhs) { + return lhs.term < rhs.term; + }); + + MemoryFile file; + wtesting::reset_bigram_prune_counters(); + writer::SniiCompoundWriter writer(&file); + assert_ok(writer.add_logical_index(input)); + assert_ok(writer.finish()); + + EXPECT_EQ(wtesting::bigram_terms_pruned(), 1U); // (aa,bb) via the MIN gate + EXPECT_EQ(wtesting::bigram_terms_max_pruned(), 1U); // (dd,ee) via the MAX gate + EXPECT_EQ(wtesting::bigram_terms_materialized(), 2U); // the middle band + + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(reader::SniiSegmentReader::open(&file, &segment_reader)); + assert_ok(segment_reader.open_index(input.index_id, input.index_suffix, &index_reader)); + + format::DictEntry entry; + EXPECT_FALSE(lookup_term(index_reader, format::make_phrase_bigram_term("aa", "bb"), &entry)); + ASSERT_TRUE(lookup_term(index_reader, format::make_phrase_bigram_term("bb", "cc"), &entry)); + EXPECT_EQ(entry.df, kMin); + ASSERT_TRUE(lookup_term(index_reader, format::make_phrase_bigram_term("cc", "dd"), &entry)); + EXPECT_EQ(entry.df, kMax); + EXPECT_FALSE(lookup_term(index_reader, format::make_phrase_bigram_term("dd", "ee"), &entry)); + ASSERT_TRUE(lookup_term(index_reader, format::make_phrase_bigram_sentinel_term(), &entry)); + + EXPECT_EQ(index_reader.bigram_prune_min_df(), kMin); + EXPECT_EQ(index_reader.bigram_prune_max_df(), kMax); +} + +TEST(SniiBigramPruneWriter, ThresholdZeroKeepsLegacyLayout) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + + wtesting::reset_bigram_prune_counters(); + assert_ok( + build_boundary_index(&file, /*bigram_prune_min_df=*/0, &segment_reader, &index_reader)); + + // No pruning; both bigrams materialize (the seam counts them either way). + EXPECT_EQ(wtesting::bigram_terms_pruned(), 0U); + EXPECT_EQ(wtesting::bigram_terms_materialized(), 2U); + + // Legacy layout: every bigram present WITH positions; meta declares nothing. + format::DictEntry entry; + ASSERT_TRUE(lookup_term(index_reader, format::make_phrase_bigram_term("aa", "bb"), &entry)); + EXPECT_TRUE(entry.prx_len > 0 || !entry.prx_bytes.empty()); + ASSERT_TRUE(lookup_term(index_reader, format::make_phrase_bigram_term("bb", "cc"), &entry)); + EXPECT_TRUE(entry.prx_len > 0 || !entry.prx_bytes.empty()); + EXPECT_EQ(index_reader.bigram_prune_min_df(), 0U); + EXPECT_EQ(index_reader.bigram_prune_max_df(), 0U); +} + +TEST(SniiBigramPruneWriter, DocsOnlyConfigForcesThresholdOff) { + // A non-positional index never emits bigrams; a (mis)configured threshold must + // not leak into its meta (the reader would otherwise take a pointless + // fallback branch for every 2-term phrase-shaped lookup). + MemoryFile file; + writer::SniiIndexInput input; + input.index_id = 4; + input.index_suffix = "Tag"; + input.config = format::IndexConfig::kDocsOnly; + input.doc_count = kDocCount; + input.bigram_prune_min_df = kThreshold; + input.bigram_prune_max_df = kThreshold; + input.terms = {make_term("aa", one_position_docs({1, 2, 3}))}; + + writer::SniiCompoundWriter writer(&file); + assert_ok(writer.add_logical_index(input)); + assert_ok(writer.finish()); + + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(reader::SniiSegmentReader::open(&file, &segment_reader)); + assert_ok(segment_reader.open_index(input.index_id, input.index_suffix, &index_reader)); + EXPECT_EQ(index_reader.bigram_prune_min_df(), 0U); + EXPECT_EQ(index_reader.bigram_prune_max_df(), 0U); +} + +TEST(SniiBigramPruneWriter, DefaultThresholdFormula) { + // max(64, doc_count / 10000): the floor holds through 640k docs, then the + // 0.01% ratio takes over. + EXPECT_EQ(format::default_phrase_bigram_prune_min_df(0), 64U); + EXPECT_EQ(format::default_phrase_bigram_prune_min_df(9000), 64U); + EXPECT_EQ(format::default_phrase_bigram_prune_min_df(640000), 64U); + EXPECT_EQ(format::default_phrase_bigram_prune_min_df(649999), 64U); + EXPECT_EQ(format::default_phrase_bigram_prune_min_df(650000), 65U); + EXPECT_EQ(format::default_phrase_bigram_prune_min_df(10'000'000), 1000U); +} + +// G16: a prune-mode bigram whose df crosses the windowed threshold writes NO +// freq-block -- its frq span is exactly the docs-only prefix [prelude][dd-block] +// (frq_docs_len == frq_len), the prelude flags declare has_freq=false, and the +// per-window freq locators/crcs vanish with it. A same-df unigram keeps its +// freq suffix untouched. The docid-only read path (the ONLY consumer of pair +// postings) must decode the freq-less posting identically. Slim survivors are +// covered by the boundary tests above and KEEP freq (their DictEntry region +// metadata is tier-conditioned, not per-entry) -- the elision seam counts +// windowed entries only. +TEST(SniiBigramPruneWriter, WindowedSurvivorDropsFreqBlock) { + wtesting::reset_bigram_prune_counters(); + constexpr uint32_t kDf = format::kSlimDfThreshold + 300; // windowed on both terms + std::vector ids(kDf); + std::iota(ids.begin(), ids.end(), 1U); + + writer::SniiIndexInput input; + input.index_id = 3; + input.index_suffix = "Body"; + input.config = format::IndexConfig::kDocsPositions; + input.doc_count = 2 * kDf; + input.bigram_prune_min_df = kThreshold; // prune mode ON + input.terms = { + make_term("aa", one_position_docs(ids)), + make_term(format::make_phrase_bigram_sentinel_term(), one_position_docs({0})), + make_term(format::make_phrase_bigram_term("aa", "bb"), one_position_docs(ids)), + }; + std::ranges::sort(input.terms, + [](const writer::TermPostings& lhs, const writer::TermPostings& rhs) { + return lhs.term < rhs.term; + }); + + MemoryFile file; + writer::SniiCompoundWriter writer(&file); + assert_ok(writer.add_logical_index(input)); + assert_ok(writer.finish()); + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader idx; + assert_ok(reader::SniiSegmentReader::open(&file, &segment_reader)); + assert_ok(segment_reader.open_index(input.index_id, input.index_suffix, &idx)); + + bool found = false; + format::DictEntry bigram_entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + assert_ok(idx.lookup(format::make_phrase_bigram_term("aa", "bb"), &found, &bigram_entry, + &frq_base, &prx_base)); + ASSERT_TRUE(found); + EXPECT_TRUE(bigram_entry.has_sb); // windowed, not slim + EXPECT_EQ(bigram_entry.prx_len, 0U); // G01 diet + EXPECT_EQ(bigram_entry.frq_docs_len, bigram_entry.frq_len); // G16: no freq-block + EXPECT_EQ(wtesting::bigram_freqs_elided(), 1U); // sentinel is slim: no count + + format::DictEntry unigram_entry; + uint64_t uni_frq_base = 0; + uint64_t uni_prx_base = 0; + assert_ok(idx.lookup("aa", &found, &unigram_entry, &uni_frq_base, &uni_prx_base)); + ASSERT_TRUE(found); + EXPECT_TRUE(unigram_entry.has_sb); + EXPECT_LT(unigram_entry.frq_docs_len, unigram_entry.frq_len); // unigram keeps freq + + // G16-e: the docs-only bigram uses kBigramWindowDocs-sized windows -- at + // this df it collapses to ONE window, so its prelude is strictly smaller + // than the unigram's (4 x 256-doc adaptive windows at the same df). + EXPECT_LT(bigram_entry.prelude_len, unigram_entry.prelude_len); + + std::vector docids; + assert_ok(query::internal::read_docid_posting(idx, bigram_entry, frq_base, prx_base, &docids)); + ASSERT_EQ(docids.size(), kDf); + EXPECT_EQ(docids.front(), 1U); + EXPECT_EQ(docids.back(), kDf); + + // A want_freq read of the elided entry must fail with the SEMANTIC guard + // error, not a deep region-decode corruption. This also pins the prelude + // flags: had the writer declared has_freq=true over an empty freq-block, + // the guard would pass and the failure (if any) would read differently. + reader::DecodedPosting decoded; + Status st = + reader::read_windowed_posting(idx, bigram_entry, frq_base, prx_base, + /*want_positions=*/false, /*want_freq=*/true, &decoded); + ASSERT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("freqs requested but prelude has none"), std::string::npos) + << st.to_string(); +} + +// Legacy mode (threshold 0) must keep the pre-G01 layout for WINDOWED bigrams +// too: positions AND freq both present. Pins the write_freq predicate to the +// same min-df escape hatch as write_prx (the G16 review found no test asserted +// legacy freq presence). +TEST(SniiBigramPruneWriter, LegacyWindowedBigramKeepsFreqAndPrx) { + constexpr uint32_t kDf = format::kSlimDfThreshold + 300; + std::vector ids(kDf); + std::iota(ids.begin(), ids.end(), 1U); + + writer::SniiIndexInput input; + input.index_id = 3; + input.index_suffix = "Body"; + input.config = format::IndexConfig::kDocsPositions; + input.doc_count = 2 * kDf; + input.bigram_prune_min_df = 0; // legacy: no pruning, full layout + input.terms = { + make_term("aa", one_position_docs(ids)), + make_term(format::make_phrase_bigram_sentinel_term(), one_position_docs({0})), + make_term(format::make_phrase_bigram_term("aa", "bb"), one_position_docs(ids)), + }; + std::ranges::sort(input.terms, + [](const writer::TermPostings& lhs, const writer::TermPostings& rhs) { + return lhs.term < rhs.term; + }); + + MemoryFile file; + writer::SniiCompoundWriter writer(&file); + assert_ok(writer.add_logical_index(input)); + assert_ok(writer.finish()); + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader idx; + assert_ok(reader::SniiSegmentReader::open(&file, &segment_reader)); + assert_ok(segment_reader.open_index(input.index_id, input.index_suffix, &idx)); + + bool found = false; + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + assert_ok(idx.lookup(format::make_phrase_bigram_term("aa", "bb"), &found, &entry, &frq_base, + &prx_base)); + ASSERT_TRUE(found); + EXPECT_TRUE(entry.has_sb); + EXPECT_GT(entry.prx_len, 0U); // legacy keeps positions + EXPECT_LT(entry.frq_docs_len, entry.frq_len); // legacy keeps freq + + reader::DecodedPosting decoded; + assert_ok(reader::read_windowed_posting(idx, entry, frq_base, prx_base, + /*want_positions=*/true, /*want_freq=*/true, &decoded)); + ASSERT_EQ(decoded.docids.size(), kDf); + ASSERT_EQ(decoded.freqs.size(), kDf); + EXPECT_EQ(decoded.freqs.front(), 1U); +} + +// G16-c: write_freq == false drops the freq layout for UNIGRAMS too (freq +// serves only BM25 scoring, which the Doris adapter resolves as unreachable +// for plain positions indexes). Windowed unigrams self-describe via the +// prelude flags; slim pod / inline unigrams carry a zero-length freq region +// (value-driven: frq_docs_len == frq_len / frq_bytes == [dd]). Positions stay +// intact -- the phrase fallback (want_positions=true, want_freq=false) must +// decode identically. +TEST(SniiBigramPruneWriter, WriteFreqOffDropsUnigramFreqLayout) { + constexpr uint32_t kDf = format::kSlimDfThreshold + 300; // windowed + std::vector ids(kDf); + std::iota(ids.begin(), ids.end(), 1U); + + writer::SniiIndexInput input; + input.index_id = 3; + input.index_suffix = "Body"; + input.config = format::IndexConfig::kDocsPositions; + input.doc_count = 2 * kDf; + input.bigram_prune_min_df = kThreshold; + input.write_freq = false; // G16-c drop + input.terms = { + make_term("aa", one_position_docs(ids)), // windowed unigram + make_term("zz", one_position_docs({1, 2, 3, 4})), // slim unigram + make_term(format::make_phrase_bigram_sentinel_term(), one_position_docs({0})), + make_term(format::make_phrase_bigram_term("aa", "bb"), one_position_docs(ids)), + }; + std::ranges::sort(input.terms, + [](const writer::TermPostings& lhs, const writer::TermPostings& rhs) { + return lhs.term < rhs.term; + }); + + MemoryFile file; + writer::SniiCompoundWriter writer(&file); + assert_ok(writer.add_logical_index(input)); + assert_ok(writer.finish()); + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader idx; + assert_ok(reader::SniiSegmentReader::open(&file, &segment_reader)); + assert_ok(segment_reader.open_index(input.index_id, input.index_suffix, &idx)); + + bool found = false; + format::DictEntry uni; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + assert_ok(idx.lookup("aa", &found, &uni, &frq_base, &prx_base)); + ASSERT_TRUE(found); + EXPECT_TRUE(uni.has_sb); + EXPECT_EQ(uni.frq_docs_len, uni.frq_len); // no freq-block + EXPECT_GT(uni.prx_len, 0U); // positions kept + // G16-f: the freq-dropped index also elides the per-entry ttf/max_freq + // stats (block header kNoTermStats); df stays for the count fastpath. + EXPECT_EQ(uni.ttf_delta, 0U); + EXPECT_EQ(uni.max_freq, 0U); + EXPECT_EQ(uni.df, kDf); + + // Phrase-fallback-shaped read (positions without freq) decodes fully. + reader::DecodedPosting decoded; + assert_ok(reader::read_windowed_posting(idx, uni, frq_base, prx_base, + /*want_positions=*/true, /*want_freq=*/false, + &decoded)); + ASSERT_EQ(decoded.docids.size(), kDf); + ASSERT_EQ(decoded.positions.size(), kDf); + EXPECT_TRUE(decoded.freqs.empty()); + + // A scoring-shaped read fails with the semantic guard error. + Status st = + reader::read_windowed_posting(idx, uni, frq_base, prx_base, + /*want_positions=*/false, /*want_freq=*/true, &decoded); + ASSERT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("freqs requested but prelude has none"), std::string::npos); + + // Slim unigram: zero-length freq region, docs-only prefix == whole span. + format::DictEntry slim; + uint64_t sfrq = 0; + uint64_t sprx = 0; + assert_ok(idx.lookup("zz", &found, &slim, &sfrq, &sprx)); + ASSERT_TRUE(found); + EXPECT_FALSE(slim.has_sb); + if (slim.kind == format::DictEntryKind::kInline) { + EXPECT_EQ(slim.frq_bytes.size(), slim.inline_dd_disk_len); + } else { + EXPECT_EQ(slim.frq_docs_len, slim.frq_len); + } + std::vector slim_docids; + assert_ok(query::internal::read_docid_posting(idx, slim, sfrq, sprx, &slim_docids)); + EXPECT_EQ(slim_docids.size(), 4U); +} + +namespace doris::segment_v2 { +// Production freq policy resolver (index_file_writer.cpp) -- declared here so +// the UT drives the actual decision line the Doris adapter uses. +bool snii_effective_write_freq(doris::snii::format::IndexConfig index_config); +} // namespace doris::segment_v2 + +// G16-c: the adapter-level policy line. A scoring config ALWAYS keeps freq; a +// plain positions config follows the escape-hatch BE config (default false == +// drop). This is the only live control of the production freq layout, so pin +// all three states. +TEST(SniiBigramPruneWriter, EffectiveWriteFreqResolver) { + const bool saved = doris::config::snii_positions_index_write_freq; + doris::config::snii_positions_index_write_freq = false; + EXPECT_FALSE(doris::segment_v2::snii_effective_write_freq(format::IndexConfig::kDocsPositions)); + EXPECT_TRUE(doris::segment_v2::snii_effective_write_freq( + format::IndexConfig::kDocsPositionsScoring)); + doris::config::snii_positions_index_write_freq = true; + EXPECT_TRUE(doris::segment_v2::snii_effective_write_freq(format::IndexConfig::kDocsPositions)); + doris::config::snii_positions_index_write_freq = saved; +} diff --git a/be/test/storage/index/snii/writer/bigram_vocab_cap_test.cpp b/be/test/storage/index/snii/writer/bigram_vocab_cap_test.cpp new file mode 100644 index 00000000000000..8cb84d84a227ad --- /dev/null +++ b/be/test/storage/index/snii/writer/bigram_vocab_cap_test.cpp @@ -0,0 +1,380 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/logical_index_writer.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" +#include "storage/index/snii_query_test_util.h" + +// G04 "bigram diet" phase 2, buffer/writer level: +// (a) the vocab cap evicts ONLY current-df==1 bigram terms (hot pairs and the +// sentinel are untouchable) via the incremental sweep, keeping +// bigram_intern_bytes() bounded by the cap; +// (b) every survivor's postings EQUAL an uncapped control build; +// (c) an evicted-then-reappearing pair re-interns fine but lands in the +// ever-dropped bloom, so the flush drops it no matter how large its +// re-accumulated df grows (the completeness invariant); +// (d) position suppression: bigram tokens stop buffering positions entirely +// (in-memory and across the spill/merge round trip) while unigrams keep +// theirs. +using doris::Status; +using doris::snii::format::make_phrase_bigram_sentinel_term; +using doris::snii::format::make_phrase_bigram_term; +using doris::snii::writer::SpimiTermBuffer; +using doris::snii::writer::TermPostings; +using namespace doris::snii; +using namespace doris::snii::snii_test; +namespace spimi_testing = doris::snii::writer::testing; + +namespace { + +// Deterministic distinct alpha-only pair for filler bigrams. +std::pair filler_pair(uint32_t i) { + std::string l = "u"; + l += static_cast('a' + (i / 26) % 26); + l += static_cast('a' + i % 26); + l += static_cast('a' + (i / 676) % 26); + std::string r = "v"; + r += static_cast('a' + i % 26); + r += static_cast('a' + (i / 26) % 26); + r += static_cast('a' + (i / 676) % 26); + return {std::move(l), std::move(r)}; +} + +std::map drain_to_map(SpimiTermBuffer* buf) { + std::map out; + for (TermPostings& tp : buf->finalize_sorted()) { + std::string key = tp.term; + out.emplace(std::move(key), std::move(tp)); + } + return out; +} + +} // namespace + +// (a)+(b): eviction fires once the cap is crossed, takes ONLY df==1 bigrams, +// keeps the intern storage bounded, and every survivor equals the uncapped +// control byte-for-byte. +TEST(SniiBigramVocabCap, EvictsOnlyDfOneTailAndSurvivorsEqualControl) { + constexpr uint64_t kCap = 8 * 1024; + spimi_testing::reset_bigram_vocab_cap_counters(); + + SpimiTermBuffer capped(/*has_positions=*/true); + SpimiTermBuffer control(/*has_positions=*/true); + capped.configure_bigram_diet(kCap); + control.configure_bigram_diet(0); // diet (docs-only bigrams) but NO cap + ASSERT_TRUE(capped.status().ok()); + ASSERT_TRUE(control.status().ok()); + + auto feed_both = [&](std::string_view l, std::string_view r, uint32_t docid) { + capped.add_bigram_token(l, r, docid, 0); + control.add_bigram_token(l, r, docid, 0); + }; + + // Hot pair reaches df==2 before any cap pressure exists -> never evictable. + feed_both("hot", "pair", 0); + feed_both("hot", "pair", 1); + // 400 unique df==1 tail pairs blow the cap several times over. + for (uint32_t i = 0; i < 400; ++i) { + const auto [l, r] = filler_pair(i); + feed_both(l, r, 2 + i); + // The incremental sweep runs inside the add: the intern storage must + // stay bounded by the cap plus a small hover margin at every step. + EXPECT_LE(capped.bigram_intern_bytes(), kCap + 2048); + } + ASSERT_TRUE(capped.status().ok()); + + EXPECT_GT(spimi_testing::bigram_evictions(), 0U); + EXPECT_GT(spimi_testing::vocab_cap_sweeps(), 0U); + EXPECT_LE(capped.bigram_intern_bytes(), kCap + 2048); + // The uncapped control kept the whole tail resident. + EXPECT_GT(control.bigram_intern_bytes(), 4 * kCap); + + const std::string hot_term = make_phrase_bigram_term("hot", "pair"); + // The hot pair was never evicted: not in the bloom (the filter exists, + // evictions fired). + ASSERT_NE(capped.bigram_dropped_filter(), nullptr); + EXPECT_FALSE(capped.bigram_dropped_filter()->maybe_contains(hot_term)); + + std::map capped_terms = drain_to_map(&capped); + std::map control_terms = drain_to_map(&control); + ASSERT_TRUE(capped.status().ok()); + ASSERT_TRUE(control.status().ok()); + + // Eviction removed terms; it must never have touched a survivor's postings. + EXPECT_LT(capped_terms.size(), control_terms.size()); + ASSERT_TRUE(capped_terms.contains(hot_term)); + EXPECT_EQ(capped_terms.at(hot_term).docids, (std::vector {0, 1})); + for (const auto& [term, tp] : capped_terms) { + ASSERT_TRUE(control_terms.contains(term)) << term; + const TermPostings& ct = control_terms.at(term); + EXPECT_EQ(tp.docids, ct.docids) << term; + EXPECT_EQ(tp.freqs, ct.freqs) << term; + EXPECT_EQ(tp.positions_flat, ct.positions_flat) << term; + // df==1-only rule: every EVICTED term had df 1, so anything with df >= 2 + // must still be here. Equivalently: every control term with df >= 2 + // survives in the capped build. + } + for (const auto& [term, tp] : control_terms) { + if (tp.docids.size() >= 2) { + EXPECT_TRUE(capped_terms.contains(term)) << "df>=2 term evicted: " << term; + } + } +} + +// The sentinel (df==1 forever by construction) is never evicted regardless of +// cap pressure -- it gates reader semantics. +TEST(SniiBigramVocabCap, SentinelNeverEvicted) { + constexpr uint64_t kCap = 2 * 1024; + spimi_testing::reset_bigram_vocab_cap_counters(); + SpimiTermBuffer buf(/*has_positions=*/true); + buf.configure_bigram_diet(kCap); + + const std::string sentinel = make_phrase_bigram_sentinel_term(); + buf.add_token(sentinel, 0, 0); + for (uint32_t i = 0; i < 300; ++i) { + const auto [l, r] = filler_pair(i); + buf.add_bigram_token(l, r, 1 + i, 0); + } + ASSERT_TRUE(buf.status().ok()); + ASSERT_GT(spimi_testing::bigram_evictions(), 0U); // pressure definitely fired + + if (buf.bigram_dropped_filter() != nullptr) { + EXPECT_FALSE(buf.bigram_dropped_filter()->maybe_contains(sentinel)); + } + std::map terms = drain_to_map(&buf); + ASSERT_TRUE(terms.contains(sentinel)); + EXPECT_EQ(terms.at(sentinel).docids, (std::vector {0})); +} + +// (c): an evicted pair that REAPPEARS re-interns as a fresh term (its id space +// stays sane), but it is in the ever-dropped bloom, so the flush drops it even +// when its re-accumulated df is far above the threshold. Asserted end-to-end: +// the built segment has no dict entry for it, and the drop is attributed to +// the bloom seam (not the df threshold). +TEST(SniiBigramVocabCap, ReappearingEvictedPairDroppedAtFlushViaBloom) { + constexpr uint64_t kCap = 2 * 1024; + constexpr uint32_t kThreshold = 4; + spimi_testing::reset_bigram_vocab_cap_counters(); + doris::snii::writer::testing::reset_bigram_prune_counters(); + + SpimiTermBuffer buf(/*has_positions=*/true); + buf.configure_bigram_diet(kCap); + + // Hot survivor for sanity, df >= threshold before any pressure. + for (uint32_t d = 0; d < 8; ++d) { + buf.add_bigram_token("hot", "pair", d, 0); + } + // The victim pair: one occurrence -> df==1. + buf.add_bigram_token("gone", "away", 10, 0); + const std::string victim = make_phrase_bigram_term("gone", "away"); + + // Blow the cap until the victim is provably evicted (the incremental sweep + // walks the id space in cursor order; keep feeding unique tail pairs until + // its cursor has consumed the victim). Bounded: fails loudly if it never + // fires. + uint32_t docid = 20; + for (uint32_t i = 0; i < 4000; ++i) { + if (buf.bigram_dropped_filter() != nullptr && + buf.bigram_dropped_filter()->maybe_contains(victim)) { + break; + } + const auto [l, r] = filler_pair(i); + buf.add_bigram_token(l, r, docid++, 0); + } + ASSERT_NE(buf.bigram_dropped_filter(), nullptr); + ASSERT_TRUE(buf.bigram_dropped_filter()->maybe_contains(victim)); + const uint64_t evictions_after_pressure = spimi_testing::bigram_evictions(); + EXPECT_GE(evictions_after_pressure, 1U); + + // The pair REAPPEARS across many docs: re-interned (fresh term), df grows + // far past the threshold. Churn (the sweep may evict early re-incarnations + // again) is fine -- each re-eviction re-blooms it; the FINAL incarnation + // reaches the flush with a large df. + for (uint32_t d = 5000; d < 5200; ++d) { + buf.add_bigram_token("gone", "away", d, 0); + } + buf.add_token(make_phrase_bigram_sentinel_term(), 0, 0); + ASSERT_TRUE(buf.status().ok()); + + // Flush through the real writer with the bloom wired (as production does). + MemoryFile file; + writer::SniiIndexInput input; + input.index_id = 11; + input.index_suffix = "Body"; + input.config = format::IndexConfig::kDocsPositions; + input.doc_count = 5200; + input.bigram_prune_min_df = kThreshold; + input.term_source = &buf; + input.bigram_ever_dropped = buf.bigram_dropped_filter(); + + writer::SniiCompoundWriter writer(&file); + assert_ok(writer.add_logical_index(input)); + assert_ok(writer.finish()); + + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(reader::SniiSegmentReader::open(&file, &segment_reader)); + assert_ok(segment_reader.open_index(input.index_id, input.index_suffix, &index_reader)); + + auto lookup = [&](const std::string& term, format::DictEntry* entry) { + bool found = false; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + assert_ok(index_reader.lookup(term, &found, entry, &frq_base, &prx_base)); + return found; + }; + + format::DictEntry entry; + // The victim's final df (~200) is far above the threshold, yet the bloom + // dropped it: no dict entry, and the bloom seam (not the df seam) counted it. + EXPECT_FALSE(lookup(victim, &entry)); + EXPECT_EQ(doris::snii::writer::testing::bigram_bloom_dropped(), 1U); + // The hot pair materialized untouched, docs-only (G01 layout). + ASSERT_TRUE(lookup(make_phrase_bigram_term("hot", "pair"), &entry)); + EXPECT_EQ(entry.df, 8U); + EXPECT_EQ(entry.prx_len, 0U); + EXPECT_TRUE(entry.prx_bytes.empty()); + ASSERT_TRUE(lookup(make_phrase_bigram_sentinel_term(), &entry)); + // The meta still declares the df threshold (the bloom rides its contract). + EXPECT_EQ(index_reader.bigram_prune_min_df(), kThreshold); +} + +// RSS proxy: a synthetic long-document feed (many unique pairs per doc) keeps +// the intern storage bounded by the cap at EVERY probe point. +TEST(SniiBigramVocabCap, InternBytesBoundedByCapOnSyntheticLongDocs) { + constexpr uint64_t kCap = 16 * 1024; + SpimiTermBuffer capped(/*has_positions=*/true); + capped.configure_bigram_diet(kCap); + + uint32_t serial = 0; + for (uint32_t d = 0; d < 100; ++d) { + for (uint32_t k = 0; k < 50; ++k) { + const auto [l, r] = filler_pair(serial++); + capped.add_bigram_token(l, r, d, k); + } + // Probe once per "document": bounded by the cap plus the hover margin + // (the sweep runs inside each add; one term's footprint of slack). + ASSERT_LE(capped.bigram_intern_bytes(), kCap + 1024) << "doc " << d; + } + ASSERT_TRUE(capped.status().ok()); + EXPECT_GT(spimi_testing::bigram_evictions(), 0U); + // 5000 uniques at ~100 B apiece would be ~500 KiB uncapped; the cap held. + EXPECT_LE(capped.bigram_intern_bytes(), kCap + 1024); +} + +// (d) in-memory: with the diet on, bigram tokens stop buffering positions +// entirely (empty positions_flat, freqs intact); unigrams are untouched. +// Legacy (no diet) keeps bigram positions byte-for-byte. +TEST(SniiBigramVocabCap, DietStopsBufferingBigramPositions) { + SpimiTermBuffer diet(/*has_positions=*/true); + diet.configure_bigram_diet(0); // suppression without any cap + SpimiTermBuffer legacy(/*has_positions=*/true); + + for (uint32_t d = 0; d < 3; ++d) { + for (SpimiTermBuffer* buf : {&diet, &legacy}) { + buf->add_token("alpha", d, 5); + buf->add_bigram_token("alpha", "beta", d, 6); + } + } + ASSERT_TRUE(diet.status().ok()); + ASSERT_TRUE(legacy.status().ok()); + EXPECT_EQ(diet.total_tokens(), legacy.total_tokens()); + + std::map diet_terms = drain_to_map(&diet); + std::map legacy_terms = drain_to_map(&legacy); + + const std::string bigram = make_phrase_bigram_term("alpha", "beta"); + ASSERT_TRUE(diet_terms.contains(bigram)); + ASSERT_TRUE(legacy_terms.contains(bigram)); + // Same docids/freqs; the diet dropped ONLY the (never-emitted) positions. + EXPECT_EQ(diet_terms.at(bigram).docids, legacy_terms.at(bigram).docids); + EXPECT_EQ(diet_terms.at(bigram).freqs, legacy_terms.at(bigram).freqs); + EXPECT_TRUE(diet_terms.at(bigram).positions_flat.empty()); + EXPECT_EQ(legacy_terms.at(bigram).positions_flat, (std::vector {6, 6, 6})); + // Unigrams keep positions under the diet. + ASSERT_TRUE(diet_terms.contains("alpha")); + EXPECT_EQ(diet_terms.at("alpha").positions_flat, (std::vector {5, 5, 5})); + EXPECT_EQ(diet_terms.at("alpha").docids, legacy_terms.at("alpha").docids); +} + +// (d) out-of-core: suppressed bigram terms survive the spill/merge round trip +// -- runs serialize an empty position block per record, the k-way merge +// coalesces docids/freqs across runs (including the same-docid boundary case +// that would index an empty positions_flat without the per-term guard) and the +// merged term still carries no positions. Unigram positions are intact. +TEST(SniiBigramVocabCap, SuppressedBigramSurvivesSpillMergeRoundTrip) { + // A spill threshold below the 32 KiB arena block size forces a spill on + // (nearly) every token -- the established pattern from the spill tests -- + // so one doc's two bigram tokens land in DIFFERENT runs: the merge must + // boundary-coalesce the same docid with EMPTY position blocks. + SpimiTermBuffer buf(/*has_positions=*/true, /*spill_threshold_bytes=*/4096); + buf.configure_bigram_diet(0); + + for (uint32_t d = 0; d < 10; ++d) { + buf.add_token("word", d, 0); + buf.add_bigram_token("aa", "bb", d, 0); + buf.add_token("word", d, 1); + buf.add_bigram_token("aa", "bb", d, 5); + } + ASSERT_TRUE(buf.status().ok()); + EXPECT_GE(buf.run_count_for_test(), 2U); + + // Retaining the TermPostings past the callback is safe here: every term is + // tiny (ntok 20 and merged df 10), far below the pos_pump streaming gates, + // so positions are always fully materialized. + std::map terms; + assert_ok(buf.for_each_term_sorted([&](TermPostings&& tp) { + std::string key = tp.term; + terms.emplace(std::move(key), std::move(tp)); + })); + + const std::string bigram = make_phrase_bigram_term("aa", "bb"); + ASSERT_TRUE(terms.contains(bigram)); + const TermPostings& bt = terms.at(bigram); + std::vector want_docs(10); + for (uint32_t d = 0; d < 10; ++d) { + want_docs[d] = d; + } + EXPECT_EQ(bt.docids, want_docs); + EXPECT_EQ(bt.freqs, std::vector(10, 2)); // boundary-coalesced + EXPECT_TRUE(bt.positions_flat.empty()); + + ASSERT_TRUE(terms.contains("word")); + const TermPostings& wt = terms.at("word"); + EXPECT_EQ(wt.docids, want_docs); + EXPECT_EQ(wt.freqs, std::vector(10, 2)); + std::vector want_pos; + for (uint32_t d = 0; d < 10; ++d) { + want_pos.push_back(0); + want_pos.push_back(1); + } + EXPECT_EQ(wt.positions_flat, want_pos); +} diff --git a/be/test/storage/index/snii/writer/compact_posting_pool_test.cpp b/be/test/storage/index/snii/writer/compact_posting_pool_test.cpp new file mode 100644 index 00000000000000..9b9f7aa6021680 --- /dev/null +++ b/be/test/storage/index/snii/writer/compact_posting_pool_test.cpp @@ -0,0 +1,602 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/compact_posting_pool.h" + +#include + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +using doris::snii::writer::CompactPostingPool; +using doris::snii::writer::SpimiTermBuffer; +using doris::snii::writer::TermPostings; + +namespace { + +// Test helper bundling a chain's append handle, its level, and its head -- the +// same per-term state the real accumulator keeps -- so tests can append by value. +struct Chain { + CompactPostingPool::SliceWriter w; + uint8_t level = 0; + uint32_t head = 0; + void start(CompactPostingPool* pool) { head = pool->start_chain(&w, &level); } + void put(CompactPostingPool* pool, uint8_t b) { pool->append_byte(&w, &level, b); } +}; + +// Reads back the whole chain into a vector for comparison. +std::vector ReadChain(const CompactPostingPool& pool, uint32_t head, uint64_t len) { + std::vector out; + out.reserve(len); + CompactPostingPool::Cursor c = pool.cursor(head, len); + while (c.has_next()) { + out.push_back(c.next()); + } + return out; +} + +} // namespace + +// A single chain shorter than one slice round-trips exactly. +TEST(SniiCompactPostingPool, TinyChainRoundTrips) { + CompactPostingPool pool; + Chain ch; + ch.start(&pool); + const std::vector data = {7, 0, 255, 42}; + for (uint8_t b : data) { + ch.put(&pool, b); + } + EXPECT_EQ(ReadChain(pool, ch.head, data.size()), data); + EXPECT_EQ(pool.payload_bytes(), data.size()); +} + +// A chain that spans many slice levels round-trips exactly (exercises forward +// pointers across several geometric slice sizes). +TEST(SniiCompactPostingPool, MultiSliceChainRoundTrips) { + CompactPostingPool pool; + Chain ch; + ch.start(&pool); + std::vector data; + for (uint32_t i = 0; i < 5000; ++i) { + data.push_back(static_cast(i * 31 + 7)); + } + for (uint8_t b : data) { + ch.put(&pool, b); + } + EXPECT_EQ(ReadChain(pool, ch.head, data.size()), data); + EXPECT_EQ(pool.payload_bytes(), data.size()); +} + +// Many INTERLEAVED chains (the real SPIMI access pattern) stay independent: a byte +// written to chain A never appears in chain B's read-back. +TEST(SniiCompactPostingPool, InterleavedChainsIndependent) { + CompactPostingPool pool; + constexpr int kChains = 64; + std::vector chains(kChains); + std::vector> expect(kChains); + for (auto& ch : chains) { + ch.start(&pool); + } + + std::mt19937 rng(12345); + // Append bytes to chains in a randomized interleaving so slices for different + // chains land in the same blocks intermixed. + for (int round = 0; round < 20000; ++round) { + const int c = static_cast(rng() % kChains); + const auto b = static_cast(rng()); + chains[c].put(&pool, b); + expect[c].push_back(b); + } + for (int i = 0; i < kChains; ++i) { + EXPECT_EQ(ReadChain(pool, chains[i].head, expect[i].size()), expect[i]) << "chain " << i; + } +} + +// MANY chains + MANY bytes force the arena across several 32 KiB block +// boundaries. This is the regression for a block-boundary bump bug: a run that +// exactly fills a block must allocate the next block before handing out the +// boundary offset, never returning an offset into a not-yet-allocated block. +TEST(SniiCompactPostingPool, ManyChainsAcrossBlockBoundaries) { + CompactPostingPool pool; + constexpr int kChains = 2000; + std::vector chains(kChains); + std::vector> expect(kChains); + for (auto& ch : chains) { + ch.start(&pool); + } + + std::mt19937 rng(98765); + for (int round = 0; round < 1'000'000; ++round) { + const int c = static_cast(rng() % kChains); + const auto b = static_cast(rng()); + chains[c].put(&pool, b); + expect[c].push_back(b); + } + for (int i = 0; i < kChains; ++i) { + EXPECT_EQ(ReadChain(pool, chains[i].head, expect[i].size()), expect[i]) << "chain " << i; + } +} + +// An empty chain (started but never written) reads back as zero bytes. +TEST(SniiCompactPostingPool, EmptyChain) { + CompactPostingPool pool; + Chain ch; + ch.start(&pool); + EXPECT_TRUE(ReadChain(pool, ch.head, 0).empty()); +} + +// A chain that exactly fills a slice boundary (no extra byte) reads back exactly, +// without dereferencing the (still-zero) forward pointer. +TEST(SniiCompactPostingPool, ExactSliceBoundary) { + CompactPostingPool pool; + Chain ch; + ch.start(&pool); + // Exactly fill the level-0 slice (kSliceSizes[0] payload bytes) and stop. + std::vector data; + for (uint32_t i = 0; i < CompactPostingPool::kSliceSizes_level0(); ++i) { + data.push_back(static_cast(i + 1)); + } + for (uint8_t b : data) { + ch.put(&pool, b); + } + EXPECT_EQ(ReadChain(pool, ch.head, data.size()), data); + // Now extend by one byte (forces the forward link) and re-read fully. + ch.put(&pool, 99); + data.push_back(99); + EXPECT_EQ(ReadChain(pool, ch.head, data.size()), data); +} + +// Cursor CONTRACT: `budget` is an UPPER BOUND on bytes yielded, NOT a required exact +// length. The cursor is SELF-TERMINATING -- it stops at the chain tail (a zero forward +// pointer) no matter how large the budget. This pins both halves of the single contract: +// (1) an exact-length budget round-trips the written bytes, and +// (2) an OVER-LARGE budget (as the production caller passes -- the write-head offset) +// stays MEMORY-SAFE: next() never follows the tail's zero forward pointer off the +// chain into block 0 (UB); it yields at most the tail-slice's payload region (the +// written bytes plus the slice's zero-initialized unwritten tail) and then stops. +// +// WITHOUT the tail check (next_head == 0 -> stop), looping on has_next() with an +// over-large budget would hit the slice boundary, read the still-zero tail forward +// pointer, jump cur_ to offset 0, and re-read block 0's live bytes (an ALIAS) -- exactly +// the misuse the old "remaining" contract left latent. This test fails without the fix. +TEST(SniiCompactPostingPool, CursorOverLargeBudgetSelfTerminates) { + CompactPostingPool pool; + // Lay down a DISTINCT first chain so block 0 offset 0 holds recognizable bytes; if the + // cursor ever aliased offset 0 it would yield these, which we assert it never does. + Chain decoy; + decoy.start(&pool); + const std::vector decoy_bytes = {0xDE, 0xAD, 0xBE, 0xEF}; + for (uint8_t b : decoy_bytes) { + decoy.put(&pool, b); + } + ASSERT_EQ(decoy.head, 0U) << "first chain must own pool offset 0 (the alias target)"; + + // A second short chain whose tail forward pointer is still zero (single level-0 slice). + Chain ch; + ch.start(&pool); + const std::vector data = {11, 22, 33}; + for (uint8_t b : data) { + ch.put(&pool, b); + } + + // (1) exact-length budget round-trips the written bytes. + EXPECT_EQ(ReadChain(pool, ch.head, data.size()), data); + + // (2) MISUSE: a budget far larger than the payload. The cursor must self-terminate at + // the chain tail, yielding at most the level-0 slice's payload region and stopping. + const uint32_t slice0 = CompactPostingPool::kSliceSizes_level0(); + const uint32_t over_budget = 100U * CompactPostingPool::kBlockSize; // absurdly large + CompactPostingPool::Cursor c = pool.cursor(ch.head, over_budget); + std::vector pulled; + uint32_t pulls = 0; + while (c.has_next()) { + pulled.push_back(c.next()); + ASSERT_LT(++pulls, over_budget) << "cursor failed to self-terminate at the chain tail"; + } + // It stops at the tail: exactly the level-0 slice's payload region (written bytes plus + // the slice's zero-initialized unwritten tail), and NOTHING beyond. + EXPECT_EQ(pulled.size(), slice0) + << "over-large budget must stop at the tail slice's end, not run on"; + ASSERT_GE(pulled.size(), data.size()); + EXPECT_EQ(std::vector(pulled.begin(), pulled.begin() + data.size()), data); + for (size_t i = data.size(); i < pulled.size(); ++i) { + EXPECT_EQ(pulled[i], 0U) << "unwritten tail byte " << i << " must read as zero"; + } + // CRITICAL: the cursor must NEVER have aliased the decoy at offset 0. + for (uint8_t b : pulled) { + EXPECT_NE(b, 0xDEU) << "cursor aliased block 0 -- tail check failed"; + } +} + +// Computes an EXACT one-block fill layout analytically from the public slice schedule: +// `pad` level-0 chains (each consuming one level-0 slice allocation) followed by ONE +// chain grown until it has just allocated the slice at `grow_to_level`, such that the +// total arena bytes consumed equal exactly kBlockSize. Returns false if no such layout +// exists for the current schedule. No magic numbers -- everything derives from the +// kSliceSize_at / kNextLevel_at accessors, so a schedule change is handled automatically. +static bool ExactBlockFillLayout(uint32_t* pad, int* grow_to_level) { + const uint32_t block = CompactPostingPool::kBlockSize; + const uint32_t l0_alloc = + CompactPostingPool::kSliceSizes_level0() + CompactPostingPool::kPtrBytes; + // cum = bytes a single growing chain consumes after allocating slices for levels + // 0..L inclusive (each slice allocation is payload-cap + kPtrBytes). + uint32_t cum = 0; + for (int level = 0; level < CompactPostingPool::kLevelCount; ++level) { + cum += CompactPostingPool::kSliceSize_at(level) + CompactPostingPool::kPtrBytes; + if (cum > block) { + break; + } + const uint32_t rem = block - cum; + if (rem % l0_alloc == 0) { + *pad = rem / l0_alloc; + *grow_to_level = level; + return true; + } + } + return false; +} + +// DETERMINISTIC coverage of alloc_run case (c): a previous allocation EXACTLY fills a +// 32 KiB block, leaving next_offset_ on the block boundary (in_block == 0) over a block +// that is already fully consumed. The next allocation MUST detect this (via the +// tail_exists guard) and start a FRESH block -- it must NOT mistake in_block == 0 for an +// empty fresh block and hand back offset 0, which would alias block 0's live bytes. +// +// Today this branch is only hit probabilistically by the RNG-seeded interleave tests. +// Here we drive it EXACTLY: compute a layout (analytically, from the public schedule) +// whose allocations fill block 0 to its LAST byte (next_offset_ == kBlockSize), realize +// it on a real pool, then start one more chain. Its head MUST be exactly kBlockSize +// (block 1, byte 0) -- never 0 -- and every chain in block 0 MUST still round-trip, +// proving the case-(c) allocation did not return offset 0 and clobber block 0. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiCompactPostingPool, AllocRunExactBlockFillStartsFreshBlock) { + const uint32_t block = CompactPostingPool::kBlockSize; + + uint32_t pad = 0; + int grow_to_level = 0; + ASSERT_TRUE(ExactBlockFillLayout(&pad, &grow_to_level)) + << "no exact one-block fill exists for the current slice schedule"; + + CompactPostingPool pool; + std::vector chains; + std::vector> data; + + // Lay down `pad` level-0 chains, each with one identifiable payload byte. None may + // spill out of block 0 (the layout was computed so they all fit). + for (uint32_t i = 0; i < pad; ++i) { + Chain c; + c.start(&pool); + const auto b = static_cast(0x40 + (i & 0x3F)); + c.put(&pool, b); + chains.push_back(c); + data.push_back({b}); + ASSERT_EQ(pool.arena_bytes(), block) << "padding chain " << i << " spilled early"; + } + + // Grow ONE chain until it has just allocated the slice at `grow_to_level`, writing no + // byte into that final slice so its tail ends exactly on the block boundary. + Chain g; + g.start(&pool); + std::vector gbytes; + uint32_t guard = 0; + while (std::cmp_less(g.level, grow_to_level)) { + const auto b = static_cast(0x80 + (gbytes.size() & 0x3F)); + g.put(&pool, b); + gbytes.push_back(b); + ASSERT_EQ(pool.arena_bytes(), block) << "grown chain spilled before the boundary"; + ASSERT_LT(++guard, block) << "grow loop did not converge"; + } + chains.push_back(g); + data.push_back(gbytes); + + // Block 0 is now full to its LAST byte: next_offset_ == kBlockSize, still one block. + ASSERT_EQ(pool.arena_bytes(), block) << "block 0 should be exactly full, one block"; + + // The case-(c) allocation: starting a new chain must open a FRESH block at offset + // kBlockSize, NOT alias offset 0. + Chain probe; + probe.start(&pool); + EXPECT_EQ(probe.head, block) << "case (c) must hand out block 1 byte 0, not offset 0"; + EXPECT_NE(probe.head, 0U) << "case (c) must not alias block 0"; + EXPECT_EQ(pool.arena_bytes(), 2U * block) << "exactly two blocks after the probe"; + + // The probe chain owns block 1 byte 0; write+read it to confirm it is live. + probe.put(&pool, 0xEE); + EXPECT_EQ(ReadChain(pool, probe.head, 1U), (std::vector {0xEE})); + + // Every chain laid down in block 0 must still round-trip -- proving the case-(c) + // allocation did NOT return offset 0 and overwrite block 0's live bytes. + for (size_t i = 0; i < chains.size(); ++i) { + EXPECT_EQ(ReadChain(pool, chains[i].head, data[i].size()), data[i]) + << "block-0 chain " << i << " corrupted across the exact block boundary"; + } +} + +// reset() drops all blocks; a fresh chain after reset starts clean. +TEST(SniiCompactPostingPool, ResetClears) { + CompactPostingPool pool; + Chain ch; + ch.start(&pool); + for (int i = 0; i < 1000; ++i) { + ch.put(&pool, static_cast(i)); + } + EXPECT_GT(pool.payload_bytes(), 0U); + pool.reset(); + EXPECT_EQ(pool.payload_bytes(), 0U); + Chain ch2; + ch2.start(&pool); + std::vector data = {5, 6, 7}; + for (uint8_t b : data) { + ch2.put(&pool, b); + } + EXPECT_EQ(ReadChain(pool, ch2.head, data.size()), data); +} + +// ====================================================================================== +// T13: regression net for the append_byte / Cursor::has_next / Cursor::next INLINE move. +// +// The move is a pure code relocation (.cpp out-of-line bodies -> .h inline), so every +// assertion below must hold byte-for-byte BOTH before and after the change: the golden +// values are the contract. F1-F8 drive the arena encode (append_byte) and decode +// (Cursor) micro-loops directly; F9 covers the full SpimiTermBuffer encode+decode path +// (put_byte->append_byte ... DecodeChainVarint->Cursor::next) end to end; F10 pins the +// out-of-vocab error path. F4/F5/F9/F10 add coverage the pre-existing suite lacked +// (tail no-phantom via an over-large budget, budget truncation, end-to-end postings, +// latched InvalidArgument). +// ====================================================================================== + +// T13-F1: a chain that fits in one level-0 slice round-trips and never overflows. +TEST(SniiCompactPostingPoolTest, RoundTripsSingleSlice) { + CompactPostingPool pool; + Chain ch; + ch.start(&pool); + std::vector data; + for (uint32_t i = 0; i < CompactPostingPool::kSliceSizes_level0(); ++i) { + data.push_back(static_cast(i * 9 + 1)); + } + for (uint8_t b : data) { + ch.put(&pool, b); + } + EXPECT_EQ(ReadChain(pool, ch.head, data.size()), data); + EXPECT_EQ(pool.payload_bytes(), data.size()); + EXPECT_EQ(ch.level, uint8_t {0}) << "a single-slice fill must stay at level 0"; +} + +// T13-F2: 5000 pseudo-random bytes (fixed seed) cross many slice levels and round-trip +// byte-identically -- the encode/decode forward-pointer chain golden. +TEST(SniiCompactPostingPoolTest, RoundTripsAcrossSliceLevels) { + CompactPostingPool pool; + Chain ch; + ch.start(&pool); + std::mt19937 rng(0xC0FFEEU); + std::vector data; + data.reserve(5000); + for (uint32_t i = 0; i < 5000; ++i) { + const auto b = static_cast(rng()); + data.push_back(b); + ch.put(&pool, b); + } + EXPECT_EQ(ReadChain(pool, ch.head, data.size()), data); + EXPECT_EQ(pool.payload_bytes(), data.size()); + EXPECT_GT(ch.level, uint8_t {0}) << "5000 bytes must advance past level 0"; +} + +// T13-F3: a single chain longer than one 32 KiB block spans >= 2 blocks and round-trips +// byte-identically (exercises at()'s two-level block index across alloc_run new blocks). +TEST(SniiCompactPostingPoolTest, RoundTripsAcrossBlockBoundary) { + CompactPostingPool pool; + Chain ch; + ch.start(&pool); + std::mt19937 rng(0xB10CU); + const uint32_t n = 3U * CompactPostingPool::kBlockSize; // payload alone exceeds 2 blocks + std::vector data; + data.reserve(n); + for (uint32_t i = 0; i < n; ++i) { + const auto b = static_cast(rng()); + data.push_back(b); + ch.put(&pool, b); + } + EXPECT_GE(pool.arena_bytes(), 2ULL * CompactPostingPool::kBlockSize) + << "payload over one block must occupy >= 2 blocks"; + EXPECT_EQ(ReadChain(pool, ch.head, data.size()), data); + EXPECT_EQ(pool.payload_bytes(), data.size()); +} + +// T13-F4: with a budget LARGER than the payload, has_next() must stop at the tail slice's +// zero forward pointer and report no phantom trailing byte (the stop is the tail, not the +// budget). next() at the tail yields 0. +TEST(SniiCompactPostingPoolTest, HasNextStopsAtTailNoPhantom) { + CompactPostingPool pool; + Chain ch; + ch.start(&pool); + const uint32_t cap = CompactPostingPool::kSliceSize_at(0); + std::vector data; + for (uint32_t i = 0; i < cap; ++i) { + const auto b = static_cast(i + 1); + data.push_back(b); + ch.put(&pool, b); + } + // Budget far exceeds the payload, so a false has_next() can only come from the tail + // zero pointer (a phantom-byte bug would instead read the zero pointer as a byte). + CompactPostingPool::Cursor c = pool.cursor(ch.head, CompactPostingPool::kBlockSize); + std::vector out; + while (c.has_next()) { + out.push_back(c.next()); + } + EXPECT_EQ(out, data) << "exactly the written payload region, no phantom tail byte"; + EXPECT_EQ(out.size(), cap); + EXPECT_FALSE(c.has_next()) << "tail zero pointer must stop has_next()"; + EXPECT_EQ(c.next(), 0U) << "next() past the tail yields 0"; +} + +// T13-F5: a budget SMALLER than the payload truncates the cursor to exactly `budget` +// bytes (the first ones, in write order), then has_next() goes false. +TEST(SniiCompactPostingPoolTest, BudgetCapsYieldedBytes) { + CompactPostingPool pool; + Chain ch; + ch.start(&pool); + std::vector data; + for (uint32_t i = 0; i < 100; ++i) { + const auto b = static_cast(i * 7 + 1); + data.push_back(b); + ch.put(&pool, b); + } + constexpr uint64_t kBudget = 10; + CompactPostingPool::Cursor c = pool.cursor(ch.head, kBudget); + std::vector out; + while (c.has_next()) { + out.push_back(c.next()); + } + EXPECT_EQ(out.size(), kBudget); + EXPECT_EQ(out, std::vector(data.begin(), data.begin() + kBudget)); + EXPECT_FALSE(c.has_next()) << "budget spent"; + EXPECT_EQ(c.next(), 0U); +} + +// T13-F6: an absurd budget over a short multi-slice chain self-terminates at the tail +// slice's payload end -- yielding the written bytes plus the slice's zero-initialized +// unwritten tail, and NEVER aliasing block 0 (offset 0 is owned by a decoy chain). +TEST(SniiCompactPostingPoolTest, OverLargeBudgetSelfTerminates) { + CompactPostingPool pool; + // Decoy owns offset 0 so an erroneous alias to block 0 is detectable. + Chain decoy; + decoy.start(&pool); + for (uint8_t b : {static_cast(0xDE), static_cast(0xAD)}) { + decoy.put(&pool, b); + } + ASSERT_EQ(decoy.head, 0U) << "first chain must own pool offset 0"; + + Chain ch; + ch.start(&pool); + std::vector data; + for (uint32_t i = 0; i < 7; ++i) { + const auto b = static_cast(0x11 * (i + 1)); // 0x11..0x77, never 0 or 0xDE + data.push_back(b); + ch.put(&pool, b); + } + + const uint64_t over = 1ULL << 20; // far beyond the 7-byte payload + CompactPostingPool::Cursor c = pool.cursor(ch.head, over); + std::vector out; + uint64_t guard = 0; + while (c.has_next()) { + out.push_back(c.next()); + ASSERT_LT(++guard, over) << "cursor failed to self-terminate at the chain tail"; + } + // 7 bytes fill level 0 (cap kSliceSize_at(0)) then spill into the next level; the + // cursor stops at that tail slice's payload end. + const size_t expect_len = + CompactPostingPool::kSliceSize_at(0) + + CompactPostingPool::kSliceSize_at(CompactPostingPool::kNextLevel_at(0)); + EXPECT_EQ(out.size(), expect_len) << "must stop at the tail slice's end, not run on"; + ASSERT_GE(out.size(), data.size()); + EXPECT_EQ(std::vector(out.begin(), out.begin() + data.size()), data); + for (size_t i = data.size(); i < out.size(); ++i) { + EXPECT_EQ(out[i], 0U) << "unwritten tail byte " << i << " must read as zero"; + } + for (uint8_t b : out) { + EXPECT_NE(b, static_cast(0xDE)) << "cursor aliased block 0"; + } +} + +// T13-F7: a started-but-never-written chain with a zero budget yields nothing. +TEST(SniiCompactPostingPoolTest, EmptyChainYieldsNothing) { + CompactPostingPool pool; + Chain ch; + ch.start(&pool); + CompactPostingPool::Cursor c = pool.cursor(ch.head, 0); + EXPECT_FALSE(c.has_next()); + EXPECT_EQ(c.next(), 0U); +} + +// T13-F8: exactly filling a level-0 slice then writing ONE more byte advances the chain +// to the scheduled next level and links the slices; the whole chain round-trips. +TEST(SniiCompactPostingPoolTest, SliceOverflowLinksCorrectly) { + CompactPostingPool pool; + Chain ch; + ch.start(&pool); + const uint32_t cap0 = CompactPostingPool::kSliceSize_at(0); + std::vector data; + for (uint32_t i = 0; i < cap0; ++i) { + const auto b = static_cast(0xA0 + i); + data.push_back(b); + ch.put(&pool, b); + } + EXPECT_EQ(ch.level, uint8_t {0}) << "an exact fill has not overflowed yet"; + // The boundary+1 byte forces the overflow + forward link. + ch.put(&pool, 0x5A); + data.push_back(0x5A); + EXPECT_EQ(ch.level, CompactPostingPool::kNextLevel_at(0)) + << "overflow must advance to the scheduled next level"; + EXPECT_EQ(ReadChain(pool, ch.head, data.size()), data); +} + +// T13-F9 (equivalence/golden): the full SpimiTermBuffer encode+decode path. The token +// feed exercises lexicographic term ordering (NOT first-seen), a multi-doc term, a freq>1 +// doc (banana@10 twice), and an out-of-order docid (cherry 100 then 50) that drives the +// finalize SortByDocid + position reorder. The golden TermPostings were derived by hand +// from the documented tagged-varint contract and MUST stay identical across the inline +// move -- this is the core "new path == old path" check for T13. +TEST(SniiCompactPostingPoolTest, EndToEndPostingsEquivalence) { + SpimiTermBuffer buf(/*has_positions=*/true); // owned-vocab (string-keyed add_token) + buf.add_token("banana", 10, 0); + buf.add_token("apple", 5, 3); + buf.add_token("cherry", 100, 0); + buf.add_token("banana", 10, 5); // same doc -> freq 2, positions {0,5} + buf.add_token("apple", 7, 2); + buf.add_token("cherry", 50, 9); // docid < previous -> triggers SortByDocid + reorder + buf.add_token("banana", 20, 1); + + std::vector got = buf.finalize_sorted(); + EXPECT_TRUE(buf.status().ok()) << buf.status().to_string(); + ASSERT_EQ(got.size(), 3U); + + EXPECT_EQ(got[0].term, "apple"); + EXPECT_EQ(got[0].docids, (std::vector {5, 7})); + EXPECT_EQ(got[0].freqs, (std::vector {1, 1})); + EXPECT_EQ(got[0].positions_flat, (std::vector {3, 2})); + + EXPECT_EQ(got[1].term, "banana"); + EXPECT_EQ(got[1].docids, (std::vector {10, 20})); + EXPECT_EQ(got[1].freqs, (std::vector {2, 1})); + EXPECT_EQ(got[1].positions_flat, (std::vector {0, 5, 1})); + + EXPECT_EQ(got[2].term, "cherry"); + EXPECT_EQ(got[2].docids, (std::vector {50, 100})); + EXPECT_EQ(got[2].freqs, (std::vector {1, 1})); + EXPECT_EQ(got[2].positions_flat, (std::vector {9, 0})); +} + +// T13-F10 (error path): a borrowed-vocab buffer fed an out-of-range term-id latches an +// InvalidArgument into status(), ignores the token, and finalize_sorted() yields nothing +// (no spill, no crash). +TEST(SniiCompactPostingPoolTest, OutOfVocabTokenLatchesError) { + const std::vector vocab = {"alpha", "beta"}; + SpimiTermBuffer buf(&vocab, /*has_positions=*/true); + buf.add_token(/*term_id=*/5U, /*docid=*/1U, /*pos=*/0U); // 5 >= vocab.size() == 2 + + std::vector got = buf.finalize_sorted(); + EXPECT_TRUE(got.empty()); + EXPECT_FALSE(buf.status().ok()); + EXPECT_TRUE(buf.status().is()) << buf.status().to_string(); +} diff --git a/be/test/storage/index/snii/writer/logical_index_writer_freq_stats_test.cpp b/be/test/storage/index/snii/writer/logical_index_writer_freq_stats_test.cpp new file mode 100644 index 00000000000000..256b055f126b9d --- /dev/null +++ b/be/test/storage/index/snii/writer/logical_index_writer_freq_stats_test.cpp @@ -0,0 +1,423 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/logical_index_writer.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" +#include "storage/index/snii_query_test_util.h" + +// T12 -- writer fused freqs statistics (single pass total + max), reused for the +// has_prx position-count check, stats_.sum_total_term_freq, and the DictEntry +// ttf_delta/max_freq. This suite guards: +// * the deterministic op-count: exactly ONE term-level freqs scan per term +// (was 3N docs-only / 4N with positions) via the term_freq_scans() seam; +// * value bit-identity: ttf_delta / max_freq / sum_total_term_freq read back +// equal to an independent reference, across windowed + slim(pod_ref/inline); +// * the fused pure helper on boundary inputs (empty/single/zeros/equal/u32max/ +// large random) vs a naive reference; +// * the validate_term error paths preserved after dropping its internal +// freqs-sum loop (length / position-count / strict-ascending), and that the +// has_prx position-count check now consumes the FUSED total. +namespace doris::snii::writer { +namespace { + +using doris::Status; // RETURN_IF_ERROR / Status::OK() expand to a bare Status. +using snii_test::assert_ok; +using snii_test::make_term; +using snii_test::MemoryFile; +using snii_test::PostingDoc; + +// One term whose postings are known up front, so the reference total/max can be +// recomputed in the test independently of the writer. +struct KnownTerm { + std::string term; + std::vector docs; + + uint64_t ref_sum() const { + uint64_t sum = 0; + for (const PostingDoc& doc : docs) { + sum += doc.positions.size(); + } + return sum; + } + uint32_t ref_max() const { + uint32_t max = 0; + for (const PostingDoc& doc : docs) { + max = std::max(max, static_cast(doc.positions.size())); + } + return max; + } + uint32_t df() const { return static_cast(docs.size()); } +}; + +// 480 docids with irregular gaps (deterministic LCG): the slim docs region PFOR +// exceeds the 256B inline threshold so the term becomes a pod_ref, while df < 512 +// keeps it slim (not windowed). Mirrors the proven generator in snii_query_test. +std::vector slim_pod_ref_docids() { + std::vector ids; + ids.reserve(480); + uint32_t cur = 0; + uint32_t state = 0x9e3779b9U; + for (int i = 0; i < 480; ++i) { + ids.push_back(cur); + state = state * 1664525U + 1013904223U; + cur += 1U + (state >> 23) % 250U; // gap in [1, 250] + } + return ids; +} + +// A corpus covering every term-level encoding branch with non-trivial freqs: +// "wide" df=600 >= kSlimDfThreshold(512) -> windowed (3 base-unit windows), +// freqs cycle 1,2,3 (max=3) so the per-window MaxOf is also exercised. +// "slimref" df=480 < 512, irregular gaps -> slim pod_ref, freq 1 (max=1). +// "tiny" df=4 -> slim inline, one doc freq 2 (max=2). +// "mid" df=300 consecutive -> slim, one doc freq 2 (max=2). +std::vector make_known_corpus() { + std::vector corpus; + + KnownTerm wide; + wide.term = "wide"; + for (uint32_t i = 0; i < 600; ++i) { + const uint32_t freq = (i % 3) + 1; + std::vector positions; + positions.reserve(freq); + for (uint32_t p = 0; p < freq; ++p) { + positions.push_back(p); + } + wide.docs.push_back({i, std::move(positions)}); + } + corpus.push_back(std::move(wide)); + + KnownTerm slimref; + slimref.term = "slimref"; + for (uint32_t docid : slim_pod_ref_docids()) { + slimref.docs.push_back({docid, {0}}); + } + corpus.push_back(std::move(slimref)); + + KnownTerm tiny; + tiny.term = "tiny"; + tiny.docs = {{.docid = 10, .positions = {0, 1}}, + {.docid = 20, .positions = {0}}, + {.docid = 30, .positions = {0}}, + {.docid = 40, .positions = {0}}}; + corpus.push_back(std::move(tiny)); + + KnownTerm mid; + mid.term = "mid"; + for (uint32_t i = 0; i < 300; ++i) { + mid.docs.push_back({i, i == 0 ? std::vector {0, 1} : std::vector {0}}); + } + corpus.push_back(std::move(mid)); + + return corpus; +} + +const KnownTerm& find_term(const std::vector& corpus, std::string_view term) { + for (const KnownTerm& kt : corpus) { + if (std::string_view(kt.term) == term) { + return kt; + } + } + ADD_FAILURE() << "term not in corpus: " << term; + return corpus.front(); +} + +// Builds the corpus into `file` and opens a reader over it. The corpus is left +// untouched so the caller can recompute references. +Status build_corpus_reader(MemoryFile* file, reader::SniiSegmentReader* segment_reader, + reader::LogicalIndexReader* index_reader, + const std::vector& corpus, format::IndexConfig config) { + SniiIndexInput input; + input.index_id = 7; + input.index_suffix = "Body"; + input.config = config; + uint32_t max_docid = 0; + input.terms.reserve(corpus.size()); + for (const KnownTerm& kt : corpus) { + for (const PostingDoc& doc : kt.docs) { + max_docid = std::max(max_docid, doc.docid); + } + input.terms.push_back(make_term(kt.term, kt.docs)); + } + input.doc_count = max_docid + 1; + std::ranges::sort(input.terms, [](const TermPostings& lhs, const TermPostings& rhs) { + return lhs.term < rhs.term; + }); + + SniiCompoundWriter writer(file); + RETURN_IF_ERROR(writer.add_logical_index(input)); + RETURN_IF_ERROR(writer.finish()); + EXPECT_TRUE(file->finalized()); + + RETURN_IF_ERROR(reader::SniiSegmentReader::open(file, segment_reader)); + return segment_reader->open_index(input.index_id, input.index_suffix, index_reader); +} + +format::DictEntry lookup_entry(const reader::LogicalIndexReader& index_reader, + std::string_view term) { + bool found = false; + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + assert_ok(index_reader.lookup(term, &found, &entry, &frq_base, &prx_base)); + EXPECT_TRUE(found) << term; + return entry; +} + +// ------------------------------------------------------------------------- +// Deterministic perf: exactly one term-level freqs scan per term. +// ------------------------------------------------------------------------- + +// The op-count seam (testing::term_freq_scans) increments once per fuse_freq_stats +// call, and process_term calls it exactly once per term -> N for an N-term build. +// Before the fusion the instrumented build scanned freqs 4x per term with +// positions (validate-sum + stats-sum + ttf-sum + max), i.e. 4 * N, so this +// assertion fails on the un-fused code and passes after the single-pass fuse. +TEST(SniiWriterTest, ProcessTermScansFreqsOncePerTerm) { + const std::vector corpus = make_known_corpus(); + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + + testing::reset_term_freq_scans(); + assert_ok(build_corpus_reader(&file, &segment_reader, &index_reader, corpus, + format::IndexConfig::kDocsPositions)); + + EXPECT_EQ(testing::term_freq_scans(), corpus.size()); +} + +// ------------------------------------------------------------------------- +// Value bit-identity: ttf_delta / max_freq / sum_total_term_freq. +// ------------------------------------------------------------------------- + +// FW-EQ-1 / FW-EQ-2: every term's read-back ttf_delta == sum(freqs) and +// max_freq == max(freqs), and the index-level sum_total_term_freq == the sum of +// every term's sum(freqs). The reference is computed independently from the known +// corpus. Holds before AND after the fusion (guards the CSE did not change values). +TEST(SniiWriterTest, FusedFreqStatsPreserveTtfMaxAndSum) { + const std::vector corpus = make_known_corpus(); + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_corpus_reader(&file, &segment_reader, &index_reader, corpus, + format::IndexConfig::kDocsPositions)); + + uint64_t expected_total = 0; + for (const KnownTerm& kt : corpus) { + const format::DictEntry entry = lookup_entry(index_reader, kt.term); + EXPECT_EQ(entry.df, kt.df()) << kt.term; + EXPECT_EQ(entry.ttf_delta, kt.ref_sum()) << kt.term; + EXPECT_EQ(entry.max_freq, kt.ref_max()) << kt.term; + expected_total += kt.ref_sum(); + } + EXPECT_EQ(index_reader.stats().sum_total_term_freq, expected_total); +} + +// FW-DF-windowed / FW-DF-slim-inline (+ slim pod_ref): the fused ttf/max are +// correct on each encoding branch. Also guards that the windowed per-window MaxOf +// / sum (NOT collapsed by this task) still produce a term whose term-level fused +// values match the reference. +TEST(SniiWriterTest, FusedFreqStatsCorrectAcrossWindowedAndSlimEncodings) { + const std::vector corpus = make_known_corpus(); + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_corpus_reader(&file, &segment_reader, &index_reader, corpus, + format::IndexConfig::kDocsPositions)); + + const format::DictEntry wide = lookup_entry(index_reader, "wide"); + EXPECT_EQ(wide.enc, format::DictEntryEnc::kWindowed); + EXPECT_EQ(wide.ttf_delta, find_term(corpus, "wide").ref_sum()); + EXPECT_EQ(wide.max_freq, find_term(corpus, "wide").ref_max()); + EXPECT_EQ(wide.max_freq, 3U); // sanity: cycling 1,2,3 + + const format::DictEntry slimref = lookup_entry(index_reader, "slimref"); + EXPECT_EQ(slimref.enc, format::DictEntryEnc::kSlim); + EXPECT_EQ(slimref.kind, format::DictEntryKind::kPodRef); + EXPECT_EQ(slimref.ttf_delta, find_term(corpus, "slimref").ref_sum()); + EXPECT_EQ(slimref.max_freq, find_term(corpus, "slimref").ref_max()); + + const format::DictEntry tiny = lookup_entry(index_reader, "tiny"); + EXPECT_EQ(tiny.enc, format::DictEntryEnc::kSlim); + EXPECT_EQ(tiny.kind, format::DictEntryKind::kInline); + EXPECT_EQ(tiny.ttf_delta, find_term(corpus, "tiny").ref_sum()); + EXPECT_EQ(tiny.max_freq, find_term(corpus, "tiny").ref_max()); +} + +// ------------------------------------------------------------------------- +// Pure helper boundaries (real production helper via the testing seam). +// ------------------------------------------------------------------------- + +// FW-PURE-*: fuse_freq_stats on degenerate/boundary inputs equals the hand +// computed reference. max stays 0 for an all-zero input (a freq of 0 never lowers +// the running max), and the u32 sum promotes into the u64 total. +TEST(SniiWriterTest, FuseFreqStatsMatchesReferenceOnEdgeInputs) { + { + const FreqStats fs = testing::fuse_freq_stats_for_test({}); + EXPECT_EQ(fs.total_freq, 0U); + EXPECT_EQ(fs.max_freq, 0U); + } + { + const FreqStats fs = testing::fuse_freq_stats_for_test({7}); + EXPECT_EQ(fs.total_freq, 7U); + EXPECT_EQ(fs.max_freq, 7U); + } + { + const FreqStats fs = testing::fuse_freq_stats_for_test({0, 0, 0}); + EXPECT_EQ(fs.total_freq, 0U); + EXPECT_EQ(fs.max_freq, 0U); + } + { + const FreqStats fs = testing::fuse_freq_stats_for_test({5, 5, 5, 5}); + EXPECT_EQ(fs.total_freq, 20U); + EXPECT_EQ(fs.max_freq, 5U); + } + { + const FreqStats fs = testing::fuse_freq_stats_for_test({1, UINT32_MAX, 2}); + EXPECT_EQ(fs.total_freq, static_cast(UINT32_MAX) + 3U); + EXPECT_EQ(fs.max_freq, UINT32_MAX); + } +} + +// FW-PURE-rand: a large fixed-seed random input agrees with an independent +// std::accumulate (u64) / std::max_element reference. +TEST(SniiWriterTest, FuseFreqStatsMatchesNaiveOnLargeRandomInput) { + std::mt19937 rng(0xC0FFEEU); + std::vector freqs(4096); + for (uint32_t& f : freqs) { + f = static_cast(rng()); + } + + const FreqStats fs = testing::fuse_freq_stats_for_test(freqs); + EXPECT_EQ(fs.total_freq, std::accumulate(freqs.begin(), freqs.end(), uint64_t {0})); + EXPECT_EQ(fs.max_freq, *std::ranges::max_element(freqs)); +} + +// ------------------------------------------------------------------------- +// Error paths preserved after dropping validate_term's internal sum loop. +// ------------------------------------------------------------------------- + +// FW-VAL-len-mismatch: freqs.size() != docids.size() is still rejected (the +// length check is independent of the freqs sum and must remain). +TEST(SniiWriterTest, ValidateTermRejectsFreqDocidLengthMismatch) { + TermPostings tp; + tp.term = "x"; + tp.docids = {0, 1}; + tp.freqs = {1}; // length 1 != 2 + + SniiIndexInput input; + input.index_id = 1; + input.index_suffix = "Body"; + input.config = format::IndexConfig::kDocsOnly; + input.doc_count = 2; + input.terms = {std::move(tp)}; + + MemoryFile file; + SniiCompoundWriter writer(&file); + const Status status = writer.add_logical_index(input); + EXPECT_TRUE(status.is()) << status.to_string(); +} + +// FW-VAL-nonasc: non-strictly-ascending docids still rejected. +TEST(SniiWriterTest, ValidateTermRejectsNonAscendingDocids) { + TermPostings tp; + tp.term = "x"; + tp.docids = {5, 5}; // equal -> not strictly ascending + tp.freqs = {1, 1}; + + SniiIndexInput input; + input.index_id = 1; + input.index_suffix = "Body"; + input.config = format::IndexConfig::kDocsOnly; + input.doc_count = 6; + input.terms = {std::move(tp)}; + + MemoryFile file; + SniiCompoundWriter writer(&file); + const Status status = writer.add_logical_index(input); + EXPECT_TRUE(status.is()) << status.to_string(); +} + +// FW-VAL-prx: the has_prx position-count check now consumes the FUSED total. +// A mismatch (positions != sum(freqs)) is rejected; a match is accepted AND the +// ttf/max read back from the fused stats. The accepted sub-case is the guard that +// validate_term receives the correct total (a mis-wired/zero total would wrongly +// reject this valid term). +TEST(SniiWriterTest, ValidateTermUsesFusedTotalForPositionCount) { + SniiIndexInput base; + base.index_id = 1; + base.index_suffix = "Body"; + base.config = format::IndexConfig::kDocsPositions; + base.doc_count = 1; + + { + TermPostings tp; + tp.term = "x"; + tp.docids = {0}; + tp.freqs = {2}; // sum(freqs) = 2 + tp.positions_flat = {7}; // but only 1 position + SniiIndexInput input = base; + input.terms = {std::move(tp)}; + + MemoryFile file; + SniiCompoundWriter writer(&file); + const Status status = writer.add_logical_index(input); + EXPECT_TRUE(status.is()) << status.to_string(); + } + { + TermPostings tp; + tp.term = "x"; + tp.docids = {0}; + tp.freqs = {2}; // sum(freqs) = 2 + tp.positions_flat = {7, 8}; // matches + SniiIndexInput input = base; + input.terms = {std::move(tp)}; + + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + SniiCompoundWriter writer(&file); + assert_ok(writer.add_logical_index(input)); + assert_ok(writer.finish()); + assert_ok(reader::SniiSegmentReader::open(&file, &segment_reader)); + assert_ok(segment_reader.open_index(input.index_id, input.index_suffix, &index_reader)); + + const format::DictEntry entry = lookup_entry(index_reader, "x"); + EXPECT_EQ(entry.ttf_delta, 2U); + EXPECT_EQ(entry.max_freq, 2U); + } +} + +} // namespace +} // namespace doris::snii::writer diff --git a/be/test/storage/index/snii/writer/memory_reporter_test.cpp b/be/test/storage/index/snii/writer/memory_reporter_test.cpp new file mode 100644 index 00000000000000..b48f543de3ce5c --- /dev/null +++ b/be/test/storage/index/snii/writer/memory_reporter_test.cpp @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/memory_reporter.h" + +#include + +#include +#include + +#include "common/status.h" + +using doris::snii::writer::MemoryReporter; + +TEST(SniiMemoryReporter, StartsAtZero) { + MemoryReporter reporter; // null callback + EXPECT_EQ(reporter.current_bytes(), 0); +} + +TEST(SniiMemoryReporter, TracksPositiveAndNegativeDeltas) { + MemoryReporter reporter; + reporter.report(+100); + reporter.report(+50); + EXPECT_EQ(reporter.current_bytes(), 150); + reporter.report(-30); + EXPECT_EQ(reporter.current_bytes(), 120); + reporter.report(-120); + EXPECT_EQ(reporter.current_bytes(), 0); +} + +TEST(SniiMemoryReporter, NullCallbackIsSafe) { + MemoryReporter reporter(nullptr); // explicit null ConsumeReleaseFn + reporter.report(+10); + reporter.report(-10); + EXPECT_EQ(reporter.current_bytes(), 0); +} + +TEST(SniiMemoryReporter, CallbackFiresWithSameDelta) { + std::vector sink; + MemoryReporter reporter([&sink](int64_t delta) { sink.push_back(delta); }); + reporter.report(+100); + reporter.report(-40); + ASSERT_EQ(sink.size(), 2U); + EXPECT_EQ(sink[0], 100); + EXPECT_EQ(sink[1], -40); + EXPECT_EQ(reporter.current_bytes(), 60); +} + +TEST(SniiMemoryReporter, CallbackSumMirrorsCurrentBytes) { + int64_t external_total = 0; + MemoryReporter reporter([&external_total](int64_t delta) { external_total += delta; }); + reporter.report(+100); + reporter.report(+250); + reporter.report(-75); + reporter.report(+12); + reporter.report(-200); + EXPECT_EQ(external_total, reporter.current_bytes()); +} + +TEST(SniiMemoryReporter, ZeroDeltaIsNoOpButStillReports) { + int fire_count = 0; + MemoryReporter reporter([&fire_count](int64_t) { ++fire_count; }); + reporter.report(+100); + EXPECT_EQ(reporter.current_bytes(), 100); + reporter.report(0); + EXPECT_EQ(reporter.current_bytes(), 100); + EXPECT_EQ(fire_count, 2); // report(0) still fires the callback once +} diff --git a/be/test/storage/index/snii/writer/memory_reporter_wiring_test.cpp b/be/test/storage/index/snii/writer/memory_reporter_wiring_test.cpp new file mode 100644 index 00000000000000..df912caf4cf26f --- /dev/null +++ b/be/test/storage/index/snii/writer/memory_reporter_wiring_test.cpp @@ -0,0 +1,243 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// P1 wiring + P3 seam: verifies that the writer-level MemoryReporter sees the REAL +// resident-byte deltas reported by SpillableByteBuffer (dict side: ram_bytes_) and +// SpimiTermBuffer (SPIMI side: arena_bytes() + slot_of_.capacity()*4), positive on +// grow and negative on every free, exactly once -- so the counter returns to a +// known measurement after a build and to 0 after a full drain (a missed negative +// would be a Doris LOAD-tracker leak). The reporter must NEVER count live_bytes_. + +#include + +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/io/file_writer.h" +#include "storage/index/snii/writer/memory_reporter.h" +#include "storage/index/snii/writer/spillable_byte_buffer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +namespace doris::snii::writer { +using doris::Status; +namespace { + +// Discards appended bytes; only counts how many were written (SpillableByteBuffer +// stream_into / spill targets need a sink, but these tests check the reporter). +class NullWriter : public doris::snii::io::FileWriter { +public: + Status append(Slice s) override { + written_ += s.size(); + return Status::OK(); + } + Status finalize() override { return Status::OK(); } + uint64_t bytes_written() const override { return written_; } + +private: + uint64_t written_ = 0; +}; + +std::vector Block(size_t n, uint8_t seed) { + std::vector v(n); + for (size_t i = 0; i < n; ++i) { + v[i] = static_cast((i + seed) & 0xFF); + } + return v; +} + +// ---- dict side (SpillableByteBuffer) --------------------------------------- + +// Every RAM append reports exactly its byte count; the reporter equals the +// RAM-resident size, which (un-spilled) equals buf.size(). +TEST(SniiMemoryReporterWiring, DictRamDeltasMatchSize) { + MemoryReporter reporter; + SpillableByteBuffer buf(/*cap=*/UINT64_MAX, "dict", &reporter); + int64_t total = 0; + for (int b = 0; b < 5; ++b) { + auto chunk = Block(1000 + b, static_cast(b)); + total += static_cast(chunk.size()); + ASSERT_TRUE(buf.append(Slice(chunk)).ok()); + EXPECT_EQ(reporter.current_bytes(), total); + } + // Move-append path reports identically. + auto moved = Block(777, 9); + total += 777; + ASSERT_TRUE(buf.append_move(std::move(moved)).ok()); + EXPECT_EQ(reporter.current_bytes(), total); + EXPECT_EQ(reporter.current_bytes(), static_cast(buf.size())); +} + +// Destroying an UN-spilled buffer must release its resident RAM (the common path: +// most dict buffers never spill). A missed negative here would leak into Doris's +// LOAD MemTracker. After the buffer's scope, the reporter must be back to 0. +TEST(SniiMemoryReporterWiring, DictDtorReleasesUnspilledRam) { + MemoryReporter reporter; + { + SpillableByteBuffer buf(/*cap=*/UINT64_MAX, "dict", &reporter); + ASSERT_TRUE(buf.append(Slice(Block(5000, 1))).ok()); + ASSERT_TRUE(buf.append_move(Block(3000, 2)).ok()); + EXPECT_EQ(reporter.current_bytes(), 8000); + EXPECT_FALSE(buf.spilled()); + } // ~SpillableByteBuffer must report -8000 + EXPECT_EQ(reporter.current_bytes(), 0); +} + +// UNIFIED gate-2: a small cap on the shared reporter triggers the dict spill BEFORE +// its (huge) local cap_bytes_ is reached -- proving the spill is driven by the writer's +// total RAM (reporter->over_cap()), not a per-buffer threshold. +TEST(SniiMemoryReporterWiring, UnifiedCapDrivesDictSpill) { + MemoryReporter reporter(/*consume_release=*/nullptr, /*cap_bytes=*/8000); + SpillableByteBuffer buf(/*local cap=*/UINT64_MAX, "dict", &reporter); + ASSERT_TRUE(buf.append(Slice(Block(5000, 1))).ok()); + EXPECT_FALSE(buf.spilled()); // 5000 < unified cap 8000 + ASSERT_TRUE(buf.append(Slice(Block(4000, 2))).ok()); // 9000 >= 8000 -> spill + EXPECT_TRUE(buf.spilled()); // unified cap fired, not local + EXPECT_EQ(reporter.current_bytes(), 0); // resident handed to disk + ASSERT_TRUE(buf.seal().ok()); +} + +// A spill drops the resident tier: the reporter returns to 0 (one negative == +// prior ram_bytes_) while buf.size() still counts the on-disk bytes. +TEST(SniiMemoryReporterWiring, DictSpillFreesRam) { + MemoryReporter reporter; + SpillableByteBuffer buf(/*cap=*/8192, "dict", &reporter); + uint64_t total = 0; + for (int b = 0; b < 8; ++b) { // 8 x 4 KiB through an 8 KiB cap -> spills + auto chunk = Block(4096, static_cast(b)); + total += chunk.size(); + ASSERT_TRUE(buf.append(Slice(chunk)).ok()); + } + EXPECT_TRUE(buf.spilled()); + // RAM tier was handed to disk -> reporter back to 0; size() still totals bytes. + EXPECT_EQ(reporter.current_bytes(), 0); + EXPECT_EQ(buf.size(), total); + ASSERT_TRUE(buf.seal().ok()); +} + +// ---- SPIMI side (SpimiTermBuffer) ------------------------------------------ + +// A null reporter must not affect behavior (default off-Doris path). +TEST(SniiMemoryReporterWiring, SpimiNullReporterIsSafe) { + std::vector vocab = {"a", "b", "c"}; + SpimiTermBuffer buf(&vocab, /*has_positions=*/true, /*spill=*/0, /*reporter=*/nullptr); + buf.add_token(0, 0, 0); + buf.add_token(1, 0, 1); + auto terms = buf.finalize_sorted(); + EXPECT_EQ(terms.size(), 2U); +} + +// Borrowed-vocab construction reports the vocab-sized slot index up front +// (slot_of_.capacity()*4). assign(N,0) makes capacity == N, so the initial report +// is exactly N*4 -- a directly-measurable accuracy check. +TEST(SniiMemoryReporterWiring, SpimiInitialSlotIndexIsExact) { + std::vector vocab(64, "x"); + MemoryReporter reporter; + SpimiTermBuffer buf(&vocab, /*has_positions=*/true, /*spill=*/0, &reporter); + EXPECT_EQ(reporter.current_bytes(), static_cast(vocab.size()) * 4); +} + +// Adding tokens grows the reporter ABOVE the initial slot-index figure (the arena +// is now non-empty), and a full drain returns it to 0 -- every negative reported, +// no leak. Also proves the counter tracks arena_bytes(), not the (default-mode) +// live_bytes_ estimate, which stays 0 when spill_threshold_bytes_ == 0. +TEST(SniiMemoryReporterWiring, SpimiGrowThenDrainNetZero) { + std::vector vocab = {"alpha", "beta", "gamma", "delta"}; + MemoryReporter reporter; + SpimiTermBuffer buf(&vocab, /*has_positions=*/true, /*spill=*/0, &reporter); + const int64_t base = reporter.current_bytes(); // slot index only + EXPECT_EQ(base, static_cast(vocab.size()) * 4); + for (uint32_t doc = 0; doc < 200; ++doc) { + for (uint32_t t = 0; t < 4; ++t) { + buf.add_token(t, doc, /*pos=*/0); + } + } + // Arena is now resident -> strictly above the slot-index-only baseline. Because + // spill_threshold_bytes_ == 0, live_bytes_ is never accumulated; a counter that + // mistakenly reported live_bytes_ would still be at `base` here. + EXPECT_GT(reporter.current_bytes(), base); + auto terms = buf.finalize_sorted(); + EXPECT_EQ(terms.size(), 4U); + // Drain freed the arena + slot index: every negative reported -> exactly 0. + EXPECT_EQ(reporter.current_bytes(), 0); +} + +// Streaming drain (for_each_term_sorted) frees identically -> net zero. +TEST(SniiMemoryReporterWiring, SpimiStreamingDrainNetZero) { + std::vector vocab = {"k0", "k1", "k2"}; + MemoryReporter reporter; + SpimiTermBuffer buf(&vocab, /*has_positions=*/false, /*spill=*/0, &reporter); + for (uint32_t doc = 0; doc < 100; ++doc) { + for (uint32_t t = 0; t < 3; ++t) { + buf.add_token(t, doc, 0); + } + } + EXPECT_GT(reporter.current_bytes(), 0); + size_t seen = 0; + // Integrated for_each_term_sorted returns a [[nodiscard]] doris::Status (drift + // from the standalone void); consume it and assert the happy-path drain succeeds. + ASSERT_TRUE(buf.for_each_term_sorted([&seen](TermPostings&&) { ++seen; }).ok()); + EXPECT_EQ(seen, 3U); + EXPECT_EQ(reporter.current_bytes(), 0); +} + +// The spilled finalize path (merge_runs) must also balance: a small spill threshold +// forces at least one spill mid-build; after the full drain the reporter is 0 (the +// spill's arena-reset negative AND merge_runs' slot_of_ free are both reported). +TEST(SniiMemoryReporterWiring, SpimiSpillPathNetZero) { + std::vector vocab = {"w0", "w1", "w2", "w3", "w4", "w5", "w6", "w7"}; + // Tiny UNIFIED cap on the reporter so the REAL resident size (arena + slot index) + // crosses it early and forces spills, exercising the drain_to_writer / merge_runs + // free sites. (The local spill_threshold arg is unused once a reporter is attached.) + MemoryReporter reporter(/*consume_release=*/nullptr, /*cap_bytes=*/256); + SpimiTermBuffer buf(&vocab, /*has_positions=*/true, /*spill=*/0, &reporter); + for (uint32_t doc = 0; doc < 500; ++doc) { + for (uint32_t t = 0; t < 8; ++t) { + buf.add_token(t, doc, doc & 7); + } + } + size_t seen = 0; + // for_each_term_sorted now surfaces spill/merge I/O via a [[nodiscard]] Status. + ASSERT_TRUE(buf.for_each_term_sorted([&seen](TermPostings&&) { ++seen; }).ok()); + EXPECT_TRUE(buf.status().ok()); + EXPECT_EQ(seen, 8U); + // All spill/merge frees reported -> back to 0 (no MemTracker leak). + EXPECT_EQ(reporter.current_bytes(), 0); +} + +// Destructor balance: a buffer destroyed WITHOUT a drain (e.g. an aborted build) +// must return its reported resident bytes so nothing leaks in the tracker. +TEST(SniiMemoryReporterWiring, SpimiDestructorBalancesOnAbort) { + std::vector vocab = {"p", "q", "r"}; + MemoryReporter reporter; + { + SpimiTermBuffer buf(&vocab, /*has_positions=*/true, /*spill=*/0, &reporter); + for (uint32_t doc = 0; doc < 50; ++doc) { + for (uint32_t t = 0; t < 3; ++t) { + buf.add_token(t, doc, 0); + } + } + EXPECT_GT(reporter.current_bytes(), 0); + // No drain: leave the build incomplete and let the destructor run. + } + EXPECT_EQ(reporter.current_bytes(), 0); +} + +} // namespace +} // namespace doris::snii::writer diff --git a/be/test/storage/index/snii/writer/phase_a_readback_test.cpp b/be/test/storage/index/snii/writer/phase_a_readback_test.cpp new file mode 100644 index 00000000000000..f53b601804b5bf --- /dev/null +++ b/be/test/storage/index/snii/writer/phase_a_readback_test.cpp @@ -0,0 +1,297 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/io/metered_file_reader.h" +#include "storage/index/snii/query/bm25_scorer.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/scoring_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/stats/snii_stats_provider.h" +#include "storage/index/snii/writer/logical_index_writer.h" +#include "storage/index/snii/writer/snii_compound_writer.h" + +// PHASE A read-back self-validation (design 2026-06-18-snii-read-byte-optimizations +// sections 1 + 3 + 4). Builds a scoring index with a high-df term that spans +// multiple ADAPTIVE-size windows plus low-df terms, then asserts: +// (a) for every window, decoding ONLY the docs-only prefix slice +// [frq window start, +frq_docs_len) via read_frq_window_docs yields exactly +// the same docids as decoding the full window; +// (b) sum of window doc_counts == df and the windows tile the posting in order; +// (c) max_norm is non-zero for windows whose docs have non-default norms and +// equals the tightest (smallest encoded) norm in the window; +// (d) term_query / phrase_query / scoring (exhaustive vs WAND) agree with the +// oracle docids. +namespace { + +using namespace doris::snii; // NOLINT +using namespace doris::snii::format; // NOLINT +using namespace doris::snii::writer; // NOLINT +using doris::snii::query::Bm25Params; +using doris::snii::query::ScoredDoc; +using doris::snii::stats::SniiStatsProvider; + +std::string TempPath() { + static int counter = 0; + return "/tmp/snii_phaseA_readback_" + std::to_string(getpid()) + "_" + + std::to_string(counter++) + ".idx"; +} + +// Corpus: every doc has "hot" (very high df -> adaptive 1024-doc windows). "spark" +// follows "hot" consecutively in docs where d % 7 == 0. "rare" is a tiny term. +// Doc lengths vary by docid so encoded norms differ across windows and within a +// window, exercising real max_norm. +struct Corpus { + uint32_t doc_count = 12000; // df(hot)=12000 >= kAdaptiveWindowDfThreshold(8192) + std::vector hot_docs; // every doc + std::vector spark_docs; // d % 7 == 0 + std::vector rare_docs = {5, 91, 4096, 11999}; + std::vector phrase_oracle; // docs with consecutive "hot spark" + std::vector doc_len; // per-doc length (drives encoded norm) +}; + +// Per-doc length: varies so encoded norms are non-default (>1) and differ; the +// minimum within each 1024-doc window is deterministic. +uint64_t DocLen(uint32_t d) { + return 2 + (d % 250); +} + +Corpus MakeCorpus() { + Corpus c; + c.doc_len.assign(c.doc_count, 0); + for (uint32_t d = 0; d < c.doc_count; ++d) { + c.hot_docs.push_back(d); + c.doc_len[d] = DocLen(d); + if (d % 7 == 0) { + c.spark_docs.push_back(d); + c.phrase_oracle.push_back(d); + } + } + return c; +} + +TermPostings MakePosTerm(const std::string& term, const std::vector& docs, uint32_t pos) { + TermPostings tp; + tp.term = term; + tp.docids = docs; + tp.freqs.assign(docs.size(), 1); + tp.positions_flat.assign(docs.size(), pos); // one position per doc, flat + return tp; +} + +SniiIndexInput MakeIndex(const Corpus& c) { + SniiIndexInput in; + in.index_id = 1; + in.index_suffix = "body"; + in.config = IndexConfig::kDocsPositionsScoring; // scoring -> norms available + in.doc_count = c.doc_count; + in.target_dict_block_bytes = 1; // one block per term + in.encoded_norms.resize(c.doc_count); + for (uint32_t d = 0; d < c.doc_count; ++d) { + in.encoded_norms[d] = doris::snii::query::encode_norm(c.doc_len[d]); + } + // Lexicographically sorted: hot < rare < spark. + in.terms.push_back(MakePosTerm("hot", c.hot_docs, /*pos=*/0)); + in.terms.push_back(MakePosTerm("rare", c.rare_docs, /*pos=*/0)); + in.terms.push_back(MakePosTerm("spark", c.spark_docs, /*pos=*/1)); + return in; +} + +// Smallest encoded norm across [start, end) docids (the tightest WAND norm). +uint8_t WindowMinNorm(const std::vector& norms, uint32_t start, uint32_t end) { + uint8_t best = std::numeric_limits::max(); + for (uint32_t d = start; d < end; ++d) { + best = std::min(best, norms[d]); + } + return best; +} + +} // namespace + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiPhaseAReadBack, DocsPrefixTilesAndMaxNormTightAndQueriesAgree) { + const Corpus c = MakeCorpus(); + std::vector norms(c.doc_count); + for (uint32_t d = 0; d < c.doc_count; ++d) { + norms[d] = doris::snii::query::encode_norm(c.doc_len[d]); + } + + const std::string path = TempPath(); + { + io::LocalFileWriter w; + ASSERT_TRUE(w.open(path).ok()); + SniiCompoundWriter cw(&w); + ASSERT_TRUE(cw.add_logical_index(MakeIndex(c)).ok()); + ASSERT_TRUE(cw.finish().ok()); + } + + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + io::MeteredFileReader metered(&local); + reader::SniiSegmentReader seg; + ASSERT_TRUE(reader::SniiSegmentReader::open(&metered, &seg).ok()); + reader::LogicalIndexReader idx; + ASSERT_TRUE(seg.open_index(1, "body", &idx).ok()); + + // Resolve "hot" -> windowed DictEntry + the block's frq_base. + bool found = false; + DictEntry hot; + uint64_t frq_base = 0, prx_base = 0; + ASSERT_TRUE(idx.lookup("hot", &found, &hot, &frq_base, &prx_base).ok()); + ASSERT_TRUE(found); + EXPECT_EQ(hot.df, c.doc_count); + EXPECT_EQ(hot.enc, DictEntryEnc::kWindowed); + EXPECT_TRUE(hot.has_sb); + + const uint64_t prelude_abs = + idx.section_refs().posting_region.offset + frq_base + hot.frq_off_delta; + std::vector prelude_bytes; + ASSERT_TRUE(local.read_at(prelude_abs, hot.prelude_len, &prelude_bytes).ok()); + FrqPreludeReader prelude; + ASSERT_TRUE(FrqPreludeReader::open(Slice(prelude_bytes), &prelude).ok()); + + // Adaptive sizing: df=12000 -> 1024-doc windows -> >= 2 windows. + ASSERT_GT(prelude.n_windows(), 1U); + // Most windows are the adaptive size (the last may be a remainder). + WindowMeta w0; + ASSERT_TRUE(prelude.window(0, &w0).ok()); + EXPECT_EQ(w0.doc_count, doris::snii::format::kAdaptiveWindowDocs); + + // The grouped layout puts all dd regions in the dd-block right after the prelude. + const uint64_t dd_block_start = prelude_abs + hot.prelude_len; + std::vector tiled; + uint64_t summed = 0; + uint64_t expect_win_base = 0; + uint64_t expect_dd_off = 0; + uint32_t doc_cursor = 0; + bool any_nonzero_max_norm = false; + + for (uint32_t w = 0; w < prelude.n_windows(); ++w) { + WindowMeta m; + ASSERT_TRUE(prelude.window(w, &m).ok()); + EXPECT_EQ(m.win_base, expect_win_base) << "win_base w=" << w; + ASSERT_GT(m.dd_disk_len, 0U) << "w=" << w; + // (a) dd regions are contiguous in the dd-block (the docs-only run is one range). + EXPECT_EQ(m.dd_off, expect_dd_off) << "dd_off contiguity w=" << w; + + // Decode the window's dd region ALONE (the freq region bytes are NOT fetched). + FrqRegionMeta dd_meta; + dd_meta.zstd = m.dd_zstd; + dd_meta.uncomp_len = m.dd_uncomp_len; + dd_meta.disk_len = m.dd_disk_len; + dd_meta.crc = m.crc_dd; + std::vector dd_bytes; + ASSERT_TRUE(local.read_at(dd_block_start + m.dd_off, m.dd_disk_len, &dd_bytes).ok()); + std::vector dd_docs; + ASSERT_TRUE(decode_dd_region(Slice(dd_bytes), dd_meta, m.win_base, &dd_docs).ok()) + << "dd decode w=" << w; + ASSERT_EQ(dd_docs.size(), m.doc_count) << "w=" << w; + + // (c) max_norm equals the tightest (smallest encoded) norm in the window. + const uint32_t exp_min_norm = WindowMinNorm(norms, doc_cursor, doc_cursor + m.doc_count); + EXPECT_EQ(m.max_norm, exp_min_norm) << "max_norm w=" << w; + if (m.max_norm != 0) { + any_nonzero_max_norm = true; + } + + // (b) tiling: doc_counts sum and concatenated docids stay ascending. + summed += m.doc_count; + expect_win_base = m.last_docid; + expect_dd_off += m.dd_disk_len; + doc_cursor += m.doc_count; + tiled.insert(tiled.end(), dd_docs.begin(), dd_docs.end()); + } + // The dd-block length equals the sum of per-window dd region lengths. + EXPECT_EQ(prelude.dd_block_len(), expect_dd_off); + + // (b) windows tile the whole posting [0, doc_count). + EXPECT_EQ(summed, c.doc_count); + ASSERT_EQ(tiled.size(), c.doc_count); + EXPECT_EQ(tiled.front(), 0U); + EXPECT_EQ(tiled.back(), c.doc_count - 1); + for (size_t i = 1; i < tiled.size(); ++i) { + ASSERT_LT(tiled[i - 1], tiled[i]) << "non-ascending at " << i; + } + // (c) at least one window's docs have non-default norms -> non-zero max_norm. + EXPECT_TRUE(any_nonzero_max_norm); + + // --- slim/inline term frq_docs_len plumbing ('rare' is tiny -> inline) --- + bool rfound = false; + DictEntry rare; + uint64_t rf = 0, rp = 0; + ASSERT_TRUE(idx.lookup("rare", &rfound, &rare, &rf, &rp).ok()); + ASSERT_TRUE(rfound); + EXPECT_EQ(rare.df, c.rare_docs.size()); + + // (d) term_query returns exactly the oracle docid set for each term. + std::vector hot_q; + ASSERT_TRUE(doris::snii::query::term_query(idx, "hot", &hot_q).ok()); + EXPECT_EQ(hot_q, c.hot_docs); + + std::vector rare_q; + ASSERT_TRUE(doris::snii::query::term_query(idx, "rare", &rare_q).ok()); + EXPECT_EQ(rare_q, c.rare_docs); + + std::vector spark_q; + ASSERT_TRUE(doris::snii::query::term_query(idx, "spark", &spark_q).ok()); + EXPECT_EQ(spark_q, c.spark_docs); + + // (d) phrase_query "hot spark" returns exactly the consecutive-occurrence docs. + std::vector phrase_q; + ASSERT_TRUE(doris::snii::query::phrase_query(idx, {"hot", "spark"}, &phrase_q).ok()); + EXPECT_EQ(phrase_q, c.phrase_oracle); + + // (d) scoring: WAND top-K == exhaustive top-K (uses the real per-window max_norm). + SniiStatsProvider stats; + ASSERT_TRUE(SniiStatsProvider::open(&idx, &stats).ok()); + const Bm25Params params; + for (uint32_t k : {1U, 5U, 50U}) { + std::vector ex, wa; + ASSERT_TRUE(doris::snii::query::scoring_query_exhaustive(idx, stats, {"hot", "rare"}, k, + params, &ex) + .ok()); + ASSERT_TRUE( + doris::snii::query::scoring_query_wand(idx, stats, {"hot", "rare"}, k, params, &wa) + .ok()); + ASSERT_EQ(wa.size(), ex.size()) << "k=" << k; + for (size_t i = 0; i < ex.size(); ++i) { + EXPECT_EQ(wa[i].docid, ex[i].docid) << "k=" << k << " i=" << i; + EXPECT_NEAR(wa[i].score, ex[i].score, 1e-9) << "k=" << k << " i=" << i; + } + } + + std::remove(path.c_str()); +} diff --git a/be/test/storage/index/snii/writer/posting_interleave_test.cpp b/be/test/storage/index/snii/writer/posting_interleave_test.cpp new file mode 100644 index 00000000000000..ebe1544c7c96de --- /dev/null +++ b/be/test/storage/index/snii/writer/posting_interleave_test.cpp @@ -0,0 +1,559 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/format/dict_block.h" +#include "storage/index/snii/format/dict_block_directory.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/per_index_meta.h" +#include "storage/index/snii/format/sampled_term_index.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/reader/windowed_posting.h" +#include "storage/index/snii/writer/logical_index_writer.h" +#include "storage/index/snii/writer/snii_compound_writer.h" + +// Interleaved posting-region read-back validation (docs/design/frqprx-interleave- +// design.md section 10.2). The former separate .frq POD and .prx POD are merged +// into ONE posting region in which each pod_ref term writes [prx span][frq span] +// contiguously, in term order. These tests assert the writer/reader contract for +// that layout: per-term contiguity (frq_off_delta == prx_off_delta + prx_len when +// has_prx), independent delta resolution, byte-correct sub-ranges, docs-only-prefix +// containment inside the frq span, multi-index isolation, empty-index tier recovery +// from the persisted flag, INLINE-between-pod_ref gaplessness, and the R1 docs-only- +// with-pod_ref tier-recovery regression guard. +namespace { + +using namespace doris::snii; // NOLINT +using namespace doris::snii::format; // NOLINT +using namespace doris::snii::writer; // NOLINT +using doris::snii::reader::DecodedPosting; +using doris::snii::reader::LogicalIndexReader; +using doris::snii::reader::SniiSegmentReader; +using doris::snii::reader::read_windowed_posting; + +std::string TempPath() { + static int counter = 0; + return "/tmp/snii_posting_interleave_" + std::to_string(getpid()) + "_" + + std::to_string(counter++) + ".idx"; +} + +TermPostings MakeTerm(const std::string& term, const std::vector& docs, + bool with_positions, uint32_t positions_per_doc) { + TermPostings tp; + tp.term = term; + tp.docids = docs; + tp.freqs.assign(docs.size(), positions_per_doc); + if (with_positions) { + for (size_t i = 0; i < docs.size(); ++i) { + for (uint32_t p = 0; p < positions_per_doc; ++p) { + tp.positions_flat.push_back(p); // deterministic ascending positions + } + } + } + return tp; +} + +// Writes one container with one logical index, returns its path. Keeps the +// SniiIndexInput's terms (the writer reads but does not retain them). +std::string WriteOne(const SniiIndexInput& in) { + const std::string path = TempPath(); + io::LocalFileWriter w; + EXPECT_TRUE(w.open(path).ok()); + SniiCompoundWriter cw(&w); + EXPECT_TRUE(cw.add_logical_index(in).ok()); + EXPECT_TRUE(cw.finish().ok()); + return path; +} + +// Enumerates every term's DictEntry + owning block frq/prx bases via prefix_terms +// (empty prefix = all terms, in lexicographic order). +std::vector AllEntries(const LogicalIndexReader& idx) { + std::vector out; + EXPECT_TRUE(idx.prefix_terms("", &out).ok()); + return out; +} + +SniiIndexInput BaseInput(uint64_t id, std::string suffix, IndexConfig cfg) { + SniiIndexInput in; + in.index_id = id; + in.index_suffix = std::move(suffix); + in.config = cfg; + in.target_dict_block_bytes = 256; // many small blocks -> exercise multiple bases + return in; +} + +} // namespace + +// Test #1: round-trip with positions (interleave correctness). Slim-pod_ref and +// windowed terms (df below and above kSlimDfThreshold). For each pod_ref term: +// - frq_off_delta == prx_off_delta + prx_len (gated on has_prx); +// - resolve_frq_window / resolve_prx_window land inside posting_region; +// - decoded docids/freqs/positions equal the input. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiPostingInterleave, RoundTripWithPositionsContiguous) { + SniiIndexInput in = BaseInput(1, "body", IndexConfig::kDocsPositions); + const uint32_t doc_count = 4096; + in.doc_count = doc_count; + + // windowed term (df=2000 >= kSlimDfThreshold=512), slim pod_ref (df=600 forced + // large enough to exceed the inline threshold via several positions/doc), and an + // INLINE tiny term -- all positions-bearing. + std::vector wide_docs, slim_docs; + for (uint32_t d = 0; d < 2000; ++d) { + wide_docs.push_back(d * 2); + } + for (uint32_t d = 0; d < 600; ++d) { + slim_docs.push_back(d * 3 + 1); + } + in.terms.push_back(MakeTerm("aa_wide", wide_docs, /*with_positions=*/true, 3)); + in.terms.push_back(MakeTerm("bb_slim", slim_docs, /*with_positions=*/true, 4)); + in.terms.push_back(MakeTerm("cc_tiny", {7, 19, 33}, /*with_positions=*/true, 1)); + + const std::string path = WriteOne(in); + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + SniiSegmentReader seg; + ASSERT_TRUE(SniiSegmentReader::open(&local, &seg).ok()); + LogicalIndexReader idx; + ASSERT_TRUE(seg.open_index(1, "body", &idx).ok()); + EXPECT_TRUE(idx.has_positions()); + + const RegionRef& region = idx.section_refs().posting_region; + for (const auto& hit : AllEntries(idx)) { + const DictEntry& e = hit.entry; + if (e.kind != DictEntryKind::kPodRef) { + continue; // INLINE term: nothing in region + } + // Contiguity property (gated on has_prx): frq span immediately follows prx span. + EXPECT_EQ(e.frq_off_delta, e.prx_off_delta + e.prx_len) << e.term; + + uint64_t foff = 0, flen = 0, poff = 0, plen = 0; + ASSERT_TRUE(idx.resolve_frq_window(e, hit.frq_base, &foff, &flen).ok()) << e.term; + ASSERT_TRUE(idx.resolve_prx_window(e, hit.prx_base, &poff, &plen).ok()) << e.term; + // Both windows land inside the single posting region. + EXPECT_GE(foff, region.offset) << e.term; + EXPECT_LE(foff + flen, region.offset + region.length) << e.term; + EXPECT_GE(poff, region.offset) << e.term; + EXPECT_LE(poff + plen, region.offset + region.length) << e.term; + } + + // Docids round-trip for every term. + for (const auto& [term, expect] : std::vector>> { + {"aa_wide", wide_docs}, {"bb_slim", slim_docs}, {"cc_tiny", {7, 19, 33}}}) { + std::vector got; + ASSERT_TRUE(query::term_query(idx, term, &got).ok()) << term; + EXPECT_EQ(got, expect) << term; + } + + // Positions round-trip for the windowed term via the full decode path. + bool found = false; + DictEntry wide_e; + uint64_t fb = 0, pb = 0; + ASSERT_TRUE(idx.lookup("aa_wide", &found, &wide_e, &fb, &pb).ok()); + ASSERT_TRUE(found); + DecodedPosting dp; + ASSERT_TRUE(read_windowed_posting(idx, wide_e, fb, pb, /*want_positions=*/true, + /*want_freq=*/true, &dp) + .ok()); + ASSERT_EQ(dp.docids.size(), wide_docs.size()); + EXPECT_EQ(dp.docids, wide_docs); + ASSERT_EQ(dp.positions.size(), wide_docs.size()); + for (const auto& pv : dp.positions) { + ASSERT_EQ(pv.size(), 3U); // positions_per_doc + EXPECT_EQ(pv[0], 0U); + EXPECT_EQ(pv[1], 1U); + EXPECT_EQ(pv[2], 2U); + } + std::remove(path.c_str()); +} + +// Test #2 / #3: byte-correctness of the combined region. Read the raw posting +// region bytes; assert each term's [prx_off_delta, +prx_len) and +// [frq_off_delta, +frq_len) sub-ranges, and that the docs-only prefix +// [frq_off_delta, +frq_docs_len) stays INSIDE the frq span (never reads prx). +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiPostingInterleave, ByteCorrectnessAndDocsPrefixStaysInFrqSpan) { + SniiIndexInput in = BaseInput(1, "body", IndexConfig::kDocsPositions); + in.doc_count = 4096; + std::vector wide_docs; + for (uint32_t d = 0; d < 1500; ++d) { + wide_docs.push_back(d); + } + in.terms.push_back(MakeTerm("aa_wide", wide_docs, /*with_positions=*/true, 2)); + std::vector slim_docs; + for (uint32_t d = 0; d < 600; ++d) { + slim_docs.push_back(d + 5); + } + in.terms.push_back(MakeTerm("bb_slim", slim_docs, /*with_positions=*/true, 3)); + + const std::string path = WriteOne(in); + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + SniiSegmentReader seg; + ASSERT_TRUE(SniiSegmentReader::open(&local, &seg).ok()); + LogicalIndexReader idx; + ASSERT_TRUE(seg.open_index(1, "body", &idx).ok()); + + const RegionRef& region = idx.section_refs().posting_region; + std::vector region_bytes; + ASSERT_TRUE(local.read_at(region.offset, region.length, ®ion_bytes).ok()); + + for (const auto& hit : AllEntries(idx)) { + const DictEntry& e = hit.entry; + if (e.kind != DictEntryKind::kPodRef) { + continue; + } + // prx span precedes frq span; both lie inside the region (relative to base). + const uint64_t prx_in = hit.prx_base + e.prx_off_delta; + const uint64_t frq_in = hit.frq_base + e.frq_off_delta; + ASSERT_LE(prx_in + e.prx_len, region.length) << e.term; + ASSERT_LE(frq_in + e.frq_len, region.length) << e.term; + // prx span ends exactly where frq span begins (contiguous, prx first). + EXPECT_EQ(prx_in + e.prx_len, frq_in) << e.term; + + // The docs-only prefix [frq_off_delta, +frq_docs_len) stays inside the frq span + // (begins after the prx span and fits within frq_len) -- it never reads prx. + EXPECT_GE(frq_in, prx_in + e.prx_len) << e.term; // prefix begins after prx + EXPECT_LE(e.frq_docs_len, e.frq_len) << e.term; // prefix fits in frq span + + // The absolute frq span stays inside the posting region (reader reads exactly + // what the writer wrote). + const uint64_t frq_span_abs = region.offset + frq_in; + EXPECT_LE(frq_span_abs + e.frq_len, region.offset + region.length) << e.term; + } + std::remove(path.c_str()); +} + +// Test #4: multi-index. One docs-positions index and one docs-only index in one +// container; each posting_region placement is independent (no offset bleed) and +// both resolve correctly. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiPostingInterleave, MultiIndexIndependentPlacements) { + SniiIndexInput a = BaseInput(1, "body", IndexConfig::kDocsPositions); + a.doc_count = 2048; + std::vector a_docs; + for (uint32_t d = 0; d < 1000; ++d) { + a_docs.push_back(d); + } + a.terms.push_back(MakeTerm("aa_wide", a_docs, /*with_positions=*/true, 2)); + + SniiIndexInput b = BaseInput(2, "tag", IndexConfig::kDocsOnly); + b.doc_count = 2048; + std::vector b_docs; + for (uint32_t d = 0; d < 800; ++d) { + b_docs.push_back(d * 2); + } + b.terms.push_back(MakeTerm("kk_keyword", b_docs, /*with_positions=*/false, 1)); + + const std::string path = TempPath(); + { + io::LocalFileWriter w; + ASSERT_TRUE(w.open(path).ok()); + SniiCompoundWriter cw(&w); + ASSERT_TRUE(cw.add_logical_index(a).ok()); + ASSERT_TRUE(cw.add_logical_index(b).ok()); + ASSERT_TRUE(cw.finish().ok()); + } + + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + SniiSegmentReader seg; + ASSERT_TRUE(SniiSegmentReader::open(&local, &seg).ok()); + + LogicalIndexReader ia, ib; + ASSERT_TRUE(seg.open_index(1, "body", &ia).ok()); + ASSERT_TRUE(seg.open_index(2, "tag", &ib).ok()); + EXPECT_TRUE(ia.has_positions()); + EXPECT_FALSE(ib.has_positions()); // docs-only recovered from the flag + EXPECT_EQ(ia.tier(), IndexTier::kT2); + EXPECT_EQ(ib.tier(), IndexTier::kT1); + + const RegionRef ra = ia.section_refs().posting_region; + const RegionRef rb = ib.section_refs().posting_region; + // Non-overlapping regions (independent placement; no cross-index bleed). + const bool disjoint = + (ra.offset + ra.length <= rb.offset) || (rb.offset + rb.length <= ra.offset); + EXPECT_TRUE(disjoint) << "ra=[" << ra.offset << "," << ra.length << "] rb=[" << rb.offset << "," + << rb.length << "]"; + + std::vector got_a, got_b; + ASSERT_TRUE(query::term_query(ia, "aa_wide", &got_a).ok()); + ASSERT_TRUE(query::term_query(ib, "kk_keyword", &got_b).ok()); + EXPECT_EQ(got_a, a_docs); + EXPECT_EQ(got_b, b_docs); + std::remove(path.c_str()); +} + +// Test #5: no-positions index -- the posting region holds only frq spans; +// prx_off_delta / prx_len are unset; the bytes are identical to today's .frq POD +// for the same input (a docs-only index's posting_region IS the frq concatenation, +// since the prx span is always empty). We verify prx spans are absent and the +// posting region tiles exactly the per-term frq spans with no gaps/prx. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiPostingInterleave, NoPositionsRegionIsFrqOnly) { + SniiIndexInput in = BaseInput(1, "tag", IndexConfig::kDocsOnly); + in.doc_count = 4096; + std::vector wide_docs, slim_docs; + for (uint32_t d = 0; d < 1500; ++d) { + wide_docs.push_back(d); + } + for (uint32_t d = 0; d < 600; ++d) { + slim_docs.push_back(d + 2); + } + in.terms.push_back(MakeTerm("aa_wide", wide_docs, /*with_positions=*/false, 1)); + in.terms.push_back(MakeTerm("bb_slim", slim_docs, /*with_positions=*/false, 1)); + + const std::string path = WriteOne(in); + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + SniiSegmentReader seg; + ASSERT_TRUE(SniiSegmentReader::open(&local, &seg).ok()); + LogicalIndexReader idx; + ASSERT_TRUE(seg.open_index(1, "tag", &idx).ok()); + EXPECT_FALSE(idx.has_positions()); + EXPECT_EQ(idx.tier(), IndexTier::kT1); + + const RegionRef region = idx.section_refs().posting_region; + uint64_t frq_bytes_sum = 0; + for (const auto& hit : AllEntries(idx)) { + const DictEntry& e = hit.entry; + if (e.kind != DictEntryKind::kPodRef) { + continue; + } + EXPECT_EQ(e.prx_len, 0U) << e.term; // no prx span at all + EXPECT_EQ(e.prx_off_delta, 0U) << e.term; + // frq span is the whole per-term posting (no preceding prx span). + EXPECT_EQ(hit.frq_base + e.frq_off_delta, frq_bytes_sum) << e.term; + frq_bytes_sum += e.frq_len; + } + // The frq spans tile the region exactly: no prx bytes, no gaps. + EXPECT_EQ(frq_bytes_sum, region.length); + + std::vector got; + ASSERT_TRUE(query::term_query(idx, "aa_wide", &got).ok()); + EXPECT_EQ(got, wide_docs); + std::remove(path.c_str()); +} + +// Test #6: empty term set. posting_region.length == 0 and dict_region.length == 0; +// the reader opens cleanly and lookups miss. Built BOTH docs-only and docs-positions +// to assert tier is recovered correctly from the flag despite both having a +// zero-length posting region. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiPostingInterleave, EmptyTermSetTierFromFlag) { + for (IndexConfig cfg : {IndexConfig::kDocsOnly, IndexConfig::kDocsPositions}) { + SniiIndexInput in = BaseInput(1, "body", cfg); + in.doc_count = 16; // docs exist, but no terms + const std::string path = WriteOne(in); + + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + SniiSegmentReader seg; + ASSERT_TRUE(SniiSegmentReader::open(&local, &seg).ok()); + LogicalIndexReader idx; + ASSERT_TRUE(seg.open_index(1, "body", &idx).ok()); + + EXPECT_EQ(idx.section_refs().posting_region.length, 0U); + EXPECT_EQ(idx.section_refs().dict_region.length, 0U); + const bool want_positions = (cfg != IndexConfig::kDocsOnly); + EXPECT_EQ(idx.has_positions(), want_positions); + EXPECT_EQ(idx.tier(), want_positions ? IndexTier::kT2 : IndexTier::kT1); + + std::vector got; + ASSERT_TRUE(query::term_query(idx, "anything", &got).ok()); + EXPECT_TRUE(got.empty()); + std::remove(path.c_str()); + } +} + +// Test #8: INLINE-between-pod_ref. Mix tiny (INLINE) and large (pod_ref) terms; +// INLINE terms append nothing to the region (region length == sum of pod_ref +// spans) and pod_ref deltas stay contiguous across the interleaving. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiPostingInterleave, InlineBetweenPodRefIsGapless) { + SniiIndexInput in = BaseInput(1, "body", IndexConfig::kDocsPositions); + in.doc_count = 4096; + std::vector wide1, wide2; + for (uint32_t d = 0; d < 1500; ++d) { + wide1.push_back(d); + } + for (uint32_t d = 0; d < 1200; ++d) { + wide2.push_back(d + 1); + } + // Lexicographic order interleaves tiny INLINE terms between the two windowed + // pod_ref terms: aa_big < bb_tiny < cc_big < dd_tiny. + in.terms.push_back(MakeTerm("aa_big", wide1, /*with_positions=*/true, 2)); + in.terms.push_back(MakeTerm("bb_tiny", {3, 8}, /*with_positions=*/true, 1)); + in.terms.push_back(MakeTerm("cc_big", wide2, /*with_positions=*/true, 2)); + in.terms.push_back(MakeTerm("dd_tiny", {1, 99}, /*with_positions=*/true, 1)); + + const std::string path = WriteOne(in); + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + SniiSegmentReader seg; + ASSERT_TRUE(SniiSegmentReader::open(&local, &seg).ok()); + LogicalIndexReader idx; + ASSERT_TRUE(seg.open_index(1, "body", &idx).ok()); + + const RegionRef region = idx.section_refs().posting_region; + uint64_t pod_span_sum = 0; + uint64_t cursor = 0; // running region offset of the next pod_ref term's span + for (const auto& hit : AllEntries(idx)) { + const DictEntry& e = hit.entry; + if (e.kind == DictEntryKind::kInline) { + EXPECT_EQ(e.prx_len, 0U) << e.term; // inline carries bytes in the entry + EXPECT_EQ(e.frq_len, 0U) << e.term; + continue; // appends nothing to the region + } + // pod_ref term's prx span begins exactly where the previous span ended (no gap + // introduced by the interleaved INLINE terms). + const uint64_t prx_in = hit.prx_base + e.prx_off_delta; + EXPECT_EQ(prx_in, cursor) << e.term; + const uint64_t term_span = e.prx_len + e.frq_len; + cursor += term_span; + pod_span_sum += term_span; + } + // Region length is exactly the sum of pod_ref spans (INLINE adds nothing). + EXPECT_EQ(pod_span_sum, region.length); + + // All terms still resolve. + for (const auto& [term, expect] : std::vector>> { + {"aa_big", wide1}, {"bb_tiny", {3, 8}}, {"cc_big", wide2}, {"dd_tiny", {1, 99}}}) { + std::vector got; + ASSERT_TRUE(query::term_query(idx, term, &got).ok()) << term; + EXPECT_EQ(got, expect) << term; + } + std::remove(path.c_str()); +} + +// Test #9: tier recovery for docs-only-with-pod_ref (R1 regression guard). A +// docs-only (T1) index containing a windowed pod_ref term so posting_region.length +// > 0. open_index must recover tier == kT1 / has_positions == false (NOT inferred +// from the non-zero region length), and a lookup + docid decode must succeed with +// NO spurious positions-flag parse (DictBlockReader::open does not InvalidArgument). +TEST(SniiPostingInterleave, DocsOnlyWithPodRefTierRecovery) { + SniiIndexInput in = BaseInput(1, "tag", IndexConfig::kDocsOnly); + in.doc_count = 4096; + std::vector wide_docs; + for (uint32_t d = 0; d < 2000; ++d) { + wide_docs.push_back(d); // windowed pod_ref + } + in.terms.push_back(MakeTerm("kk_keyword", wide_docs, /*with_positions=*/false, 1)); + + const std::string path = WriteOne(in); + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + SniiSegmentReader seg; + ASSERT_TRUE(SniiSegmentReader::open(&local, &seg).ok()); + LogicalIndexReader idx; + // This open would FAIL under the naive posting_region.length > 0 heuristic: + // has_positions would be wrongly true, DictBlockReader::open -> check_flags would + // hard-fail with InvalidArgument. With the persisted flag it opens cleanly. + ASSERT_TRUE(seg.open_index(1, "tag", &idx).ok()); + EXPECT_EQ(idx.tier(), IndexTier::kT1); + EXPECT_FALSE(idx.has_positions()); + EXPECT_GT(idx.section_refs().posting_region.length, 0U); // non-empty region + + // Lookup + docid decode succeeds (no spurious positions parse). + bool found = false; + DictEntry e; + uint64_t fb = 0, pb = 0; + ASSERT_TRUE(idx.lookup("kk_keyword", &found, &e, &fb, &pb).ok()); + ASSERT_TRUE(found); + EXPECT_EQ(e.kind, DictEntryKind::kPodRef); + EXPECT_EQ(e.prx_len, 0U); // docs-only: no prx span + std::vector got; + ASSERT_TRUE(query::term_query(idx, "kk_keyword", &got).ok()); + EXPECT_EQ(got, wide_docs); + std::remove(path.c_str()); +} + +// M1 (code-review blocker): the SLIM pod_ref path is the ONLY write path whose +// physical byte order flipped to [prx][frq], yet no prior test exercised it -- every +// "slim" term elsewhere is df>=512 (windowed) or tiny (inline). Build a genuine +// kSlim/kPodRef term (df<512 so build_slim_entry runs; docids spread so the PFOR'd +// frq exceeds the 256B inline threshold so it is kPodRef not kInline), lock the +// routing, assert the interleave invariant, and round-trip docids (frq span) AND +// positions (prx span, decoded via the slim phrase path). +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiPostingInterleave, SlimPodRefInterleaveAndPositionsRoundTrip) { + SniiIndexInput in = BaseInput(1, "body", IndexConfig::kDocsPositions); + in.doc_count = 30000; + std::vector docs; + for (uint32_t d = 0; d < 500; ++d) { + docs.push_back(d * 50 + 1); // df=500 (<512), spread + } + in.terms.push_back(MakeTerm("ss_aaa", docs, /*with_positions=*/true, 4)); + in.terms.push_back(MakeTerm("ss_bbb", docs, /*with_positions=*/true, 4)); + + const std::string path = WriteOne(in); + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + SniiSegmentReader seg; + ASSERT_TRUE(SniiSegmentReader::open(&local, &seg).ok()); + LogicalIndexReader idx; + ASSERT_TRUE(seg.open_index(1, "body", &idx).ok()); + ASSERT_TRUE(idx.has_positions()); + + bool found = false; + DictEntry e; + uint64_t fb = 0, pb = 0; + ASSERT_TRUE(idx.lookup("ss_aaa", &found, &e, &fb, &pb).ok()); + ASSERT_TRUE(found); + // Lock the routing onto the slim pod_ref path (the one whose order flipped). + ASSERT_EQ(e.kind, DictEntryKind::kPodRef) << "frq must exceed the inline threshold"; + ASSERT_EQ(e.enc, DictEntryEnc::kSlim) << "df must be below kSlimDfThreshold"; + + // Interleave invariant + adjacency: the prx span ends exactly where the frq begins. + EXPECT_EQ(e.frq_off_delta, e.prx_off_delta + e.prx_len); + const RegionRef& region = idx.section_refs().posting_region; + uint64_t foff = 0, flen = 0, poff = 0, plen = 0; + ASSERT_TRUE(idx.resolve_frq_window(e, fb, &foff, &flen).ok()); + ASSERT_TRUE(idx.resolve_prx_window(e, pb, &poff, &plen).ok()); + EXPECT_GE(poff, region.offset); + EXPECT_LE(foff + flen, region.offset + region.length); + EXPECT_EQ(poff + plen, foff) << "prx span must abut the frq span ([prx][frq])"; + + // Docids round-trip via term_query: decodes dd from the FRQ span -> proves the frq + // offset lands after the prx in the [prx][frq] layout. + std::vector got; + ASSERT_TRUE(query::term_query(idx, "ss_aaa", &got).ok()); + EXPECT_EQ(got, docs); + + // Positions round-trip via the SLIM read path: both terms sit at positions {0,1,2,3} + // in every shared doc, so "ss_aaa ss_bbb" matches (ss_aaa@0, ss_bbb@1) iff each + // term's prx window decodes correctly from its prx span. + std::vector phr; + ASSERT_TRUE(query::phrase_query(idx, {"ss_aaa", "ss_bbb"}, &phr).ok()); + EXPECT_EQ(phr, docs); + std::remove(path.c_str()); +} diff --git a/be/test/storage/index/snii/writer/snii_bigram_defer_wiring_test.cpp b/be/test/storage/index/snii/writer/snii_bigram_defer_wiring_test.cpp new file mode 100644 index 00000000000000..868841db1873c5 --- /dev/null +++ b/be/test/storage/index/snii/writer/snii_bigram_defer_wiring_test.cpp @@ -0,0 +1,367 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// SNII bigram-defer PRODUCTION-WIRING tests, one layer above +// snii_bigram_defer_writer_test.cpp: these drive the real ColumnWriter stack +// (ColumnWriter::create -> ScalarColumnWriter / ArrayColumnWriter ::init -> +// IndexColumnWriter::create -> set_direct_load forwarding) so that the +// `opts.is_direct_load` -> `set_direct_load(true)` hand-off in +// column_writer.cpp is what decides deferral for scalar indexes. ARRAY indexes +// deliberately keep their bigram build to preserve existing SNII full-build +// phrase behavior at array-element boundaries. +// The scenarios below pin both outcomes: +// 1. scalar direct load + config on -> no bigram sentinel; +// 2. ARRAY direct load + config on -> byte-identical to the full-build +// baseline, with a phrase query excluding a cross-element false match; +// 3. any not-direct load + config on -> byte-identical to the config-off +// baseline (compaction / schema change shape stays full). +// The segment-writer-level `write_type == TYPE_DIRECT` assignments feeding +// this flag are covered end-to-end in snii_bigram_defer_e2e_test.cpp. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/status.h" +#include "core/block/block.h" +#include "core/column/column_string.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_string.h" +#include "core/field.h" +#include "core/types.h" +#include "gen_cpp/segment_v2.pb.h" +#include "io/fs/file_writer.h" +#include "io/fs/local_file_system.h" +#include "storage/index/index_file_writer.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/iterator/olap_data_convertor.h" +#include "storage/segment/column_writer.h" +#include "storage/segment/variant/variant_column_writer_impl.h" // _init_column_meta +#include "storage/tablet/tablet_schema.h" + +namespace doris::segment_v2 { +namespace { + +constexpr int64_t kIndexId = 3; +constexpr const char* kTestDir = "./ut_dir/snii_bigram_defer_wiring_test"; + +void assert_ok(const Status& status) { + ASSERT_TRUE(status.ok()) << status.to_string(); +} + +// English-parser fulltext phrase index on column unique id 0 -- the shape whose +// hidden bigram build the direct-load deferral targets. +void init_phrase_index_meta(TabletIndex* meta) { + TabletIndexPB pb; + pb.set_index_type(IndexType::INVERTED); + pb.set_index_id(kIndexId); + pb.set_index_name("defer_wiring_idx"); + pb.add_col_unique_id(0); + pb.mutable_properties()->insert({"parser", "english"}); + pb.mutable_properties()->insert({"lower_case", "true"}); + pb.mutable_properties()->insert({"support_phrase", "true"}); + meta->init_from_pb(pb); +} + +TabletColumn make_string_column() { + TabletColumn column; + column.set_unique_id(0); + column.set_name("c1"); + column.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + column.set_length(INT_MAX); + column.set_is_nullable(false); + return column; +} + +TabletColumn make_array_string_column() { + TabletColumn array; + array.set_unique_id(0); + array.set_name("arr1"); + array.set_type(FieldType::OLAP_FIELD_TYPE_ARRAY); + array.set_length(0); + array.set_index_length(0); + array.set_is_nullable(false); + array.set_is_bf_column(false); + TabletColumn item; + item.set_unique_id(1); + item.set_name("arr_sub_string"); + item.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + item.set_length(INT_MAX); + array.add_sub_column(item); + return array; +} + +// Runs one column through the production ColumnWriter stack exactly as a +// segment writer would: create() (whose init() creates the index writer and +// forwards opts.is_direct_load via set_direct_load), append converted olap +// data, then the finish/write sequence and the index file close. The block +// column at position 0 must match `column`'s type. +void write_column_segment(const std::string& name, const TabletColumn& column, + const TabletIndex& index_meta, bool is_direct_load, const Block& block) { + const std::string dat_path = std::string(kTestDir) + "/" + name + ".dat"; + const std::string idx_path = std::string(kTestDir) + "/" + name + ".idx"; + io::FileWriterPtr dat_writer; + assert_ok(io::global_local_filesystem()->create_file(dat_path, &dat_writer)); + io::FileWriterPtr idx_file; + assert_ok(io::global_local_filesystem()->create_file(idx_path, &idx_file)); + IndexFileWriter index_file_writer(io::global_local_filesystem(), idx_path, "test_rowset", + /*seg_id=*/0, InvertedIndexStorageFormatPB::SNII, + std::move(idx_file), /*can_use_ram_dir=*/true, + /*tablet_id=*/300); + + ColumnMetaPB meta; + ColumnWriterOptions opts; + opts.meta = &meta; + opts.compression_type = CompressionTypePB::LZ4; + opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2; + opts.need_inverted_index = true; + opts.inverted_indexes = {&index_meta}; + opts.index_file_writer = &index_file_writer; + opts.is_direct_load = is_direct_load; // the wire under test + _init_column_meta(&meta, /*column_id=*/0, column, opts); + + std::unique_ptr writer; + assert_ok(ColumnWriter::create(opts, &column, dat_writer.get(), &writer)); + assert_ok(writer->init()); + + const size_t num_rows = block.rows(); + OlapBlockDataConvertor converter; + converter.add_column_data_convertor(column); + converter.set_source_content(&block, 0, num_rows); + auto [convert_status, accessor] = converter.convert_column_data(0); + assert_ok(convert_status); + assert_ok(writer->append(accessor->get_nullmap(), accessor->get_data(), num_rows)); + + assert_ok(writer->finish()); + assert_ok(writer->write_data()); + assert_ok(writer->write_ordinal_index()); + assert_ok(writer->write_inverted_index()); + assert_ok(index_file_writer.begin_close()); + assert_ok(index_file_writer.finish_close()); + assert_ok(dat_writer->close()); +} + +// 80 rows with the adjacent (hello, world) pair: df 80 sits inside the default +// prune survival window [64, 128], so a full build provably materializes the +// pair and the deferred segment's pair absence is a real difference. +Block make_string_block() { + auto column = ColumnString::create(); + for (uint32_t i = 0; i < 80; ++i) { + const std::string row = "hello world item" + std::to_string(i); + column->insert_data(row.data(), row.size()); + } + Block block; + block.insert({std::move(column), std::make_shared(), "c1"}); + return block; +} + +// 80 values carry the pair inside one array element, then one row splits the +// same terms across two elements. The pair df remains in the default survival +// window [64, 128], so a full build must exclude the split row from the phrase +// result using its hidden pair posting. +Block make_array_string_block() { + DataTypePtr array_type = std::make_shared(std::make_shared()); + MutableColumnPtr column = array_type->create_column(); + for (uint32_t i = 0; i < 80; ++i) { + Array row; + row.push_back( + Field::create_field(String("hello world item") + std::to_string(i))); + column->insert(Field::create_field(row)); + } + Array split_row; + split_row.push_back(Field::create_field(String("hello"))); + split_row.push_back(Field::create_field(String("world"))); + column->insert(Field::create_field(split_row)); + Block block; + block.insert({std::move(column), array_type, "arr1"}); + return block; +} + +bool lookup_found(const ::doris::snii::reader::LogicalIndexReader& idx, const std::string& term) { + bool found = false; + ::doris::snii::format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + EXPECT_TRUE(idx.lookup(term, &found, &entry, &frq_base, &prx_base).ok()); + return found; +} + +std::vector run_phrase(const ::doris::snii::reader::LogicalIndexReader& idx, + const std::vector& terms) { + std::vector docids; + EXPECT_TRUE(::doris::snii::query::phrase_query(idx, terms, &docids).ok()); + return docids; +} + +std::vector phrase_docids(size_t count) { + std::vector docids(count); + std::iota(docids.begin(), docids.end(), 0); + return docids; +} + +std::string read_file_bytes(const std::string& path) { + std::ifstream in(path, std::ios::binary); + EXPECT_TRUE(in.good()) << path; + std::ostringstream out; + out << in.rdbuf(); + return out.str(); +} + +void expect_file_bytes_equal(const std::string& path, const std::string& expected, + const std::string& shape) { + EXPECT_EQ(read_file_bytes(path), expected) << shape; +} + +void expect_deferred_bigram_layout(const ::doris::snii::reader::LogicalIndexReader& idx, + const std::string& shape) { + EXPECT_FALSE(lookup_found(idx, ::doris::snii::format::make_phrase_bigram_sentinel_term())) + << shape; + EXPECT_FALSE( + lookup_found(idx, ::doris::snii::format::make_phrase_bigram_term("hello", "world"))) + << shape; + EXPECT_TRUE(lookup_found(idx, "hello")) << shape; +} + +void expect_full_bigram_layout(const ::doris::snii::reader::LogicalIndexReader& idx, + const std::string& shape) { + EXPECT_TRUE(lookup_found(idx, ::doris::snii::format::make_phrase_bigram_sentinel_term())) + << shape; + EXPECT_TRUE(lookup_found(idx, ::doris::snii::format::make_phrase_bigram_term("hello", "world"))) + << shape; + EXPECT_TRUE(lookup_found(idx, "hello")) << shape; +} + +void expect_phrase_result(const ::doris::snii::reader::LogicalIndexReader& idx, + const std::vector& expected, const std::string& shape) { + EXPECT_EQ(run_phrase(idx, {"hello", "world"}), expected) << shape; +} + +class SniiBigramDeferWiringTest : public testing::Test { +protected: + void SetUp() override { + assert_ok(io::global_local_filesystem()->delete_directory(kTestDir)); + assert_ok(io::global_local_filesystem()->create_directory(kTestDir)); + _saved_defer = config::snii_bigram_defer_build_to_compaction; + _saved_prune_min_df = config::snii_bigram_prune_min_df; + _saved_prune_max_ratio = config::snii_bigram_prune_max_df_ratio; + init_phrase_index_meta(&_meta); + } + + void TearDown() override { + config::snii_bigram_defer_build_to_compaction = _saved_defer; + config::snii_bigram_prune_min_df = _saved_prune_min_df; + config::snii_bigram_prune_max_df_ratio = _saved_prune_max_ratio; + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(kTestDir).ok()); + } + + // A baseline (config off), then with the config ON one direct-load segment + // and one explicitly-not-direct segment through the same ColumnWriter + // stack. ARRAY indexes preserve the full-build baseline even when direct. + void run_wiring_scenario(const std::string& shape, const TabletColumn& column, + const Block& block, bool expect_direct_defer, + const std::vector& expected_phrase) { + config::snii_bigram_defer_build_to_compaction = false; + write_column_segment(shape + "_baseline", column, _meta, /*is_direct_load=*/false, block); + + config::snii_bigram_defer_build_to_compaction = true; + write_column_segment(shape + "_direct", column, _meta, /*is_direct_load=*/true, block); + write_column_segment(shape + "_not_direct", column, _meta, /*is_direct_load=*/false, block); + + const std::string baseline_path = std::string(kTestDir) + "/" + shape + "_baseline.idx"; + const std::string baseline_bytes = read_file_bytes(baseline_path); + ASSERT_FALSE(baseline_bytes.empty()); + + // Direct load + config on: scalars defer, while arrays retain the full + // build to preserve existing SNII full-build behavior at element + // boundaries. + ::doris::snii::io::LocalFileReader direct_file; + ::doris::snii::reader::SniiSegmentReader direct_segment; + ::doris::snii::reader::LogicalIndexReader direct_idx; + assert_ok(direct_file.open(std::string(kTestDir) + "/" + shape + "_direct.idx")); + assert_ok(::doris::snii::reader::SniiSegmentReader::open(&direct_file, &direct_segment)); + assert_ok(direct_segment.open_index(static_cast(kIndexId), /*index_suffix=*/"", + &direct_idx)); + if (expect_direct_defer) { + expect_deferred_bigram_layout(direct_idx, shape); + } else { + expect_file_bytes_equal(std::string(kTestDir) + "/" + shape + "_direct.idx", + baseline_bytes, shape); + expect_full_bigram_layout(direct_idx, shape); + } + expect_phrase_result(direct_idx, expected_phrase, shape); + + // Not a direct load, config on: full build, byte-identical to the + // config-off baseline. An inverted forward (deferring compaction / + // schema change instead of loads) fails here. + expect_file_bytes_equal(std::string(kTestDir) + "/" + shape + "_not_direct.idx", + baseline_bytes, shape); + ::doris::snii::io::LocalFileReader full_file; + ::doris::snii::reader::SniiSegmentReader full_segment; + ::doris::snii::reader::LogicalIndexReader full_idx; + assert_ok(full_file.open(std::string(kTestDir) + "/" + shape + "_not_direct.idx")); + assert_ok(::doris::snii::reader::SniiSegmentReader::open(&full_file, &full_segment)); + assert_ok(full_segment.open_index(static_cast(kIndexId), /*index_suffix=*/"", + &full_idx)); + expect_full_bigram_layout(full_idx, shape); + expect_phrase_result(full_idx, expected_phrase, shape); + } + + TabletIndex _meta; + +private: + bool _saved_defer = false; + int32_t _saved_prune_min_df = 0; + double _saved_prune_max_ratio = 0.0; +}; + +// ScalarColumnWriter::init forwards opts.is_direct_load to the SNII index +// writer (the scalar leg of the column_writer.cpp wiring). +TEST_F(SniiBigramDeferWiringTest, ScalarColumnWriterForwardsDirectLoadHint) { + const auto expected_phrase = phrase_docids(80); + run_wiring_scenario("scalar", make_string_column(), make_string_block(), + /*expect_direct_defer=*/true, expected_phrase); +} + +// ArrayColumnWriter passes single_field=false to SNII. Direct array loads keep +// the full bigram layout, including its legacy element-boundary behavior. +TEST_F(SniiBigramDeferWiringTest, ArrayColumnWriterKeepsElementBoundaries) { + // Force the legacy full layout so a missing pair has the existing + // unambiguous "no adjacency" meaning. This makes the boundary regression + // independent of the ambient df-prune configuration. + config::snii_bigram_prune_min_df = 0; + config::snii_bigram_prune_max_df_ratio = 0.0; + const auto expected_phrase = phrase_docids(80); + run_wiring_scenario("array", make_array_string_column(), make_array_string_block(), + /*expect_direct_defer=*/false, expected_phrase); +} + +} // namespace +} // namespace doris::segment_v2 diff --git a/be/test/storage/index/snii/writer/snii_bigram_defer_writer_test.cpp b/be/test/storage/index/snii/writer/snii_bigram_defer_writer_test.cpp new file mode 100644 index 00000000000000..a88d9a53ec9767 --- /dev/null +++ b/be/test/storage/index/snii/writer/snii_bigram_defer_writer_test.cpp @@ -0,0 +1,622 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// SNII bigram-defer-to-compaction writer tests (see +// config::snii_bigram_defer_build_to_compaction). Full-stack through the +// PRODUCTION glue -- SniiIndexColumnWriter (init -> set_direct_load -> +// add_values -> finish) into IndexFileWriter::add_snii_index -- then read back +// with the raw SNII segment reader and phrase-queried: +// 1. A deferred (direct-load + config on) segment materializes NO bigram +// sentinel and NO hidden pair terms, so every phrase takes the +// positions-verification fallback -- and its phrase / phrase-prefix +// results are IDENTICAL to a normal segment built from the same rows. +// Covered for both reader fallback flavors: df-prune meta declared +// (default config) and legacy no-prune meta (sentinel gate). +// 2. Any non-deferring combination (config off, or not a direct load, or the +// hint never delivered) leaves the output BYTE-IDENTICAL to the baseline +// -- the hint alone must not perturb a single byte. +// 3. The defer decision is CAPTURED ONCE in set_direct_load(): a live mBool +// config flip landing after the hint -- before any row, through finish() +// -- changes nothing in either direction (on->off keeps deferring, +// off->on never arms), byte-for-byte against the constant-config twins. +// 4. set_direct_load is a no-op on the IndexColumnWriter base class (what a +// V3 InvertedIndexColumnWriter inherits) and safely gated by +// _has_positions on a docs-only SNII writer. + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/status.h" +#include "io/fs/file_writer.h" +#include "io/fs/local_file_system.h" +#include "storage/index/index_file_writer.h" +#include "storage/index/index_writer.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/query/count_query.h" +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/snii_index_writer.h" +#include "storage/tablet/tablet_schema.h" +#include "util/slice.h" + +namespace doris::segment_v2 { +namespace { + +namespace qinternal = ::doris::snii::query::internal; + +constexpr int64_t kIndexId = 1; +constexpr const char* kTestDir = "./ut_dir/snii_bigram_defer_writer_test"; + +void assert_ok(const Status& status) { + ASSERT_TRUE(status.ok()) << status.to_string(); +} + +// English-parser fulltext index with phrase support: _has_positions == true and +// the hidden bigram build active -- the production shape the deferral targets. +void init_phrase_index_meta(TabletIndex* meta) { + TabletIndexPB pb; + pb.set_index_type(IndexType::INVERTED); + pb.set_index_id(kIndexId); + pb.set_index_name("defer_test_idx"); + pb.add_col_unique_id(0); + pb.mutable_properties()->insert({"parser", "english"}); + pb.mutable_properties()->insert({"lower_case", "true"}); + pb.mutable_properties()->insert({"support_phrase", "true"}); + meta->init_from_pb(pb); +} + +// How the segment writer delivers (or does not deliver) the direct-load hint. +enum class DirectLoadHint { + kNone, // set_direct_load never called (index build / default paths) + kDirect, // set_direct_load(true): stream/broker load + kNotDirect, // set_direct_load(false): compaction / schema change +}; + +// Writes one single-index SNII segment file through the production writer +// stack, mirroring the production call order exactly: create()'s init() runs +// BEFORE the segment writer's set_direct_load, which precedes every row. +void write_segment(const std::string& path, const TabletIndex& meta, + const std::vector& rows, DirectLoadHint hint) { + io::FileWriterPtr file_writer; + assert_ok(io::global_local_filesystem()->create_file(path, &file_writer)); + IndexFileWriter index_file_writer(io::global_local_filesystem(), path, "test_rowset", + /*seg_id=*/0, InvertedIndexStorageFormatPB::SNII, + std::move(file_writer), /*can_use_ram_dir=*/true, + /*tablet_id=*/200); + + SniiIndexColumnWriter writer(&index_file_writer, &meta, /*single_field=*/true); + assert_ok(writer.init()); + if (hint == DirectLoadHint::kDirect) { + writer.set_direct_load(true); + } else if (hint == DirectLoadHint::kNotDirect) { + writer.set_direct_load(false); + } + + std::vector slices; + slices.reserve(rows.size()); + for (const std::string& row : rows) { + slices.emplace_back(row); + } + assert_ok(writer.add_values("c1", slices.data(), slices.size())); + assert_ok(writer.finish()); + assert_ok(index_file_writer.begin_close()); + assert_ok(index_file_writer.finish_close()); +} + +// Spec-G capture-once pin. Same production call order as write_segment, but +// hot-flips config::snii_bigram_defer_build_to_compaction to `config_after_hint` +// right AFTER set_direct_load(true) captured the defer decision -- the earliest +// instant a live mBool flip can land -- and keeps the flipped value through +// add_values AND finish(). If _add_value_tokens or finish() ever re-read the +// config instead of the captured _phrase_bigrams_deferred, the pair feed and the sentinel +// would follow the flipped value and the segment would diverge from its +// constant-config twin (worst case: pair postings whose vouching sentinel went +// missing, or a sentinel vouching for pairs that were never fed -- the exact +// half-fed-segment hazard the capture exists to prevent). +void write_segment_flipping_config_after_hint(const std::string& path, const TabletIndex& meta, + const std::vector& rows, + bool config_after_hint) { + io::FileWriterPtr file_writer; + assert_ok(io::global_local_filesystem()->create_file(path, &file_writer)); + IndexFileWriter index_file_writer(io::global_local_filesystem(), path, "test_rowset", + /*seg_id=*/0, InvertedIndexStorageFormatPB::SNII, + std::move(file_writer), /*can_use_ram_dir=*/true, + /*tablet_id=*/200); + + SniiIndexColumnWriter writer(&index_file_writer, &meta, /*single_field=*/true); + assert_ok(writer.init()); + writer.set_direct_load(true); // the capture happens HERE, under the pre-flip config + config::snii_bigram_defer_build_to_compaction = config_after_hint; + + std::vector slices; + slices.reserve(rows.size()); + for (const std::string& row : rows) { + slices.emplace_back(row); + } + assert_ok(writer.add_values("c1", slices.data(), slices.size())); + assert_ok(writer.finish()); + assert_ok(index_file_writer.begin_close()); + assert_ok(index_file_writer.finish_close()); +} + +// 80 rows with the adjacent (hello, world) pair: df 80 sits inside the default +// prune survival window [auto floor 64, 2 x floor 128], so any regression that +// re-feeds pairs on a should-be-deferred segment MATERIALIZES the pair -- the +// dict probes below catch it independently of the byte comparison. +std::vector eighty_pair_rows() { + std::vector rows; + rows.reserve(80); + for (uint32_t i = 0; i < 80; ++i) { + rows.push_back("hello world item" + std::to_string(i)); + } + return rows; +} + +// Opens the single logical index of a segment file written by write_segment. +// The reader pair must outlive every query against *idx. +void open_index(const std::string& path, ::doris::snii::io::LocalFileReader* file, + ::doris::snii::reader::SniiSegmentReader* segment, + ::doris::snii::reader::LogicalIndexReader* idx) { + assert_ok(file->open(path)); + assert_ok(::doris::snii::reader::SniiSegmentReader::open(file, segment)); + assert_ok(segment->open_index(static_cast(kIndexId), /*index_suffix=*/"", idx)); +} + +bool lookup_found(const ::doris::snii::reader::LogicalIndexReader& idx, const std::string& term) { + bool found = false; + ::doris::snii::format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + EXPECT_TRUE(idx.lookup(term, &found, &entry, &frq_base, &prx_base).ok()); + return found; +} + +std::vector run_phrase(const ::doris::snii::reader::LogicalIndexReader& idx, + const std::vector& terms) { + std::vector docids; + EXPECT_TRUE(::doris::snii::query::phrase_query(idx, terms, &docids).ok()); + return docids; +} + +std::vector run_phrase_prefix(const ::doris::snii::reader::LogicalIndexReader& idx, + const std::vector& terms) { + std::vector docids; + EXPECT_TRUE( + ::doris::snii::query::phrase_prefix_query(idx, terms, &docids, /*max_expansions=*/50) + .ok()); + return docids; +} + +std::string read_file_bytes(const std::string& path) { + std::ifstream in(path, std::ios::binary); + EXPECT_TRUE(in.good()) << path; + std::ostringstream out; + out << in.rdbuf(); + return out.str(); +} + +class SniiBigramDeferWriterTest : public testing::Test { +protected: + void SetUp() override { + assert_ok(io::global_local_filesystem()->delete_directory(kTestDir)); + assert_ok(io::global_local_filesystem()->create_directory(kTestDir)); + _saved_defer = config::snii_bigram_defer_build_to_compaction; + _saved_prune_min_df = config::snii_bigram_prune_min_df; + _saved_prune_max_ratio = config::snii_bigram_prune_max_df_ratio; + init_phrase_index_meta(&_meta); + } + + void TearDown() override { + config::snii_bigram_defer_build_to_compaction = _saved_defer; + config::snii_bigram_prune_min_df = _saved_prune_min_df; + config::snii_bigram_prune_max_df_ratio = _saved_prune_max_ratio; + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(kTestDir).ok()); + } + + std::string test_path(const std::string& name) const { + return std::string(kTestDir) + "/" + name + ".idx"; + } + + TabletIndex _meta; + +private: + bool _saved_defer = false; + int32_t _saved_prune_min_df = 0; + double _saved_prune_max_ratio = 0.0; +}; + +// Deferral flavor 1 -- default config (df-pruning armed, so the reader's +// dict-miss fallback is meta-declared): the deferred segment carries neither +// the sentinel nor any pair term, unigrams are untouched, and phrase / +// phrase-prefix answers equal the normal segment's exactly. +TEST_F(SniiBigramDeferWriterTest, DeferredSegmentOmitsBigramsAndPhraseFallsBack) { + // 80 rows with the adjacent pair (hello, world): df 80 clears the auto + // prune floor of 64, so the NORMAL segment provably materializes the pair + // (a below-floor pair would be dict-absent on BOTH segments and the + // presence assertions below would go vacuous). Plus non-matching shapes: + // reversed order, non-adjacent, and a second "wor"-prefixed tail for the + // multi-expansion phrase-prefix path. + std::vector rows; + std::vector expect_phrase; + for (uint32_t i = 0; i < 80; ++i) { + rows.push_back("hello world item" + std::to_string(i)); + expect_phrase.push_back(i); + } + rows.emplace_back("world hello reversed"); // docid 80: not "hello world" + rows.emplace_back("hello alpha world"); // docid 81: not adjacent + std::vector expect_prefix = expect_phrase; + for (uint32_t i = 82; i < 85; ++i) { + rows.push_back("hello worm data" + std::to_string(i)); // "hello wor*" only + expect_prefix.push_back(i); + } + + config::snii_bigram_defer_build_to_compaction = true; + const std::string normal_path = test_path("normal"); + const std::string defer_path = test_path("defer"); + // Config on but the hint never delivered (index build shape): full build. + write_segment(normal_path, _meta, rows, DirectLoadHint::kNone); + write_segment(defer_path, _meta, rows, DirectLoadHint::kDirect); + + ::doris::snii::io::LocalFileReader normal_file; + ::doris::snii::reader::SniiSegmentReader normal_segment; + ::doris::snii::reader::LogicalIndexReader normal_idx; + open_index(normal_path, &normal_file, &normal_segment, &normal_idx); + ::doris::snii::io::LocalFileReader defer_file; + ::doris::snii::reader::SniiSegmentReader defer_segment; + ::doris::snii::reader::LogicalIndexReader defer_idx; + open_index(defer_path, &defer_file, &defer_segment, &defer_idx); + + const std::string sentinel = ::doris::snii::format::make_phrase_bigram_sentinel_term(); + const std::string pair = ::doris::snii::format::make_phrase_bigram_term("hello", "world"); + // Normal segment: sentinel + the high-df pair are materialized. + EXPECT_TRUE(lookup_found(normal_idx, sentinel)); + EXPECT_TRUE(lookup_found(normal_idx, pair)); + // Deferred segment: no sentinel or pair term, while the unigrams stay + // identical. Its resident meta flag avoids probing an impossible pair. + EXPECT_FALSE(lookup_found(defer_idx, sentinel)); + EXPECT_FALSE(lookup_found(defer_idx, pair)); + EXPECT_TRUE(lookup_found(defer_idx, "hello")); + EXPECT_TRUE(lookup_found(defer_idx, "world")); + // Deferral is independent of the prune declaration: the meta still records + // the flush-resolved thresholds, plus the resident deferred capability that + // bypasses the otherwise unavoidable missing-pair probe. + EXPECT_GT(defer_idx.bigram_prune_min_df(), 0U); + EXPECT_FALSE(normal_idx.phrase_bigrams_deferred()); + EXPECT_TRUE(defer_idx.phrase_bigrams_deferred()); + + // POSITIVE CONTROL first: the normal segment's 2-term fast path actually + // probes the pair dict (and hits) -- proving bigram_probe_attempts is a + // live instrument, so the ==0 assertions on the deferred segment below + // cannot go vacuous if the counting site is ever moved or removed. + qinternal::query_test_counters() = qinternal::QueryTestCounters {}; + EXPECT_EQ(run_phrase(normal_idx, {"hello", "world"}), expect_phrase); + EXPECT_EQ(qinternal::query_test_counters().bigram_probe_attempts, 1U); + EXPECT_EQ(qinternal::query_test_counters().bigram_hits, 1U); + // Same rows -> same answers; the deferred segment just takes the + // positions-verification route without a single pair probe. + qinternal::query_test_counters() = qinternal::QueryTestCounters {}; + EXPECT_EQ(run_phrase(defer_idx, {"hello", "world"}), expect_phrase); + EXPECT_EQ(qinternal::query_test_counters().bigram_probe_attempts, 0U); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 1U); + + // Count-only first probes the hidden pair before deciding whether it can + // fabricate a count. The same resident gate prevents that redundant probe + // and lets the normal phrase path own verification. + bool count_handled = true; + uint64_t count = 1; + qinternal::query_test_counters() = qinternal::QueryTestCounters {}; + assert_ok(::doris::snii::query::count_only_two_term_phrase_bigram_df( + defer_idx, "hello", "world", &count_handled, &count)); + EXPECT_FALSE(count_handled); + EXPECT_EQ(count, 0U); + EXPECT_EQ(qinternal::query_test_counters().bigram_probe_attempts, 0U); + // Positive control for the count path's instrument: the normal segment + // probes once and answers from the pair's df alone (df 80). + qinternal::query_test_counters() = qinternal::QueryTestCounters {}; + assert_ok(::doris::snii::query::count_only_two_term_phrase_bigram_df( + normal_idx, "hello", "world", &count_handled, &count)); + EXPECT_TRUE(count_handled); + EXPECT_EQ(count, 80U); + EXPECT_EQ(qinternal::query_test_counters().bigram_probe_attempts, 1U); + + EXPECT_EQ(run_phrase(normal_idx, {"world", "hello"}), + run_phrase(defer_idx, {"world", "hello"})); + // Multi-expansion phrase-prefix ("wor" -> world + worm) crosses the + // single-leading bigram fast path on the normal segment (one probe per + // tail: world hits, worm misses into the prune-ambiguous fallback) and the + // pure verification path on the deferred one: identical answers. + qinternal::query_test_counters() = qinternal::QueryTestCounters {}; + EXPECT_EQ(run_phrase_prefix(normal_idx, {"hello", "wor"}), expect_prefix); + EXPECT_EQ(qinternal::query_test_counters().bigram_probe_attempts, 2U); + EXPECT_EQ(qinternal::query_test_counters().bigram_hits, 1U); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 1U); + // Deferred segment: the phrase_prefix_query guard skips the per-tail + // TryTwoTermPhraseBigram loop OUTRIGHT -- zero probes AND zero fallbacks. + // (Without the guard each tail would early-return inside + // TryTwoTermPhraseBigram and count one fallback per tail, so the + // fallbacks==0 assertion is what pins the guard itself.) + qinternal::query_test_counters() = qinternal::QueryTestCounters {}; + EXPECT_EQ(run_phrase_prefix(defer_idx, {"hello", "wor"}), expect_prefix); + EXPECT_EQ(qinternal::query_test_counters().bigram_probe_attempts, 0U); + EXPECT_EQ(qinternal::query_test_counters().bigram_fallbacks, 0U); +} + +// Deferral flavor 2 -- both prune gates disarmed. Newly written deferred +// segments still use their resident metadata capability gate; older segments +// without that bit retain the sentinel fallback covered by the no-bigram query +// fixtures. +TEST_F(SniiBigramDeferWriterTest, DeferredSegmentFallsBackUnderLegacyNoPruneConfig) { + config::snii_bigram_prune_min_df = 0; // legacy: materialize every pair + config::snii_bigram_prune_max_df_ratio = 0.0; // and disarm the upper gate + config::snii_bigram_defer_build_to_compaction = true; + + std::vector rows; + std::vector expect_phrase; + for (uint32_t i = 0; i < 5; ++i) { + rows.push_back("quick fox item" + std::to_string(i)); + expect_phrase.push_back(i); + } + rows.emplace_back("fox quick reversed"); // docid 5: adjacency only as (fox, quick) + + const std::string normal_path = test_path("legacy_normal"); + const std::string defer_path = test_path("legacy_defer"); + write_segment(normal_path, _meta, rows, DirectLoadHint::kNone); + write_segment(defer_path, _meta, rows, DirectLoadHint::kDirect); + + ::doris::snii::io::LocalFileReader normal_file; + ::doris::snii::reader::SniiSegmentReader normal_segment; + ::doris::snii::reader::LogicalIndexReader normal_idx; + open_index(normal_path, &normal_file, &normal_segment, &normal_idx); + ::doris::snii::io::LocalFileReader defer_file; + ::doris::snii::reader::SniiSegmentReader defer_segment; + ::doris::snii::reader::LogicalIndexReader defer_idx; + open_index(defer_path, &defer_file, &defer_segment, &defer_idx); + + const std::string sentinel = ::doris::snii::format::make_phrase_bigram_sentinel_term(); + // Legacy normal segment: every pair materialized, sentinel present, nothing + // declared in the meta. + EXPECT_TRUE(lookup_found(normal_idx, sentinel)); + EXPECT_TRUE(lookup_found(normal_idx, + ::doris::snii::format::make_phrase_bigram_term("quick", "fox"))); + EXPECT_EQ(normal_idx.bigram_prune_min_df(), 0U); + // Deferred segment: no sentinel or pairs. The metadata capability gate + // routes it to positions verification even with no prune declaration. + EXPECT_FALSE(lookup_found(defer_idx, sentinel)); + EXPECT_FALSE(lookup_found(defer_idx, + ::doris::snii::format::make_phrase_bigram_term("quick", "fox"))); + EXPECT_EQ(defer_idx.bigram_prune_min_df(), 0U); + EXPECT_EQ(defer_idx.bigram_prune_max_df(), 0U); + EXPECT_TRUE(defer_idx.phrase_bigrams_deferred()); + + EXPECT_EQ(run_phrase(normal_idx, {"quick", "fox"}), expect_phrase); + EXPECT_EQ(run_phrase(defer_idx, {"quick", "fox"}), expect_phrase); + // Legacy fast "dict miss == empty" on the normal segment vs the deferred + // segment's positions verification: a bigram-indexable pair whose terms + // both exist but are never ADJACENT ("fox"@0 / "reversed"@2 in docid 5) + // must come back empty through BOTH routes. + EXPECT_TRUE(run_phrase(normal_idx, {"fox", "reversed"}).empty()); + EXPECT_TRUE(run_phrase(defer_idx, {"fox", "reversed"}).empty()); +} + +// Every non-deferring combination must leave the segment BYTE-IDENTICAL to the +// untouched baseline: the hint or the config alone (or delivering an explicit +// "not direct load") may not perturb the output. +TEST_F(SniiBigramDeferWriterTest, NonDeferringCombinationsStayByteIdentical) { + const std::vector rows = {"hello world one", "hello world two", + "gamma delta three", "world hello four"}; + + config::snii_bigram_defer_build_to_compaction = false; + const std::string baseline_path = test_path("baseline"); + write_segment(baseline_path, _meta, rows, DirectLoadHint::kNone); + const std::string baseline_bytes = read_file_bytes(baseline_path); + ASSERT_FALSE(baseline_bytes.empty()); + + // Direct load but config off. + const std::string direct_off_path = test_path("direct_config_off"); + write_segment(direct_off_path, _meta, rows, DirectLoadHint::kDirect); + + // Config on but not a direct load (compaction / schema change shape). + config::snii_bigram_defer_build_to_compaction = true; + const std::string not_direct_path = test_path("not_direct_config_on"); + write_segment(not_direct_path, _meta, rows, DirectLoadHint::kNotDirect); + + // Config on but the hint never delivered (ADD INDEX build shape). + const std::string no_hint_path = test_path("no_hint_config_on"); + write_segment(no_hint_path, _meta, rows, DirectLoadHint::kNone); + + EXPECT_EQ(read_file_bytes(direct_off_path), baseline_bytes); + EXPECT_EQ(read_file_bytes(not_direct_path), baseline_bytes); + EXPECT_EQ(read_file_bytes(no_hint_path), baseline_bytes); + + // And the deferring combination DOES change bytes (the negative control + // proving the three assertions above are not vacuous). + const std::string defer_path = test_path("defer_control"); + write_segment(defer_path, _meta, rows, DirectLoadHint::kDirect); + EXPECT_NE(read_file_bytes(defer_path), baseline_bytes); +} + +// Spec-G capture pin, flip direction on -> off: the defer decision is captured +// ONCE in set_direct_load(); a live-config flip landing one instruction later +// (config::snii_bigram_defer_build_to_compaction is an mBool, hot-changeable +// mid-load) must not leak into the pair feed or the finish()-time sentinel. +// If _add_value_tokens re-read the config it would feed all 80 pairs under +// config=off (df 80 -> the pair MATERIALIZES: dict probe goes red); if finish() +// re-read it it would add the sentinel; either way the bytes diverge from the +// constant-config deferred twin. +TEST_F(SniiBigramDeferWriterTest, CapturedDeferralIgnoresMidLoadConfigFlipToOff) { + const std::vector rows = eighty_pair_rows(); + + // Reference twin: config=on held constant across the whole deferred write. + config::snii_bigram_defer_build_to_compaction = true; + const std::string defer_ref_path = test_path("flip_off_ref"); + write_segment(defer_ref_path, _meta, rows, DirectLoadHint::kDirect); + + // Flip scenario: capture under config=on, then config=off through every + // add_values call and through finish(). + config::snii_bigram_defer_build_to_compaction = true; + const std::string flipped_path = test_path("flip_off"); + write_segment_flipping_config_after_hint(flipped_path, _meta, rows, + /*config_after_hint=*/false); + + // The mid-load flip must be invisible: byte-identical to the constant twin. + EXPECT_EQ(read_file_bytes(flipped_path), read_file_bytes(defer_ref_path)); + + // Belt and braces, pinpointing WHICH re-read regressed on a failure: a + // finish()-side config re-read would have written the sentinel, a + // feed-side one the (materialized, df 80) pair term. + ::doris::snii::io::LocalFileReader flip_file; + ::doris::snii::reader::SniiSegmentReader flip_segment; + ::doris::snii::reader::LogicalIndexReader flip_idx; + open_index(flipped_path, &flip_file, &flip_segment, &flip_idx); + EXPECT_FALSE(lookup_found(flip_idx, ::doris::snii::format::make_phrase_bigram_sentinel_term())); + EXPECT_FALSE(lookup_found(flip_idx, + ::doris::snii::format::make_phrase_bigram_term("hello", "world"))); + EXPECT_TRUE(lookup_found(flip_idx, "hello")); // unigrams untouched by the flip +} + +// Spec-G capture pin, flip direction off -> on: set_direct_load(true) under +// config=off captures NO deferral; turning the config on before the first row +// must not arm it retroactively. A config re-read anywhere downstream would +// skip pair feeds and/or the sentinel and the bytes would leave the baseline. +TEST_F(SniiBigramDeferWriterTest, CapturedNonDeferralIgnoresMidLoadConfigFlipToOn) { + const std::vector rows = eighty_pair_rows(); + + // Reference twin: config=off held constant -- the full build (proven + // byte-identical to every other non-deferring shape by + // NonDeferringCombinationsStayByteIdentical above). + config::snii_bigram_defer_build_to_compaction = false; + const std::string baseline_path = test_path("flip_on_ref"); + write_segment(baseline_path, _meta, rows, DirectLoadHint::kDirect); + + // Flip scenario: capture under config=off, then config=on through feed and + // finish. Deferral must stay disarmed. + config::snii_bigram_defer_build_to_compaction = false; + const std::string flipped_path = test_path("flip_on"); + write_segment_flipping_config_after_hint(flipped_path, _meta, rows, + /*config_after_hint=*/true); + + EXPECT_EQ(read_file_bytes(flipped_path), read_file_bytes(baseline_path)); + + // Explicit full-build evidence on the flipped segment: sentinel and the + // df-80 pair are both materialized. + ::doris::snii::io::LocalFileReader flip_file; + ::doris::snii::reader::SniiSegmentReader flip_segment; + ::doris::snii::reader::LogicalIndexReader flip_idx; + open_index(flipped_path, &flip_file, &flip_segment, &flip_idx); + EXPECT_TRUE(lookup_found(flip_idx, ::doris::snii::format::make_phrase_bigram_sentinel_term())); + EXPECT_TRUE(lookup_found(flip_idx, + ::doris::snii::format::make_phrase_bigram_term("hello", "world"))); +} + +// Docs-only index (no support_phrase -> no positions -> no bigram build to +// defer): a direct load with the config ON must neither publish the resident +// kPhraseBigramsDeferred flag nor perturb a single byte vs the config-off +// baseline. Pins the writer-side _has_positions gate through the REAL persisted +// meta -- the scaffold-free BaseHook test below can only show the writer stays +// usable, not what lands on disk. +TEST_F(SniiBigramDeferWriterTest, DocsOnlyIndexNeverPublishesDeferFlag) { + TabletIndex docs_meta; + { + TabletIndexPB pb; + pb.set_index_type(IndexType::INVERTED); + pb.set_index_id(kIndexId); + pb.set_index_name("defer_docs_only_idx"); + pb.add_col_unique_id(0); + pb.mutable_properties()->insert({"parser", "english"}); + pb.mutable_properties()->insert({"lower_case", "true"}); + // No support_phrase: the index resolves docs-only, _has_positions false. + docs_meta.init_from_pb(pb); + } + const std::vector rows = {"hello world one", "hello world two"}; + + config::snii_bigram_defer_build_to_compaction = false; + const std::string baseline_path = test_path("docs_only_baseline"); + write_segment(baseline_path, docs_meta, rows, DirectLoadHint::kNone); + + config::snii_bigram_defer_build_to_compaction = true; + const std::string direct_path = test_path("docs_only_direct"); + write_segment(direct_path, docs_meta, rows, DirectLoadHint::kDirect); + + EXPECT_EQ(read_file_bytes(direct_path), read_file_bytes(baseline_path)); + + ::doris::snii::io::LocalFileReader file; + ::doris::snii::reader::SniiSegmentReader segment; + ::doris::snii::reader::LogicalIndexReader idx; + open_index(direct_path, &file, &segment, &idx); + EXPECT_FALSE(idx.has_positions()); + EXPECT_FALSE(idx.phrase_bigrams_deferred()); +} + +// The base-class hook is what a V3 InvertedIndexColumnWriter inherits: it must +// be a callable no-op on a writer that never overrides it. +TEST(SniiBigramDeferBaseHook, SetDirectLoadDefaultIsNoOp) { + // Minimal non-overriding IndexColumnWriter, mirroring the DBUG empty writer + // in column_writer.cpp: exercises the exact inherited base method a V3 + // writer would dispatch to. + class NoOverrideWriter final : public IndexColumnWriter { + public: + Status init() override { return Status::OK(); } + Status add_values(const std::string, const void*, size_t) override { return Status::OK(); } + Status add_array_values(size_t, const void*, const uint8_t*, const uint8_t*, + size_t) override { + return Status::OK(); + } + Status add_nulls(uint32_t) override { return Status::OK(); } + Status add_array_nulls(const uint8_t*, size_t) override { return Status::OK(); } + Status finish() override { return Status::OK(); } + int64_t size() const override { return 0; } + void close_on_error() override {} + }; + + NoOverrideWriter writer; + IndexColumnWriter* base = &writer; + base->set_direct_load(true); // inherited default: must be a harmless no-op + base->set_direct_load(false); + EXPECT_TRUE(base->finish().ok()); +} + +// The SNII override's _has_positions gate: on a writer whose index has no +// positions (scaffold-free, init() never run -> _has_positions == false, the +// same value a docs-only index resolves), a direct-load marking with the +// config on must NOT arm deferral -- there is no bigram build to defer -- and +// the writer stays fully usable. +TEST(SniiBigramDeferBaseHook, SniiWriterWithoutPositionsNeverDefers) { + const bool saved = config::snii_bigram_defer_build_to_compaction; + config::snii_bigram_defer_build_to_compaction = true; + { + SniiIndexColumnWriter writer(nullptr, nullptr, /*single_field=*/true); + writer.set_direct_load(true); + EXPECT_TRUE(writer.add_nulls(3).ok()); + EXPECT_EQ(writer.null_docids_for_test().size(), 3U); + } + config::snii_bigram_defer_build_to_compaction = saved; +} + +} // namespace +} // namespace doris::segment_v2 diff --git a/be/test/storage/index/snii/writer/snii_compound_writer_test.cpp b/be/test/storage/index/snii/writer/snii_compound_writer_test.cpp new file mode 100644 index 00000000000000..3ee78f0d5da2cb --- /dev/null +++ b/be/test/storage/index/snii/writer/snii_compound_writer_test.cpp @@ -0,0 +1,759 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/snii_compound_writer.h" + +#include + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/bootstrap_header.h" +#include "storage/index/snii/format/bsbf.h" +#include "storage/index/snii/format/dict_block.h" +#include "storage/index/snii/format/dict_block_directory.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/frq_pod.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/format/per_index_meta.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/format/sampled_term_index.h" +#include "storage/index/snii/format/tail_meta_region.h" +#include "storage/index/snii/format/tail_pointer.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/io/metered_file_reader.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/logical_index_writer.h" + +using namespace doris::snii; +using namespace doris::snii::format; +using namespace doris::snii::writer; +using doris::Status; + +namespace { + +// Temp file path helper (process-unique). +std::string TempPath() { + static int counter = 0; + return "/tmp/snii_cw_test_" + std::to_string(getpid()) + "_" + std::to_string(counter++) + + ".idx"; +} + +// Reads the whole file into a buffer. +std::vector ReadAll(const std::string& path) { + io::LocalFileReader r; + EXPECT_TRUE(r.open(path).ok()); + std::vector out; + EXPECT_TRUE(r.read_at(0, r.size(), &out).ok()); + return out; +} + +// Writes bytes to a fresh temp file and returns its path. +std::string WriteTemp(const std::vector& bytes) { + const std::string p = TempPath(); + io::LocalFileWriter w; + EXPECT_TRUE(w.open(p).ok()); + EXPECT_TRUE(w.append(Slice(bytes)).ok()); + EXPECT_TRUE(w.finalize().ok()); + return p; +} + +// A FileReader decorator that counts how many physical reads (single or batched) +// touch a given byte window. Used to assert that the BSBF section is NOT read at +// all on the non-resident (L1) path: with the P1 cold-read fix, open must not +// read the 28B bloom header and lookup must not issue a 32B bloom probe. +class WindowTouchCountingReader : public io::FileReader { +public: + WindowTouchCountingReader(io::FileReader* inner, uint64_t win_off, uint64_t win_len) + : inner_(inner), win_off_(win_off), win_len_(win_len) {} + + Status read_at(uint64_t offset, size_t len, std::vector* out) override { + account(offset, len); + return inner_->read_at(offset, len, out); + } + Status read_batch(const std::vector& ranges, + std::vector>* outs) override { + for (const auto& r : ranges) { + account(r.offset, r.len); + } + return inner_->read_batch(ranges, outs); + } + uint64_t size() const override { return inner_->size(); } + + uint64_t window_touches() const { return window_touches_; } + +private: + void account(uint64_t offset, size_t len) { + if (win_len_ == 0 || len == 0) { + return; + } + const uint64_t end = offset + len; + const uint64_t win_end = win_off_ + win_len_; + if (offset < win_end && win_off_ < end) { + ++window_touches_; + } + } + io::FileReader* inner_; + uint64_t win_off_; + uint64_t win_len_; + uint64_t window_touches_ = 0; +}; + +// Builds a TermPostings with constant freq per doc and (optionally) positions. +TermPostings MakeTerm(const std::string& term, const std::vector& docids, + bool with_positions) { + TermPostings tp; + tp.term = term; + tp.docids = docids; + tp.freqs.assign(docids.size(), 0); + for (size_t i = 0; i < docids.size(); ++i) { + tp.freqs[i] = static_cast((i % 3) + 1); + } + if (with_positions) { + for (size_t i = 0; i < docids.size(); ++i) { + for (uint32_t k = 0; k < tp.freqs[i]; ++k) { + tp.positions_flat.push_back(k * 2); // ascending positions (flat, by freq) + } + } + } + return tp; +} + +// Builds a FrqRegionMeta for a window's dd region from its WindowMeta. +FrqRegionMeta DdMetaOf(const WindowMeta& m) { + FrqRegionMeta r; + r.zstd = m.dd_zstd; + r.uncomp_len = m.dd_uncomp_len; + r.disk_len = m.dd_disk_len; + r.crc = m.crc_dd; + return r; +} + +// Index 0: ~30 docs. Vocab includes a HIGH-df term (df=600 > 512 -> windowed) +// and several LOW-df terms (-> slim/inline). Terms must be lexicographically +// sorted. +SniiIndexInput MakeIndex(uint64_t index_id, const std::string& suffix, uint32_t doc_count) { + SniiIndexInput in; + in.index_id = index_id; + in.index_suffix = suffix; + in.config = IndexConfig::kDocsPositions; + in.doc_count = doc_count; + // Force one DICT block per term so the sampled-term index covers every term + // (its max_term == the lexicographically largest term). + in.target_dict_block_bytes = 1; + + // low-df "apple" in a handful of docs. + in.terms.push_back(MakeTerm("apple", {0, 5, 12, 20}, true)); + // mid-low "banana". + in.terms.push_back(MakeTerm("banana", {1, 2, 3, 4, 9, 15}, true)); + // HIGH-df "common": df=600 (>=512) -> windowed pod_ref. + std::vector common_docs; + for (uint32_t d = 0; d < 600; ++d) { + common_docs.push_back(d); + } + in.terms.push_back(MakeTerm("common", common_docs, true)); + // low-df "zebra". + in.terms.push_back(MakeTerm("zebra", {7, 21, 29}, true)); + return in; +} + +// Locate a term through the full reader walk and return its DictEntry. +Status LocateEntry(const std::vector& file, const PerIndexMetaReader& meta, + const std::string& term, bool* found, DictEntry* out) { + std::vector sti_scratch; + Slice sti_frame; + RETURN_IF_ERROR(meta.sampled_term_index_frame(&sti_scratch, &sti_frame)); + SampledTermIndexReader sti; + RETURN_IF_ERROR(SampledTermIndexReader::open(sti_frame, &sti)); + std::vector dbd_scratch; + Slice dbd_frame; + RETURN_IF_ERROR(meta.dict_block_directory_frame(&dbd_scratch, &dbd_frame)); + DictBlockDirectoryReader dbd; + RETURN_IF_ERROR(DictBlockDirectoryReader::open(dbd_frame, &dbd)); + + bool maybe = false; + uint32_t ordinal = 0; + RETURN_IF_ERROR(sti.locate(term, &maybe, &ordinal)); + if (!maybe) { + *found = false; + return Status::OK(); + } + BlockRef ref {}; + RETURN_IF_ERROR(dbd.get(ordinal, &ref)); + Slice block(file.data() + ref.offset, ref.length); + DictBlockReader br; + RETURN_IF_ERROR(DictBlockReader::open(block, IndexTier::kT2, true, &br)); + return br.find_term(term, found, out); +} + +} // namespace + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiCompoundWriter, ReadBackSelfValidation) { + const std::string path = TempPath(); + { + io::LocalFileWriter w; + ASSERT_TRUE(w.open(path).ok()); + SniiCompoundWriter cw(&w); + ASSERT_TRUE(cw.add_logical_index(MakeIndex(10, "title", 30)).ok()); + ASSERT_TRUE(cw.add_logical_index(MakeIndex(11, "body", 30)).ok()); + ASSERT_TRUE(cw.finish().ok()); + } + + std::vector file = ReadAll(path); + ASSERT_GT(file.size(), kBootstrapHeaderSize + tail_pointer_size()); + + // --- bootstrap header --- + BootstrapHeader bh; + ASSERT_TRUE(decode_bootstrap_header(Slice(file.data(), kBootstrapHeaderSize), &bh).ok()); + EXPECT_EQ(bh.magic, kContainerMagic); + EXPECT_EQ(bh.format_version, kFormatVersion); + EXPECT_EQ(bh.tail_pointer_size, static_cast(tail_pointer_size())); + + // --- tail pointer (last tail_pointer_size() bytes) --- + Slice tail_bytes(file.data() + file.size() - tail_pointer_size(), tail_pointer_size()); + TailPointer tp; + ASSERT_TRUE(decode_tail_pointer(tail_bytes, &tp).ok()); + EXPECT_EQ(tp.hot_off, 0U); + ASSERT_GT(tp.meta_region_length, 0U); + ASSERT_LE(tp.meta_region_offset + tp.meta_region_length, file.size() - tail_pointer_size()); + + // --- tail meta region --- + Slice region(file.data() + tp.meta_region_offset, tp.meta_region_length); + TailMetaRegionReader tmr; + ASSERT_TRUE(TailMetaRegionReader::open(region, &tmr).ok()); + EXPECT_EQ(tmr.n_logical_indexes(), 2U); + + struct Expect { + uint64_t id; + std::string suffix; + }; + std::vector expects = {{.id = 10, .suffix = "title"}, {.id = 11, .suffix = "body"}}; + for (const auto& e : expects) { + bool found = false; + Slice meta_bytes; + ASSERT_TRUE(tmr.find(e.id, e.suffix, &found, &meta_bytes).ok()); + ASSERT_TRUE(found) << "index " << e.id; + + PerIndexMetaReader meta; + ASSERT_TRUE(PerIndexMetaReader::open(meta_bytes, &meta).ok()); + EXPECT_EQ(meta.index_id(), e.id); + EXPECT_EQ(meta.index_suffix(), e.suffix); + EXPECT_EQ(meta.stats().doc_count, 30U); + EXPECT_EQ(meta.stats().term_count, 4U); + + const SectionRefs& refs = meta.section_refs(); + // posting_region / dict_region must be within file bounds. With the order flip + // (posting region first, then DICT trailer), the posting region precedes the + // DICT region: posting_off < dict_off and posting_off + posting_len == dict_off. + ASSERT_GT(refs.posting_region.length, 0U); + ASSERT_LE(refs.posting_region.offset + refs.posting_region.length, file.size()); + ASSERT_GT(refs.dict_region.length, 0U); + ASSERT_LE(refs.dict_region.offset + refs.dict_region.length, file.size()); + EXPECT_LT(refs.posting_region.offset, refs.dict_region.offset); + EXPECT_EQ(refs.posting_region.offset + refs.posting_region.length, refs.dict_region.offset); + // norms absent for docs-positions (no scoring). + EXPECT_EQ(refs.norms.offset, 0U); + EXPECT_EQ(refs.norms.length, 0U); + EXPECT_EQ(refs.null_bitmap.offset, 0U); + EXPECT_EQ(refs.null_bitmap.length, 0U); + + // --- XFilter (block-split bloom, physical section): present true, absent false --- + // Probe the on-disk filter directly: one 32-byte block at a self-computed offset. + EXPECT_TRUE(meta.has_bsbf()); + ASSERT_GT(refs.bsbf.length, doris::snii::format::kBsbfHeaderSize); + ASSERT_LE(refs.bsbf.offset + refs.bsbf.length, file.size()); + const uint64_t bsbf_bitset = refs.bsbf.offset + doris::snii::format::kBsbfHeaderSize; + const auto bsbf_nblocks = + static_cast((refs.bsbf.length - doris::snii::format::kBsbfHeaderSize) / + doris::snii::format::kBsbfBytesPerBlock); + auto bsbf_present = [&](std::string_view term) { + const uint64_t h = doris::snii::format::bsbf_hash(term); + const uint64_t off = + bsbf_bitset + + static_cast(doris::snii::format::bsbf_block_index(h, bsbf_nblocks)) * + doris::snii::format::kBsbfBytesPerBlock; + return doris::snii::format::bsbf_block_contains(h, file.data() + off); + }; + EXPECT_TRUE(bsbf_present("apple")); + EXPECT_TRUE(bsbf_present("common")); + EXPECT_FALSE(bsbf_present("nonexistent-term-xyzzy-12345")); + + // --- windowed high-df term "common": read its .frq window --- + bool found_common = false; + DictEntry common_entry; + ASSERT_TRUE(LocateEntry(file, meta, "common", &found_common, &common_entry).ok()); + ASSERT_TRUE(found_common); + EXPECT_EQ(common_entry.df, 600U); + EXPECT_EQ(common_entry.kind, DictEntryKind::kPodRef); + EXPECT_EQ(common_entry.enc, DictEntryEnc::kWindowed); + + // The DICT block carrying "common" supplies frq_base via DictBlockReader. + // Recompute frq_base by re-locating the block. + std::vector sti_scratch; + Slice sti_frame; + ASSERT_TRUE(meta.sampled_term_index_frame(&sti_scratch, &sti_frame).ok()); + SampledTermIndexReader sti; + ASSERT_TRUE(SampledTermIndexReader::open(sti_frame, &sti).ok()); + std::vector dbd_scratch; + Slice dbd_frame; + ASSERT_TRUE(meta.dict_block_directory_frame(&dbd_scratch, &dbd_frame).ok()); + DictBlockDirectoryReader dbd; + ASSERT_TRUE(DictBlockDirectoryReader::open(dbd_frame, &dbd).ok()); + bool maybe = false; + uint32_t ord = 0; + ASSERT_TRUE(sti.locate("common", &maybe, &ord).ok()); + ASSERT_TRUE(maybe); + BlockRef bref {}; + ASSERT_TRUE(dbd.get(ord, &bref).ok()); + Slice block(file.data() + bref.offset, bref.length); + DictBlockReader br; + ASSERT_TRUE(DictBlockReader::open(block, IndexTier::kT2, true, &br).ok()); + + // Absolute .frq offset = posting_region.offset + frq_base + frq_off_delta. The + // windowed payload is [prelude][dd-block][freq-block]; parse the two-level + // prelude, then decode every window's dd region from the dd-block (each with + // its prelude win_base) to reconstruct the full posting (docs-only path). + uint64_t frq_abs = refs.posting_region.offset + br.frq_base() + common_entry.frq_off_delta; + Slice prelude_bytes(file.data() + frq_abs, common_entry.prelude_len); + FrqPreludeReader prelude; + ASSERT_TRUE(FrqPreludeReader::open(prelude_bytes, &prelude).ok()); + const uint64_t dd_block_start = frq_abs + common_entry.prelude_len; + std::vector got_docs; + uint32_t summed = 0; + for (uint32_t w = 0; w < prelude.n_windows(); ++w) { + WindowMeta m; + ASSERT_TRUE(prelude.window(w, &m).ok()); + Slice dd(file.data() + dd_block_start + m.dd_off, m.dd_disk_len); + std::vector wdocs; + ASSERT_TRUE(decode_dd_region(dd, DdMetaOf(m), m.win_base, &wdocs).ok()); + ASSERT_EQ(wdocs.size(), m.doc_count); + summed += m.doc_count; + got_docs.insert(got_docs.end(), wdocs.begin(), wdocs.end()); + } + EXPECT_EQ(summed, 600U); // window doc_counts sum to df. + ASSERT_EQ(got_docs.size(), 600U); + EXPECT_EQ(got_docs.front(), 0U); + EXPECT_EQ(got_docs.back(), 599U); + + // --- inline low-df term "apple": decode its inline .frq bytes --- + bool found_apple = false; + DictEntry apple_entry; + ASSERT_TRUE(LocateEntry(file, meta, "apple", &found_apple, &apple_entry).ok()); + ASSERT_TRUE(found_apple); + EXPECT_EQ(apple_entry.df, 4U); + EXPECT_EQ(apple_entry.kind, DictEntryKind::kInline); + Slice apple_dd(apple_entry.frq_bytes.data(), + static_cast(apple_entry.dd_meta.disk_len)); + std::vector apple_docs; + ASSERT_TRUE(decode_dd_region(apple_dd, apple_entry.dd_meta, 0, &apple_docs).ok()); + std::vector expected_apple = {0, 5, 12, 20}; + EXPECT_EQ(apple_docs, expected_apple); + + // inline .prx bytes decode to per-doc positions. + ByteSource psrc(Slice(apple_entry.prx_bytes)); + std::vector> apple_pos; + ASSERT_TRUE(read_prx_window(&psrc, &apple_pos).ok()); + EXPECT_EQ(apple_pos.size(), 4U); + + // absent term -> not found through the dict walk. + bool found_absent = true; + DictEntry absent_entry; + ASSERT_TRUE(LocateEntry(file, meta, "zzz-not-here", &found_absent, &absent_entry).ok()); + EXPECT_FALSE(found_absent); + } + + std::remove(path.c_str()); +} + +namespace { + +// Oracle for the multi-super-block read-back test. "hot" is a very high-df term; +// at df=70000 it uses adaptive 1024-doc windows (df >= kAdaptiveWindowDfThreshold) +// -> ~69 windows -> >1 super-block at group_size=64. "spark" appears at position +// 1 immediately after "hot" in docs where d % 9 == 0, so the phrase "hot spark" +// holds exactly in those docs. "rare" is a tiny low-df term. +struct BigCorpus { + uint32_t doc_count = 70000; + std::vector hot_docs; // every doc + std::vector spark_docs; // d % 9 == 0 + std::vector rare_docs = {3, 17, 99, 1000, 19999}; + std::vector phrase_oracle; // docs where "hot spark" is consecutive +}; + +BigCorpus MakeBigCorpus() { + BigCorpus c; + for (uint32_t d = 0; d < c.doc_count; ++d) { + c.hot_docs.push_back(d); + if (d % 9 == 0) { + c.spark_docs.push_back(d); + c.phrase_oracle.push_back(d); // "hot"@0 then "spark"@1 -> phrase hit + } + } + return c; +} + +// Builds a TermPostings with all freq=1 and a single position per doc. +TermPostings MakePosTerm(const std::string& term, const std::vector& docs, uint32_t pos) { + TermPostings tp; + tp.term = term; + tp.docids = docs; + tp.freqs.assign(docs.size(), 1); + tp.positions_flat.assign(docs.size(), pos); // one position per doc, flat + return tp; +} + +SniiIndexInput MakeBigIndex(const BigCorpus& c) { + SniiIndexInput in; + in.index_id = 1; + in.index_suffix = "body"; + in.config = IndexConfig::kDocsPositions; + in.doc_count = c.doc_count; + // One DICT block per term so every term is a block first-term (covered by the + // sampled-term index regardless of order), mirroring the other read tests. + in.target_dict_block_bytes = 1; + // Terms must be lexicographically sorted. + in.terms.push_back(MakePosTerm("hot", c.hot_docs, /*pos=*/0)); // huge -> windowed + in.terms.push_back(MakePosTerm("rare", c.rare_docs, /*pos=*/0)); // tiny -> inline + in.terms.push_back(MakePosTerm("spark", c.spark_docs, /*pos=*/1)); // -> windowed + return in; +} + +} // namespace + +// PHASE A read-back self-validation: a high-df term spanning MANY windows across +// MULTIPLE super-blocks. Asserts (a) sum of window doc_counts == df, (b) the +// windows tile the posting in order, (c) locate_window resolves the covering +// window, and (d) term_query / phrase_query agree with the oracle. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiCompoundWriter, MultiSuperBlockReadBack) { + const BigCorpus c = MakeBigCorpus(); + const std::string path = TempPath(); + { + io::LocalFileWriter w; + ASSERT_TRUE(w.open(path).ok()); + SniiCompoundWriter cw(&w); + ASSERT_TRUE(cw.add_logical_index(MakeBigIndex(c)).ok()); + ASSERT_TRUE(cw.finish().ok()); + } + + io::LocalFileReader local; + ASSERT_TRUE(local.open(path).ok()); + io::MeteredFileReader metered(&local); + reader::SniiSegmentReader seg; + ASSERT_TRUE(reader::SniiSegmentReader::open(&metered, &seg).ok()); + reader::LogicalIndexReader idx; + ASSERT_TRUE(seg.open_index(1, "body", &idx).ok()); + + // Resolve "hot" to its DictEntry + the block's frq_base. + bool found = false; + DictEntry hot; + uint64_t frq_base = 0, prx_base = 0; + ASSERT_TRUE(idx.lookup("hot", &found, &hot, &frq_base, &prx_base).ok()); + ASSERT_TRUE(found); + EXPECT_EQ(hot.df, c.doc_count); + EXPECT_EQ(hot.enc, DictEntryEnc::kWindowed); + EXPECT_TRUE(hot.has_sb); + + // L0 tiering: this index's bsbf filter is tiny (<= kBsbfResidentMaxBytes), so it is + // loaded resident at open and an absent-term lookup is rejected IN MEMORY with zero + // reads (no per-lookup round). Loop a few absents to skip the rare false positive. + bool saw_resident_reject = false; + for (int i = 0; i < 8 && !saw_resident_reject; ++i) { + metered.reset_metrics(); + bool af = true; + DictEntry ad; + uint64_t afb = 0, apb = 0; + ASSERT_TRUE(idx.lookup("absent-zzz-" + std::to_string(i), &af, &ad, &afb, &apb).ok()); + if (!af) { + EXPECT_EQ(metered.metrics().read_at_calls, 0U); + EXPECT_EQ(metered.metrics().serial_rounds, 0U); + saw_resident_reject = true; + } + } + EXPECT_TRUE(saw_resident_reject); + metered.reset_metrics(); + + // L1 tiering (P1 cold-read fix): force the bloom NON-resident by lowering the + // resident threshold to 0, then re-open the SAME index through a reader that + // counts every physical read touching the bsbf section. With the fix the bloom + // is skipped ENTIRELY -- open does NOT read the 28B header and lookup does NOT + // issue a 32B probe -- so ZERO reads touch the bsbf window, yet a present term + // is still found and absent terms are still not-found, all via sti -> dict. + const RegionRef bsbf_ref = idx.section_refs().bsbf; + ASSERT_GT(bsbf_ref.length, kBsbfHeaderSize); // a real (small) filter exists on disk + ::setenv("SNII_BSBF_RESIDENT_MAX", "0", /*overwrite=*/1); + { + io::LocalFileReader l1_local; + ASSERT_TRUE(l1_local.open(path).ok()); + WindowTouchCountingReader counting(&l1_local, bsbf_ref.offset, bsbf_ref.length); + reader::SniiSegmentReader seg_l1; + ASSERT_TRUE(reader::SniiSegmentReader::open(&counting, &seg_l1).ok()); + reader::LogicalIndexReader idx_l1; + ASSERT_TRUE(seg_l1.open_index(1, "body", &idx_l1).ok()); + // open() must not have read the bsbf header (28B) on the non-resident path. + EXPECT_EQ(counting.window_touches(), 0U) << "non-resident open must skip the bsbf header"; + + // Present term: still found via sti -> dict, with no bloom involved. + bool pf = false; + DictEntry pe; + uint64_t pfb = 0, ppb = 0; + ASSERT_TRUE(idx_l1.lookup("hot", &pf, &pe, &pfb, &ppb).ok()); + EXPECT_TRUE(pf); + + // Absent terms: not found via dict. "absent-zzz-*" sorts before every + // sample (out-of-range sti reject); "rzz" sorts inside the term range so + // sti routes it to a real dict block that then misses -- both return absent + // and NEITHER probes the bloom. + for (int i = 0; i < 8; ++i) { + bool af = true; + DictEntry ad; + uint64_t afb = 0, apb = 0; + ASSERT_TRUE( + idx_l1.lookup("absent-zzz-" + std::to_string(i), &af, &ad, &afb, &apb).ok()); + EXPECT_FALSE(af) << "out-of-range absent i=" << i; + } + bool rf = true; + DictEntry rd; + uint64_t rfb = 0, rpb = 0; + ASSERT_TRUE(idx_l1.lookup("rzz", &rf, &rd, &rfb, &rpb).ok()); + EXPECT_FALSE(rf) << "in-range absent term must miss in the dict"; + + // No lookup may have probed the bsbf section: the bloom is skipped, not + // read on demand. This is the core of the P1 cold-read fix. + EXPECT_EQ(counting.window_touches(), 0U) + << "non-resident lookups must not probe the bsbf section"; + } + ::unsetenv("SNII_BSBF_RESIDENT_MAX"); + metered.reset_metrics(); + + // Fetch + parse the two-level prelude. + const uint64_t prelude_abs = + idx.section_refs().posting_region.offset + frq_base + hot.frq_off_delta; + std::vector prelude_bytes; + ASSERT_TRUE(local.read_at(prelude_abs, hot.prelude_len, &prelude_bytes).ok()); + FrqPreludeReader prelude; + ASSERT_TRUE(FrqPreludeReader::open(Slice(prelude_bytes), &prelude).ok()); + EXPECT_GT(prelude.n_super_blocks(), 1U) << "expected >1 super-block"; + + // (a)+(b): decode every window's dd region from the dd-block; doc_counts sum to + // df and the concatenated docids equal the full ascending posting [0, doc_count). + const uint64_t dd_block_start = prelude_abs + hot.prelude_len; + std::vector tiled; + uint64_t summed = 0; + uint64_t expect_win_base = 0; + for (uint32_t w = 0; w < prelude.n_windows(); ++w) { + WindowMeta m; + ASSERT_TRUE(prelude.window(w, &m).ok()); + EXPECT_EQ(m.win_base, expect_win_base) << "win_base w=" << w; + std::vector ddbytes; + ASSERT_TRUE(local.read_at(dd_block_start + m.dd_off, m.dd_disk_len, &ddbytes).ok()); + std::vector wdocs; + ASSERT_TRUE(decode_dd_region(Slice(ddbytes), DdMetaOf(m), m.win_base, &wdocs).ok()); + ASSERT_EQ(wdocs.size(), m.doc_count) << "w=" << w; + summed += m.doc_count; + expect_win_base = m.last_docid; + tiled.insert(tiled.end(), wdocs.begin(), wdocs.end()); + } + EXPECT_EQ(summed, c.doc_count); + ASSERT_EQ(tiled.size(), c.doc_count); + EXPECT_EQ(tiled, c.hot_docs); + + // (c): locate_window returns the window actually containing the docid. + const uint32_t probes[] = {0, 255, 256, 257, 5000, 16383, 16384, 19999}; + for (uint32_t docid : probes) { + bool lfound = false; + uint32_t w = 0; + ASSERT_TRUE(prelude.locate_window(docid, &lfound, &w).ok()); + ASSERT_TRUE(lfound) << "docid=" << docid; + WindowMeta m; + ASSERT_TRUE(prelude.window(w, &m).ok()); + EXPECT_GE(static_cast(docid), m.win_base + (w == 0 ? 0 : 1)) << "docid=" << docid; + EXPECT_LE(docid, m.last_docid) << "docid=" << docid; + } + // past-end -> not found. + { + bool lfound = true; + uint32_t w = 0; + ASSERT_TRUE(prelude.locate_window(c.doc_count, &lfound, &w).ok()); + EXPECT_FALSE(lfound); + } + + // (d): term_query / phrase_query agree with the oracle (full-read path). + std::vector hot_docs; + ASSERT_TRUE(query::term_query(idx, "hot", &hot_docs).ok()); + EXPECT_EQ(hot_docs, c.hot_docs); + + std::vector spark_docs; + ASSERT_TRUE(query::term_query(idx, "spark", &spark_docs).ok()); + EXPECT_EQ(spark_docs, c.spark_docs); + + std::vector rare_docs; + ASSERT_TRUE(query::term_query(idx, "rare", &rare_docs).ok()); + EXPECT_EQ(rare_docs, c.rare_docs); + + std::vector phrase_docs; + ASSERT_TRUE(query::phrase_query(idx, {"hot", "spark"}, &phrase_docs).ok()); + EXPECT_EQ(phrase_docs, c.phrase_oracle); + + // reversed phrase must be absent ("spark"@1 never precedes "hot"@0). + std::vector reversed; + ASSERT_TRUE(query::phrase_query(idx, {"spark", "hot"}, &reversed).ok()); + EXPECT_TRUE(reversed.empty()); + + std::remove(path.c_str()); +} + +// BYTE-IDENTICAL OUTPUT GUARANTEE: the section bytes (DICT region, .frq POD, +// .prx POD) now stream through scratch temp files instead of in-RAM vectors, and +// the xfilter is built from per-term hashes instead of retained strings. The +// produced container must be deterministic and byte-for-byte stable: building the +// same input twice yields identical container bytes (proves the temp-file path +// reproduces the exact same layout/offsets/content as itself, with no in-RAM +// residue affecting the bytes). The big multi-super-block index exercises every +// section type (windowed pod_ref + inline + multi-block dict + prx). +TEST(SniiCompoundWriter, StreamedOutputIsDeterministicByteForByte) { + const BigCorpus c = MakeBigCorpus(); + + auto build_one = [&](const std::string& path) { + io::LocalFileWriter w; + ASSERT_TRUE(w.open(path).ok()); + SniiCompoundWriter cw(&w); + ASSERT_TRUE(cw.add_logical_index(MakeBigIndex(c)).ok()); + ASSERT_TRUE(cw.finish().ok()); + }; + + const std::string path_a = TempPath(); + const std::string path_b = TempPath(); + build_one(path_a); + build_one(path_b); + + const std::vector file_a = ReadAll(path_a); + const std::vector file_b = ReadAll(path_b); + ASSERT_FALSE(file_a.empty()); + EXPECT_EQ(file_a.size(), file_b.size()); + EXPECT_EQ(file_a, file_b) << "streamed container must be byte-identical run-to-run"; + + std::remove(path_a.c_str()); + std::remove(path_b.c_str()); +} + +// Determinism for a multi-index docs-positions container with multiple DICT +// blocks per index (target_dict_block_bytes forces real block cuts, exercising +// the dict scratch-file concat + the BlockRecord rel_offset directory math). +TEST(SniiCompoundWriter, StreamedMultiIndexDeterministic) { + auto build_one = [&](const std::string& path) { + io::LocalFileWriter w; + ASSERT_TRUE(w.open(path).ok()); + SniiCompoundWriter cw(&w); + SniiIndexInput a = MakeIndex(10, "title", 30); + SniiIndexInput b = MakeIndex(11, "body", 30); + a.target_dict_block_bytes = 64; // force multiple dict blocks + b.target_dict_block_bytes = 64; + ASSERT_TRUE(cw.add_logical_index(a).ok()); + ASSERT_TRUE(cw.add_logical_index(b).ok()); + ASSERT_TRUE(cw.finish().ok()); + }; + const std::string p1 = TempPath(); + const std::string p2 = TempPath(); + build_one(p1); + build_one(p2); + EXPECT_EQ(ReadAll(p1), ReadAll(p2)); + std::remove(p1.c_str()); + std::remove(p2.c_str()); +} + +// A flipped on-disk byte in the bsbf filter (header or bitset) or in the meta region +// must be detected on the read path, not silently propagated as a wrong probe result. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiCompoundWriter, ChecksumCorruptionDetectedOnRead) { + const std::string good = TempPath(); + { + io::LocalFileWriter w; + ASSERT_TRUE(w.open(good).ok()); + SniiCompoundWriter cw(&w); + ASSERT_TRUE(cw.add_logical_index(MakeIndex(7, "body", 30)).ok()); + ASSERT_TRUE(cw.finish().ok()); + } + const std::vector file = ReadAll(good); + std::remove(good.c_str()); + + // Locate the bsbf section + meta region from the tail. + TailPointer tp; + ASSERT_TRUE(decode_tail_pointer( + Slice(file.data() + file.size() - tail_pointer_size(), tail_pointer_size()), + &tp) + .ok()); + TailMetaRegionReader tmr; + ASSERT_TRUE(TailMetaRegionReader::open( + Slice(file.data() + tp.meta_region_offset, tp.meta_region_length), &tmr) + .ok()); + bool found = false; + Slice meta_bytes; + ASSERT_TRUE(tmr.find(7, "body", &found, &meta_bytes).ok()); + ASSERT_TRUE(found); + PerIndexMetaReader meta; + ASSERT_TRUE(PerIndexMetaReader::open(meta_bytes, &meta).ok()); + const auto bsbf = meta.section_refs().bsbf; + ASSERT_GT(bsbf.length, doris::snii::format::kBsbfHeaderSize); // small filter -> L0 + + // The clean file opens fine. + auto opens_ok = [](const std::vector& bytes) { + const std::string p = WriteTemp(bytes); + io::LocalFileReader r; + EXPECT_TRUE(r.open(p).ok()); + reader::SniiSegmentReader seg; + Status sopen = reader::SniiSegmentReader::open(&r, &seg); + bool ok = sopen.ok(); + if (ok) { + reader::LogicalIndexReader idx; + ok = seg.open_index(7, "body", &idx).ok(); + } + std::remove(p.c_str()); + return ok; + }; + EXPECT_TRUE(opens_ok(file)); + + // (1) bsbf BITSET byte flip -> L0 open verifies the bitset crc -> rejected. + { + std::vector bad = file; + bad[bsbf.offset + doris::snii::format::kBsbfHeaderSize] ^= 0xFF; + EXPECT_FALSE(opens_ok(bad)); + } + // (2) bsbf HEADER byte flip (num_bytes field, covered by header crc) -> rejected. + { + std::vector bad = file; + bad[bsbf.offset + 8] ^= 0xFF; + EXPECT_FALSE(opens_ok(bad)); + } + // (3) META REGION byte flip -> segment open verifies meta_region_checksum -> rejected. + { + std::vector bad = file; + bad[tp.meta_region_offset] ^= 0xFF; + EXPECT_FALSE(opens_ok(bad)); + } +} diff --git a/be/test/storage/index/snii/writer/snii_prx_tier_writer_test.cpp b/be/test/storage/index/snii/writer/snii_prx_tier_writer_test.cpp new file mode 100644 index 00000000000000..d663c3c0c2172b --- /dev/null +++ b/be/test/storage/index/snii/writer/snii_prx_tier_writer_test.cpp @@ -0,0 +1,294 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// SNII prx-tier writer tests (patch C, see +// config::snii_prx_zstd_level_direct_load): a DIRECT load compresses the prx +// region at the cheaper load-tier zstd level, everything else keeps +// snii_prx_zstd_level. Contract pinned here: +// 1. The tier only changes prx BYTES, never SEMANTICS: a direct segment and +// its full-level twin answer position-dependent queries identically. +// 2. Non-direct paths (no hint / explicit not-direct) ignore the load-tier +// config completely -- byte-identical outputs whatever its value, so +// compaction / schema change / ADD INDEX segments are untouched. +// 3. The level is read at flush (same semantics as snii_prx_zstd_level): a +// mid-load change lands on the in-flight segment; the direct-load BIT +// itself stays captured-once (pinned by the defer writer tests). +// 4. The load-tier level is clamped to [3, 19] like the base level. + +#include + +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/status.h" +#include "io/fs/local_file_system.h" +#include "storage/index/index_file_writer.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/snii_index_writer.h" +#include "storage/tablet/tablet_schema.h" +#include "util/slice.h" + +namespace doris::segment_v2 { +namespace { + +constexpr int64_t kIndexId = 7; +constexpr const char* kTestDir = "./ut_dir/snii_prx_tier_writer_test"; + +void assert_ok(const Status& status) { + ASSERT_TRUE(status.ok()) << status.to_string(); +} + +// Positions-capable fulltext index: the only shape with a prx region to tier. +void init_phrase_index_meta(TabletIndex* meta) { + TabletIndexPB pb; + pb.set_index_type(IndexType::INVERTED); + pb.set_index_id(kIndexId); + pb.set_index_name("prx_tier_idx"); + pb.add_col_unique_id(0); + pb.mutable_properties()->insert({"parser", "english"}); + pb.mutable_properties()->insert({"lower_case", "true"}); + pb.mutable_properties()->insert({"support_phrase", "true"}); + meta->init_from_pb(pb); +} + +enum class DirectLoadHint { kNone, kDirect, kNotDirect }; + +// Same production call order as the defer writer tests: init() before the +// segment writer's set_direct_load, which precedes every row. `flip_after_hint` +// (optional) hot-changes snii_prx_zstd_level_direct_load right after the hint +// -- the earliest instant a live mInt32 change can land -- and keeps it through +// finish(), pinning the level's flush-read semantics. +void write_segment(const std::string& path, const TabletIndex& meta, + const std::vector& rows, DirectLoadHint hint, + const int32_t* flip_after_hint = nullptr) { + io::FileWriterPtr file_writer; + assert_ok(io::global_local_filesystem()->create_file(path, &file_writer)); + IndexFileWriter index_file_writer(io::global_local_filesystem(), path, "test_rowset", + /*seg_id=*/0, InvertedIndexStorageFormatPB::SNII, + std::move(file_writer), /*can_use_ram_dir=*/true, + /*tablet_id=*/300); + + SniiIndexColumnWriter writer(&index_file_writer, &meta, /*single_field=*/true); + assert_ok(writer.init()); + if (hint == DirectLoadHint::kDirect) { + writer.set_direct_load(true); + } else if (hint == DirectLoadHint::kNotDirect) { + writer.set_direct_load(false); + } + if (flip_after_hint != nullptr) { + config::snii_prx_zstd_level_direct_load = *flip_after_hint; + } + + std::vector slices; + slices.reserve(rows.size()); + for (const std::string& row : rows) { + slices.emplace_back(row); + } + assert_ok(writer.add_values("c1", slices.data(), slices.size())); + assert_ok(writer.finish()); + assert_ok(index_file_writer.begin_close()); + assert_ok(index_file_writer.finish_close()); +} + +// 400 rows x 30 tokens over a 40-word vocabulary with a deterministic pattern: +// enough position payload for the zstd level to visibly change the prx bytes, +// repetitive enough that higher levels find more to squeeze. +std::vector positional_rows() { + static const char* kVocab[] = { + "alpha", "beta", "gamma", "delta", "epsil", "zeta", "eta", "theta", "iota", "kappa", + "lam", "mu", "nu", "xi", "omic", "pi", "rho", "sigma", "tau", "upsil", + "phi", "chi", "psi", "omega", "one", "two", "three", "four", "five", "six", + "seven", "eight", "nine", "ten", "red", "green", "blue", "cyan", "lime", "teal"}; + constexpr size_t kVocabSize = sizeof(kVocab) / sizeof(kVocab[0]); + std::vector rows; + rows.reserve(400); + for (uint32_t i = 0; i < 400; ++i) { + std::string row; + for (uint32_t j = 0; j < 30; ++j) { + if (j > 0) { + row += ' '; + } + row += kVocab[(i * 31 + j * 7 + (i * j) % 5) % kVocabSize]; + } + rows.push_back(std::move(row)); + } + return rows; +} + +void open_index(const std::string& path, ::doris::snii::io::LocalFileReader* file, + ::doris::snii::reader::SniiSegmentReader* segment, + ::doris::snii::reader::LogicalIndexReader* idx) { + assert_ok(file->open(path)); + assert_ok(::doris::snii::reader::SniiSegmentReader::open(file, segment)); + assert_ok(segment->open_index(static_cast(kIndexId), /*index_suffix=*/"", idx)); +} + +std::vector run_phrase(const ::doris::snii::reader::LogicalIndexReader& idx, + const std::vector& terms) { + std::vector docids; + EXPECT_TRUE(::doris::snii::query::phrase_query(idx, terms, &docids).ok()); + return docids; +} + +std::string read_file_bytes(const std::string& path) { + std::ifstream in(path, std::ios::binary); + EXPECT_TRUE(in.good()) << path; + std::ostringstream out; + out << in.rdbuf(); + return out.str(); +} + +class SniiPrxTierWriterTest : public testing::Test { +protected: + void SetUp() override { + assert_ok(io::global_local_filesystem()->delete_directory(kTestDir)); + assert_ok(io::global_local_filesystem()->create_directory(kTestDir)); + _saved_prx_level = config::snii_prx_zstd_level; + _saved_load_level = config::snii_prx_zstd_level_direct_load; + _saved_defer = config::snii_bigram_defer_build_to_compaction; + config::snii_bigram_defer_build_to_compaction = false; // isolate the tier + init_phrase_index_meta(&_meta); + } + + void TearDown() override { + config::snii_prx_zstd_level = _saved_prx_level; + config::snii_prx_zstd_level_direct_load = _saved_load_level; + config::snii_bigram_defer_build_to_compaction = _saved_defer; + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(kTestDir).ok()); + } + + std::string test_path(const std::string& name) const { + return std::string(kTestDir) + "/" + name + ".idx"; + } + + TabletIndex _meta; + +private: + int32_t _saved_prx_level = 9; + int32_t _saved_load_level = 3; + bool _saved_defer = false; +}; + +// Contract 1: the tier changes prx bytes, never query semantics. Widest level +// split (base 19 vs load 3) so the size delta cannot vanish into zstd noise. +TEST_F(SniiPrxTierWriterTest, DirectLoadUsesLoadTierPrxLevelWithIdenticalAnswers) { + const std::vector rows = positional_rows(); + config::snii_prx_zstd_level = 19; + config::snii_prx_zstd_level_direct_load = 3; + + const std::string full_path = test_path("full_level"); + const std::string direct_path = test_path("direct_level"); + write_segment(full_path, _meta, rows, DirectLoadHint::kNone); + write_segment(direct_path, _meta, rows, DirectLoadHint::kDirect); + + const std::string full_bytes = read_file_bytes(full_path); + const std::string direct_bytes = read_file_bytes(direct_path); + // Level 3 compresses the prx region less than level 19: the direct segment + // must be strictly larger (this is ALSO the observable proving the tier + // actually took effect -- there is no level field in the format). + EXPECT_GT(direct_bytes.size(), full_bytes.size()); + + ::doris::snii::io::LocalFileReader full_file, direct_file; + ::doris::snii::reader::SniiSegmentReader full_segment, direct_segment; + ::doris::snii::reader::LogicalIndexReader full_idx, direct_idx; + open_index(full_path, &full_file, &full_segment, &full_idx); + open_index(direct_path, &direct_file, &direct_segment, &direct_idx); + + // Position-dependent answers must be identical and non-trivial across + // several adjacent pairs of the generated pattern. + bool any_nonempty = false; + for (const auto& phrase : std::vector> { + {"alpha", "theta"}, {"kappa", "sigma"}, {"one", "eight"}, {"red", "cyan"}}) { + const std::vector expect = run_phrase(full_idx, phrase); + EXPECT_EQ(run_phrase(direct_idx, phrase), expect) << phrase[0] << ' ' << phrase[1]; + any_nonempty |= !expect.empty(); + } + EXPECT_TRUE(any_nonempty) << "test corpus produced no phrase matches -- assertions vacuous"; +} + +// Contract 2: non-direct paths ignore the load-tier config completely. +TEST_F(SniiPrxTierWriterTest, NonDirectPathsIgnoreLoadTierLevel) { + const std::vector rows = positional_rows(); + config::snii_prx_zstd_level = 9; + + config::snii_prx_zstd_level_direct_load = 3; + const std::string none_lo = test_path("none_lo"); + const std::string notdirect_lo = test_path("notdirect_lo"); + write_segment(none_lo, _meta, rows, DirectLoadHint::kNone); + write_segment(notdirect_lo, _meta, rows, DirectLoadHint::kNotDirect); + + config::snii_prx_zstd_level_direct_load = 19; + const std::string none_hi = test_path("none_hi"); + write_segment(none_hi, _meta, rows, DirectLoadHint::kNone); + + const std::string baseline = read_file_bytes(none_lo); + ASSERT_FALSE(baseline.empty()); + EXPECT_EQ(read_file_bytes(notdirect_lo), baseline); // explicit not-direct == no hint + EXPECT_EQ(read_file_bytes(none_hi), baseline); // load-tier value is inert here +} + +// Contract 3: the LEVEL is read at flush (same semantics as +// snii_prx_zstd_level): a change landing after the captured hint but before +// finish() takes effect -- equal to the constant-config twin of the NEW value. +TEST_F(SniiPrxTierWriterTest, MidLoadLevelChangeLandsAtFlush) { + const std::vector rows = positional_rows(); + config::snii_prx_zstd_level = 9; + + config::snii_prx_zstd_level_direct_load = 9; + const std::string ref9 = test_path("flip_ref9"); + write_segment(ref9, _meta, rows, DirectLoadHint::kDirect); + + config::snii_prx_zstd_level_direct_load = 3; // captured hint under level 3 ... + const int32_t flip_to = 9; // ... flipped to 9 before any row + const std::string flipped = test_path("flip_to9"); + write_segment(flipped, _meta, rows, DirectLoadHint::kDirect, &flip_to); + + EXPECT_EQ(read_file_bytes(flipped), read_file_bytes(ref9)); +} + +// Contract 4: the load-tier level is clamped to [3, 19]. +TEST_F(SniiPrxTierWriterTest, LoadTierLevelClamped) { + const std::vector rows = positional_rows(); + config::snii_prx_zstd_level = 9; + + config::snii_prx_zstd_level_direct_load = 1; // below floor -> 3 + const std::string lvl_under = test_path("clamp_under"); + write_segment(lvl_under, _meta, rows, DirectLoadHint::kDirect); + config::snii_prx_zstd_level_direct_load = 3; + const std::string lvl_floor = test_path("clamp_floor"); + write_segment(lvl_floor, _meta, rows, DirectLoadHint::kDirect); + EXPECT_EQ(read_file_bytes(lvl_under), read_file_bytes(lvl_floor)); + + config::snii_prx_zstd_level_direct_load = 25; // above ceiling -> 19 + const std::string lvl_over = test_path("clamp_over"); + write_segment(lvl_over, _meta, rows, DirectLoadHint::kDirect); + config::snii_prx_zstd_level_direct_load = 19; + const std::string lvl_ceil = test_path("clamp_ceil"); + write_segment(lvl_ceil, _meta, rows, DirectLoadHint::kDirect); + EXPECT_EQ(read_file_bytes(lvl_over), read_file_bytes(lvl_ceil)); +} + +} // namespace +} // namespace doris::segment_v2 diff --git a/be/test/storage/index/snii/writer/snii_writer_golden_bytes_test.cpp b/be/test/storage/index/snii/writer/snii_writer_golden_bytes_test.cpp new file mode 100644 index 00000000000000..be1e1d7a92b698 --- /dev/null +++ b/be/test/storage/index/snii/writer/snii_writer_golden_bytes_test.cpp @@ -0,0 +1,216 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// GOLDEN BYTE pins for the SNII writer token path (T1a guard). Each test +// writes ONE segment from a FIXED corpus through the production writer stack +// and asserts the output file's FNV-1a-64 digest against a recorded constant +// (harvested from the pre-T1a materializing token path). Any change to +// tokenization semantics, position accounting, bigram emission, ignore_above / +// empty-value handling, or on-disk encoding flips the digest -- the T1a +// streaming rewrite must keep every digest EXACTLY. +// +// The corpus deliberately hits the edge lanes: empty value (analyzed: skipped; +// keyword: a VALID empty token), punctuation-only row (zero analyzed tokens), +// >ignore_above value (keyword: skipped row), long token, repeated terms +// (position increments + bigram df), unicode/mixed text, multiple add_values +// batches, and interleaved add_nulls runs. +// +// If a digest changes INTENTIONALLY (format or analyzer change), re-harvest by +// running the test and copying the "actual=" value from the failure message -- +// and say so loudly in the commit message. + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/status.h" +#include "io/fs/local_file_system.h" +#include "storage/index/index_file_writer.h" +#include "storage/index/snii/snii_index_writer.h" +#include "storage/tablet/tablet_schema.h" +#include "util/slice.h" + +namespace doris::segment_v2 { +namespace { + +constexpr int64_t kIndexId = 9; +constexpr const char* kTestDir = "./ut_dir/snii_writer_golden_bytes_test"; + +void assert_ok(const Status& status) { + ASSERT_TRUE(status.ok()) << status.to_string(); +} + +TabletIndex make_meta(const std::map& properties) { + TabletIndexPB pb; + pb.set_index_type(IndexType::INVERTED); + pb.set_index_id(kIndexId); + pb.set_index_name("golden_idx"); + pb.add_col_unique_id(0); + for (const auto& [k, v] : properties) { + pb.mutable_properties()->insert({k, v}); + } + TabletIndex meta; + meta.init_from_pb(pb); + return meta; +} + +uint64_t fnv1a64(const std::string& bytes) { + uint64_t h = 1469598103934665603ULL; + for (unsigned char c : bytes) { + h ^= c; + h *= 1099511628211ULL; + } + return h; +} + +std::string read_file_bytes(const std::string& path) { + std::ifstream in(path, std::ios::binary); + EXPECT_TRUE(in.good()) << path; + std::ostringstream out; + out << in.rdbuf(); + return out.str(); +} + +void add_batch(SniiIndexColumnWriter* writer, const std::vector& rows) { + std::vector slices; + slices.reserve(rows.size()); + for (const std::string& row : rows) { + slices.emplace_back(row); + } + assert_ok(writer->add_values("c1", slices.data(), slices.size())); +} + +// The fixed corpus: two add_values batches with add_nulls runs interleaved. +void feed_corpus(SniiIndexColumnWriter* writer) { + add_batch(writer, { + "hello world hello doris", + "", // analyzed: no tokens; keyword: valid EMPTY token + "The QUICK brown-fox; jumped!! over_the lazy dog 42 times", + "重复 重复 重复 词元 Doris 数据库 全文检索 mixed 中英 tokens", + }); + assert_ok(writer->add_nulls(3)); + add_batch(writer, { + std::string(300, 'x'), // keyword: > ignore_above(256) -> skipped + "single", + "!!! ??? ,,,", // analyzed: zero tokens survive + "hello world again and again and again", + }); + assert_ok(writer->add_nulls(1)); +} + +// Writes the corpus through the production stack and returns the segment +// file's digest. +uint64_t golden_digest(const std::string& name, const TabletIndex& meta) { + const std::string path = std::string(kTestDir) + "/" + name + ".idx"; + io::FileWriterPtr file_writer; + EXPECT_TRUE(io::global_local_filesystem()->create_file(path, &file_writer).ok()); + IndexFileWriter index_file_writer(io::global_local_filesystem(), path, "golden_rowset", + /*seg_id=*/0, InvertedIndexStorageFormatPB::SNII, + std::move(file_writer), /*can_use_ram_dir=*/true, + /*tablet_id=*/900); + SniiIndexColumnWriter writer(&index_file_writer, &meta, /*single_field=*/true); + EXPECT_TRUE(writer.init().ok()); + feed_corpus(&writer); + EXPECT_TRUE(writer.finish().ok()); + EXPECT_TRUE(index_file_writer.begin_close().ok()); + EXPECT_TRUE(index_file_writer.finish_close().ok()); + const std::string bytes = read_file_bytes(path); + EXPECT_FALSE(bytes.empty()); + return fnv1a64(bytes); +} + +class SniiWriterGoldenBytes : public testing::Test { +protected: + void SetUp() override { + assert_ok(io::global_local_filesystem()->delete_directory(kTestDir)); + assert_ok(io::global_local_filesystem()->create_directory(kTestDir)); + // Pin every config the write path reads, so the digests do not move + // under future default changes (the golden guards the TOKEN PATH). + _saved_defer = config::snii_bigram_defer_build_to_compaction; + _saved_prune_min = config::snii_bigram_prune_min_df; + _saved_prune_max = config::snii_bigram_prune_max_df_ratio; + _saved_dict_lvl = config::snii_dict_block_zstd_level; + _saved_prx_lvl = config::snii_prx_zstd_level; + _saved_prx_load_lvl = config::snii_prx_zstd_level_direct_load; + config::snii_bigram_defer_build_to_compaction = false; + config::snii_bigram_prune_min_df = -1; // auto (default) + config::snii_bigram_prune_max_df_ratio = 0.25; + config::snii_dict_block_zstd_level = 3; + config::snii_prx_zstd_level = 3; + config::snii_prx_zstd_level_direct_load = 3; + } + + void TearDown() override { + config::snii_bigram_defer_build_to_compaction = _saved_defer; + config::snii_bigram_prune_min_df = _saved_prune_min; + config::snii_bigram_prune_max_df_ratio = _saved_prune_max; + config::snii_dict_block_zstd_level = _saved_dict_lvl; + config::snii_prx_zstd_level = _saved_prx_lvl; + config::snii_prx_zstd_level_direct_load = _saved_prx_load_lvl; + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(kTestDir).ok()); + } + +private: + bool _saved_defer = false; + int32_t _saved_prune_min = -1; + double _saved_prune_max = 0.25; + int32_t _saved_dict_lvl = 3; + int32_t _saved_prx_lvl = 3; + int32_t _saved_prx_load_lvl = 3; +}; + +// Harvested from the pre-T1a materializing token path (vector + +// per-token std::string). The T1a streaming path must reproduce them exactly. +constexpr uint64_t kGoldenEnglishPhrase = 0x09437a1fdef5f893ULL; +constexpr uint64_t kGoldenUnicodePhrase = 0xc096270a7dce3767ULL; +constexpr uint64_t kGoldenKeywordDocsOnly = 0x77f7ac39f81ab366ULL; + +TEST_F(SniiWriterGoldenBytes, EnglishPhrase) { + const TabletIndex meta = + make_meta({{"parser", "english"}, {"lower_case", "true"}, {"support_phrase", "true"}}); + const uint64_t digest = golden_digest("english_phrase", meta); + EXPECT_EQ(digest, kGoldenEnglishPhrase) + << "actual=0x" << std::hex << digest + << " -- token-path output changed; see file header before re-harvesting"; +} + +TEST_F(SniiWriterGoldenBytes, UnicodePhrase) { + const TabletIndex meta = + make_meta({{"parser", "unicode"}, {"lower_case", "true"}, {"support_phrase", "true"}}); + const uint64_t digest = golden_digest("unicode_phrase", meta); + EXPECT_EQ(digest, kGoldenUnicodePhrase) + << "actual=0x" << std::hex << digest + << " -- token-path output changed; see file header before re-harvesting"; +} + +TEST_F(SniiWriterGoldenBytes, KeywordDocsOnly) { + const TabletIndex meta = make_meta({{"ignore_above", "256"}}); + const uint64_t digest = golden_digest("keyword_docs_only", meta); + EXPECT_EQ(digest, kGoldenKeywordDocsOnly) + << "actual=0x" << std::hex << digest + << " -- token-path output changed; see file header before re-harvesting"; +} + +} // namespace +} // namespace doris::segment_v2 diff --git a/be/test/storage/index/snii/writer/spill_run_codec_test.cpp b/be/test/storage/index/snii/writer/spill_run_codec_test.cpp new file mode 100644 index 00000000000000..55f0a2c0ce526e --- /dev/null +++ b/be/test/storage/index/snii/writer/spill_run_codec_test.cpp @@ -0,0 +1,1053 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/spill_run_codec.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/encoding/varint.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +// doris::snii::Status was deleted in the Doris integration (R01); the codec now returns +// doris::Status. Corruption is surfaced via the INVERTED_INDEX_FILE_CORRUPTED +// error code (verified against the integrated spill_run_codec.cpp), not a generic +// CORRUPTION code, so the corruption assertions below check that code explicitly. +using doris::Status; +using doris::snii::writer::MergeRuns; +using doris::snii::writer::RunReader; +using doris::snii::writer::RunWriter; +using doris::snii::writer::TermPostings; + +namespace { + +std::string RunPath() { + static int counter = 0; + return "/tmp/snii_runcodec_" + std::to_string(getpid()) + "_" + std::to_string(counter++) + + ".run"; +} + +// RAII temp file: removed on scope exit so the suite leaves no debris. +struct TempRun { + std::string path = RunPath(); + ~TempRun() { std::remove(path.c_str()); } +}; + +// A run record is keyed by term-id; this pairs the id with the postings so the +// test can both write (by id) and assert (the resolved string round-trips). +struct IdTerm { + uint32_t id; + TermPostings tp; +}; + +TermPostings MakeTerm(std::vector docids, std::vector freqs, + std::vector> positions = {}) { + TermPostings tp; + tp.docids = std::move(docids); + tp.freqs = std::move(freqs); + tp.set_positions_per_doc(positions); // flatten per-doc lists into positions_flat + return tp; +} + +// Computes the term-id -> lexicographic rank array over a dense vocab, mirroring +// SpimiTermBuffer::ensure_string_rank(). MergeRuns now takes this dense integer rank +// as its heap/gather key (instead of comparing vocab strings inline), so the tests +// hand it the same lexicographic rank the production caller derives from the vocab. +std::vector LexRank(const std::vector& vocab) { + std::vector order(vocab.size()); + std::iota(order.begin(), order.end(), 0U); + std::ranges::sort(order, [&](uint32_t a, uint32_t b) { return vocab[a] < vocab[b]; }); + std::vector rank(vocab.size(), 0U); + for (uint32_t r = 0; r < order.size(); ++r) { + rank[order[r]] = r; + } + return rank; +} + +// Writes a single run from `terms` (by id) and reads it back, asserting an exact +// round-trip of every field. The reader leaves current().term empty (runs store +// only the id), so the term-id is checked via current_id(). +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +void RoundTrip(const std::vector& terms, bool has_positions) { + TempRun run; + { + RunWriter w; + ASSERT_TRUE(w.open(run.path).ok()); + for (const auto& t : terms) { + ASSERT_TRUE(w.write_term(t.id, t.tp).ok()); + } + ASSERT_TRUE(w.close().ok()); + } + RunReader r; + ASSERT_TRUE(r.open(run.path, has_positions).ok()); + for (const auto& expect : terms) { + ASSERT_FALSE(r.exhausted()); + EXPECT_EQ(r.current_id(), expect.id); + // Positions are LAZY: the count is known after advance(), the bytes only after + // materialize_positions(). + EXPECT_EQ(r.current_pos_count(), expect.tp.positions_flat.size()); + ASSERT_TRUE(r.materialize_positions().ok()); + const TermPostings& got = r.current(); + EXPECT_EQ(got.docids, expect.tp.docids); + EXPECT_EQ(got.freqs, expect.tp.freqs); + if (has_positions) { + EXPECT_EQ(got.positions_flat, expect.tp.positions_flat); + } + ASSERT_TRUE(r.advance().ok()); + } + EXPECT_TRUE(r.exhausted()); +} + +} // namespace + +// DoS prevention: a corrupt/truncated run whose n_docs length varint decodes to an +// absurd value must yield Corruption (bounded by the run's file size), NOT an +// uncaught std::bad_alloc from read_raw_u32's resize(). No docid data follows the +// huge count, so without the file-size bound this would resize() to ~4e9 u32s. +TEST(SniiSpillRunCodec, CorruptDocCountIsCorruptionNotBadAlloc) { + TempRun run; + { + // NOLINTBEGIN(clang-analyzer-unix.Stream): closed on the success path; only an + // ASSERT failure would skip fclose, which aborts the test anyway. + std::FILE* f = std::fopen(run.path.c_str(), "wb"); + ASSERT_NE(f, nullptr); + uint8_t buf[16]; + size_t n = 0; + n += doris::snii::encode_varint64(0, buf + n); // term_id = 0 + n += doris::snii::encode_varint64(0xFFFFFFFFULL, buf + n); // n_docs ~= 4e9, no data follows + ASSERT_EQ(std::fwrite(buf, 1, n, f), n); + std::fclose(f); + // NOLINTEND(clang-analyzer-unix.Stream) + } + RunReader r; + const Status s = r.open(run.path, /*has_positions=*/false); // open() -> advance() + EXPECT_TRUE(s.is()) << s; +} + +// Empty run: open succeeds, immediately exhausted, merge yields nothing. +TEST(SniiSpillRunCodec, EmptyRun) { + TempRun run; + RunWriter w; + ASSERT_TRUE(w.open(run.path).ok()); + ASSERT_TRUE(w.close().ok()); + RunReader r; + ASSERT_TRUE(r.open(run.path, /*has_positions=*/true).ok()); + EXPECT_TRUE(r.exhausted()); +} + +// Single doc, with positions: smallest non-trivial record round-trips. +TEST(SniiSpillRunCodec, SingleDocWithPositions) { + RoundTrip({{.id = 7, .tp = MakeTerm({7}, {3}, {{0, 4, 9}})}}, /*has_positions=*/true); +} + +// Docs-only run (no positions): positions field is zero and decode skips it. +TEST(SniiSpillRunCodec, NoPositions) { + RoundTrip( + {{.id = 0, .tp = MakeTerm({0, 5, 99}, {1, 2, 1})}, {.id = 1, .tp = MakeTerm({3}, {4})}}, + /*has_positions=*/false); +} + +// Several terms with varied widths round-trip in ascending id order. +TEST(SniiSpillRunCodec, MultiTermRoundTrip) { + RoundTrip( + { + {.id = 0, .tp = MakeTerm({0, 1, 2}, {1, 1, 1}, {{0}, {1}, {2}})}, + {.id = 1, .tp = MakeTerm({10}, {2}, {{3, 8}})}, + {.id = 2, .tp = MakeTerm({4, 100}, {2, 1}, {{0, 1}, {7}})}, + }, + /*has_positions=*/true); +} + +// K-way merge: a term-id present in EVERY run is concatenated in ascending run +// order; an id present in only ONE run passes through unchanged. The merged +// stream is ordered by each id's VOCAB STRING and the string is resolved onto +// the emitted TermPostings. +TEST(SniiSpillRunCodec, MergeConcatenatesAcrossRuns) { + // Vocab: id 0 -> "common", 1 -> "only0", 2 -> "zzz". Ordered by string: + // "common" < "only0" < "zzz", which happens to match id order here. + const std::vector vocab = {"common", "only0", "zzz"}; + TempRun r0, r1, r2; + // Each run covers a strictly later docid range for the shared id 0. + { + RunWriter w; + ASSERT_TRUE(w.open(r0.path).ok()); + ASSERT_TRUE(w.write_term(0, MakeTerm({0, 1}, {1, 2}, {{0}, {1, 2}})).ok()); + ASSERT_TRUE(w.write_term(1, MakeTerm({3}, {1}, {{5}})).ok()); + ASSERT_TRUE(w.close().ok()); + } + { + RunWriter w; + ASSERT_TRUE(w.open(r1.path).ok()); + ASSERT_TRUE(w.write_term(0, MakeTerm({5}, {1}, {{0}})).ok()); + ASSERT_TRUE(w.close().ok()); + } + { + RunWriter w; + ASSERT_TRUE(w.open(r2.path).ok()); + ASSERT_TRUE(w.write_term(0, MakeTerm({8, 9}, {1, 1}, {{0}, {0}})).ok()); + ASSERT_TRUE(w.write_term(2, MakeTerm({2}, {1}, {{4}})).ok()); + ASSERT_TRUE(w.close().ok()); + } + + std::vector merged; + ASSERT_TRUE(MergeRuns({r0.path, r1.path, r2.path}, vocab, LexRank(vocab), + /*has_positions=*/true, + [&](TermPostings&& tp) { merged.push_back(std::move(tp)); }) + .ok()); + + ASSERT_EQ(merged.size(), 3U); + EXPECT_EQ(merged[0].term, "common"); + EXPECT_EQ(merged[0].docids, (std::vector {0, 1, 5, 8, 9})); + EXPECT_EQ(merged[0].freqs, (std::vector {1, 2, 1, 1, 1})); + // Flat positions: doc0{0} doc1{1,2} doc5{0} doc8{0} doc9{0}. + EXPECT_EQ(merged[0].positions_flat, (std::vector {0, 1, 2, 0, 0, 0})); + EXPECT_EQ(std::vector(merged[0].doc_positions(1).begin(), + merged[0].doc_positions(1).end()), + (std::vector {1, 2})); + EXPECT_EQ(merged[1].term, "only0"); + EXPECT_EQ(merged[1].docids, (std::vector {3})); + EXPECT_EQ(merged[2].term, "zzz"); + EXPECT_EQ(merged[2].docids, (std::vector {2})); +} + +// BOUNDARY COALESCE with FLAT positions: a spill that falls BETWEEN two tokens of +// the SAME doc leaves that doc ending one run and beginning the next with the same +// docid. The merge must fold them into ONE doc whose positions concatenate (run +// order) into the correct flat layout -- the trickiest flat-positions merge path. +TEST(SniiSpillRunCodec, MergeCoalescesBoundaryDocPositionsFlat) { + const std::vector vocab = {"alpha"}; + TempRun r0, r1; + { + RunWriter w; + ASSERT_TRUE(w.open(r0.path).ok()); + // doc 0 (pos 0,7), doc 1 first half (pos 1) -- doc 1 continues in r1. + ASSERT_TRUE(w.write_term(0, MakeTerm({0, 1}, {2, 1}, {{0, 7}, {1}})).ok()); + ASSERT_TRUE(w.close().ok()); + } + { + RunWriter w; + ASSERT_TRUE(w.open(r1.path).ok()); + // doc 1 second half (pos 4,9), then doc 2 (pos 3). + ASSERT_TRUE(w.write_term(0, MakeTerm({1, 2}, {2, 1}, {{4, 9}, {3}})).ok()); + ASSERT_TRUE(w.close().ok()); + } + std::vector merged; + ASSERT_TRUE(MergeRuns({r0.path, r1.path}, vocab, LexRank(vocab), /*has_positions=*/true, + [&](TermPostings&& tp) { merged.push_back(std::move(tp)); }) + .ok()); + ASSERT_EQ(merged.size(), 1U); + EXPECT_EQ(merged[0].docids, (std::vector {0, 1, 2})); + // doc 1 coalesced: freq 1 + 2 = 3, positions 1,4,9 (run order). + EXPECT_EQ(merged[0].freqs, (std::vector {2, 3, 1})); + // Flat: doc0{0,7} doc1{1,4,9} doc2{3}. + EXPECT_EQ(merged[0].positions_flat, (std::vector {0, 7, 1, 4, 9, 3})); + EXPECT_EQ(std::vector(merged[0].doc_positions(1).begin(), + merged[0].doc_positions(1).end()), + (std::vector {1, 4, 9})); +} + +// The merge order follows the VOCAB STRING, not the numeric id: ids whose +// strings sort in the opposite order are emitted lexicographically. +TEST(SniiSpillRunCodec, MergeOrdersByVocabStringNotId) { + // id 0 -> "zebra", id 1 -> "apple": string order is apple(1) < zebra(0). + const std::vector vocab = {"zebra", "apple"}; + TempRun r0; + { + RunWriter w; + ASSERT_TRUE(w.open(r0.path).ok()); + // Written in run order by string: apple(1) before zebra(0). + ASSERT_TRUE(w.write_term(1, MakeTerm({2}, {1})).ok()); + ASSERT_TRUE(w.write_term(0, MakeTerm({5}, {1})).ok()); + ASSERT_TRUE(w.close().ok()); + } + std::vector order; + ASSERT_TRUE(MergeRuns({r0.path}, vocab, LexRank(vocab), /*has_positions=*/false, + [&](TermPostings&& tp) { order.push_back(tp.term); }) + .ok()); + EXPECT_EQ(order, (std::vector {"apple", "zebra"})); +} + +// Lazy positions: stream_positions yields the SAME bytes as the materialized +// block, even when pulled in awkward (non-block-aligned) chunk sizes that straddle +// the reader's internal 64 KiB window boundaries. +TEST(SniiSpillRunCodec, StreamPositionsMatchesMaterialized) { + TempRun run; + // One wide term: 5000 docs, freq 3 each -> 15000 flat positions spanning several + // internal read windows. + std::vector docids, freqs, flat; + for (uint32_t d = 0; d < 5000; ++d) { + docids.push_back(d); + freqs.push_back(3); + flat.push_back(d * 7 + 0); + flat.push_back(d * 7 + 1); + flat.push_back(d * 7 + 2); + } + TermPostings tp; + tp.docids = docids; + tp.freqs = freqs; + tp.positions_flat = flat; + { + RunWriter w; + ASSERT_TRUE(w.open(run.path).ok()); + ASSERT_TRUE(w.write_term(0, tp).ok()); + ASSERT_TRUE(w.close().ok()); + } + RunReader r; + ASSERT_TRUE(r.open(run.path, /*has_positions=*/true).ok()); + ASSERT_EQ(r.current_pos_count(), flat.size()); + ASSERT_EQ(r.positions_remaining(), flat.size()); + // Pull in odd chunks (7, 1000, 7, 1000, ...) until drained. + std::vector got; + std::vector chunks = {7, 1000, 7, 1000}; + size_t ci = 0; + while (r.positions_remaining() > 0) { + size_t want = std::min(chunks[ci % chunks.size()], + static_cast(r.positions_remaining())); + ++ci; + std::vector buf(want); + ASSERT_TRUE(r.stream_positions(buf.data(), want).ok()); + got.insert(got.end(), buf.begin(), buf.end()); + } + EXPECT_EQ(got, flat); + EXPECT_TRUE(r.positions_drained()); + ASSERT_TRUE(r.advance().ok()); + EXPECT_TRUE(r.exhausted()); +} + +// advance() after a PARTIALLY-streamed term skips the unread positions and lands +// on the next record correctly. +TEST(SniiSpillRunCodec, PartialStreamThenAdvanceSkipsRemainder) { + TempRun run; + { + RunWriter w; + ASSERT_TRUE(w.open(run.path).ok()); + ASSERT_TRUE(w.write_term(0, MakeTerm({0, 1, 2}, {2, 2, 2}, {{10, 11}, {20, 21}, {30, 31}})) + .ok()); + ASSERT_TRUE(w.write_term(1, MakeTerm({9}, {1}, {{99}})).ok()); + ASSERT_TRUE(w.close().ok()); + } + RunReader r; + ASSERT_TRUE(r.open(run.path, /*has_positions=*/true).ok()); + ASSERT_EQ(r.current_id(), 0U); + // Pull only the first two positions, then advance -- the remaining 4 are skipped. + std::vector buf(2); + ASSERT_TRUE(r.stream_positions(buf.data(), 2).ok()); + EXPECT_EQ(buf, (std::vector {10, 11})); + ASSERT_TRUE(r.advance().ok()); + ASSERT_FALSE(r.exhausted()); + EXPECT_EQ(r.current_id(), 1U); + ASSERT_TRUE(r.materialize_positions().ok()); + EXPECT_EQ(r.current().positions_flat, (std::vector {99})); +} + +namespace { + +// Drains a streamed merge term's pos_pump into a flat buffer (mirrors the windowed +// writer's synchronous consumption). Returns the merged term with positions +// realized so tests can compare against the materialized path. +TermPostings DrainStreamed(TermPostings&& tp) { + if (tp.pos_pump) { + tp.positions_flat.resize(static_cast(tp.pos_total)); + if (tp.pos_total != 0) { + tp.pos_pump(tp.positions_flat.data(), static_cast(tp.pos_total)); + } + tp.pos_pump = nullptr; + } + return std::move(tp); +} + +} // namespace + +// WIDE-TERM STREAMING == MATERIALIZED (byte-identity proof at the postings level): +// a term with df >= kSlimDfThreshold split across several runs (with a boundary doc +// straddling a spill) must yield IDENTICAL docids/freqs/positions whether the merge +// streams positions via pos_pump (allow_stream=true) or materializes them +// (allow_stream=false). Pulling the pump in document order reproduces the exact +// coalesced positions_flat. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiSpillRunCodec, MergeWideTermStreamsIdenticalToMaterialized) { + const std::vector vocab = {"wide"}; + // Build a wide term (df ~ 2000) sharded across 3 runs, with the LAST doc of each + // run continuing as the FIRST doc of the next (boundary-doc coalesce). + TempRun r0, r1, r2; + auto shard = [&](TempRun& run, uint32_t lo, uint32_t hi, uint32_t carry_first) { + TermPostings tp; + for (uint32_t d = lo; d < hi; ++d) { + tp.docids.push_back(d); + // Boundary docs (lo when it's a carry) get freq 1 here; otherwise freq 2. + const uint32_t fc = 2; + tp.freqs.push_back(fc); + for (uint32_t k = 0; k < fc; ++k) { + tp.positions_flat.push_back(d * 13 + k); + } + } + (void)carry_first; + RunWriter w; + ASSERT_TRUE(w.open(run.path).ok()); + ASSERT_TRUE(w.write_term(0, tp).ok()); + ASSERT_TRUE(w.close().ok()); + }; + // Ranges chosen so doc 700 ends r0 AND begins r1 (boundary), doc 1400 likewise. + // Encode the boundary by repeating that docid at the seam with extra positions. + { + TermPostings a; + for (uint32_t d = 0; d <= 700; ++d) { + a.docids.push_back(d); + a.freqs.push_back(2); + a.positions_flat.push_back(d * 13); + a.positions_flat.push_back(d * 13 + 1); + } + RunWriter w; + ASSERT_TRUE(w.open(r0.path).ok()); + ASSERT_TRUE(w.write_term(0, a).ok()); + ASSERT_TRUE(w.close().ok()); + } + { + TermPostings b; + // doc 700 continues here (boundary): extra positions for it, then 701..1400. + b.docids.push_back(700); + b.freqs.push_back(1); + b.positions_flat.push_back(700 * 13 + 2); + for (uint32_t d = 701; d <= 1400; ++d) { + b.docids.push_back(d); + b.freqs.push_back(2); + b.positions_flat.push_back(d * 13); + b.positions_flat.push_back(d * 13 + 1); + } + RunWriter w; + ASSERT_TRUE(w.open(r1.path).ok()); + ASSERT_TRUE(w.write_term(0, b).ok()); + ASSERT_TRUE(w.close().ok()); + } + { + TermPostings c; + c.docids.push_back(1400); + c.freqs.push_back(1); + c.positions_flat.push_back(1400 * 13 + 2); + for (uint32_t d = 1401; d <= 2100; ++d) { + c.docids.push_back(d); + c.freqs.push_back(2); + c.positions_flat.push_back(d * 13); + c.positions_flat.push_back(d * 13 + 1); + } + RunWriter w; + ASSERT_TRUE(w.open(r2.path).ok()); + ASSERT_TRUE(w.write_term(0, c).ok()); + ASSERT_TRUE(w.close().ok()); + } + (void)shard; + + const std::vector paths = {r0.path, r1.path, r2.path}; + TermPostings materialized, streamed; + ASSERT_TRUE(MergeRuns( + paths, vocab, LexRank(vocab), /*has_positions=*/true, + [&](TermPostings&& tp) { materialized = std::move(tp); }, + /*allow_stream_positions=*/false) + .ok()); + ASSERT_TRUE(MergeRuns( + paths, vocab, LexRank(vocab), /*has_positions=*/true, + [&](TermPostings&& tp) { streamed = DrainStreamed(std::move(tp)); }, + /*allow_stream_positions=*/true) + .ok()); + + // The materialized path filled positions_flat; the streamed path must too (after + // draining the pump) -- identical docids, freqs, and positions. + EXPECT_GE(materialized.docids.size(), 512U); // wide enough to take the stream path + EXPECT_EQ(materialized.docids, streamed.docids); + EXPECT_EQ(materialized.freqs, streamed.freqs); + EXPECT_EQ(materialized.positions_flat, streamed.positions_flat); + // Boundary doc 700 coalesced: freq 2 (r0) + 1 (r1) = 3, positions in run order. + const auto it = std::ranges::find(materialized.docids, 700U); + ASSERT_NE(it, materialized.docids.end()); + const size_t bi = static_cast(it - materialized.docids.begin()); + EXPECT_EQ(materialized.freqs[bi], 3U); +} + +// A run record whose term-id is >= vocab.size() must make MergeRuns return +// Corruption (NOT index a vocab[id] out of bounds, which is UB / a crash). The +// id is decoded as a perfectly valid varint, so it is the in-merge vocab-range +// check -- not varint decode -- that must fire. This guards both the heap-seed +// range check and the post-advance one by placing the bad id as the SECOND term +// (the first term seeds the heap fine; the bad id is reached after advance()). +TEST(SniiSpillRunCodec, MergeTermIdOutOfVocabIsCorruption) { + const std::vector vocab = {"only"}; // valid ids: {0} + TempRun run; + { + RunWriter w; + ASSERT_TRUE(w.open(run.path).ok()); + ASSERT_TRUE(w.write_term(0, MakeTerm({0}, {1}, {{0}})).ok()); // id 0: OK + ASSERT_TRUE(w.write_term(5, MakeTerm({9}, {1}, {{0}})).ok()); // id 5: out of range + ASSERT_TRUE(w.close().ok()); + } + std::vector merged; + const Status s = MergeRuns({run.path}, vocab, LexRank(vocab), /*has_positions=*/true, + [&](TermPostings&& tp) { merged.push_back(std::move(tp)); }); + EXPECT_TRUE(s.is()) << s; +} + +// And when the BAD id is the FIRST record of a run, the heap-seed range check (in +// MergeRuns, before any term is emitted) must fire -- still Corruption, no UB. +TEST(SniiSpillRunCodec, MergeFirstTermIdOutOfVocabIsCorruption) { + const std::vector vocab = {"a", "b"}; // valid ids: {0,1} + TempRun run; + { + RunWriter w; + ASSERT_TRUE(w.open(run.path).ok()); + ASSERT_TRUE(w.write_term(9, MakeTerm({0}, {1}, {{0}})).ok()); // id 9: out of range + ASSERT_TRUE(w.close().ok()); + } + std::vector merged; + const Status s = MergeRuns({run.path}, vocab, LexRank(vocab), /*has_positions=*/true, + [&](TermPostings&& tp) { merged.push_back(std::move(tp)); }); + EXPECT_TRUE(s.is()) << s; +} + +// DoS prevention on the POSITIONS length: a record whose declared n_pos varint +// exceeds what the file can hold must yield Corruption from the resize bound in +// read_raw_u32 (count > file_size_/4), NOT an uncaught std::bad_alloc. The n_pos +// varint decodes cleanly; it is the file-size bound -- not varint decode -- that +// must fire. We hand-craft a CRC-free run (the run codec has no CRC) with a valid +// term_id/n_docs/docids/freqs header, then an absurd n_pos and NO position data, so +// materialize_positions() hits the bound on resize(). +TEST(SniiSpillRunCodec, NPosExceedsFileIsCorruption) { + TempRun run; + { + // NOLINTBEGIN(clang-analyzer-unix.Stream): closed on the success path; only an + // ASSERT failure would skip fclose, which aborts the test anyway. + std::FILE* f = std::fopen(run.path.c_str(), "wb"); + ASSERT_NE(f, nullptr); + uint8_t buf[40]; + size_t n = 0; + n += doris::snii::encode_varint64(0, buf + n); // term_id = 0 + n += doris::snii::encode_varint64(1, buf + n); // n_docs = 1 + // docid[0] = 0 and freq[0] = 1 as RAW LE u32 blocks (matching the writer). + const uint32_t one_docid = 0, one_freq = 1; + std::memcpy(buf + n, &one_docid, sizeof(uint32_t)); + n += sizeof(uint32_t); + std::memcpy(buf + n, &one_freq, sizeof(uint32_t)); + n += sizeof(uint32_t); + n += doris::snii::encode_varint64(0xFFFFFFFFULL, buf + n); // n_pos ~= 4e9, no data follows + ASSERT_EQ(std::fwrite(buf, 1, n, f), n); + std::fclose(f); + // NOLINTEND(clang-analyzer-unix.Stream) + } + RunReader r; + // open() -> advance() decodes header + parks the (bogus) n_pos count, but does + // NOT read positions; materialize_positions() is where the resize bound fires. + ASSERT_TRUE(r.open(run.path, /*has_positions=*/true).ok()); + ASSERT_FALSE(r.exhausted()); + const Status s = r.materialize_positions(); + EXPECT_TRUE(s.is()) << s; +} + +// DETERMINISTIC: the wide-term pos_pump must NEVER hand the writer uninitialized +// bytes when the positions block is TRUNCATED mid-stream. We build a wide term +// (df >= kSlimDfThreshold so the merge takes the STREAMED pump path), chop the +// run so the position block ends early, then drive the pump exactly as the +// windowed writer does -- pre-poisoning the destination with a sentinel and +// asserting every unfilled tail slot is ZERO (the fix's memset), never the +// sentinel. The merge must still surface the latched Corruption afterward. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiSpillRunCodec, WideTermPumpZeroFillsTruncatedPositions) { + const std::vector vocab = {"wide"}; + const uint32_t kDocs = 600; // > kSlimDfThreshold (512) -> streamed path + static_assert(600U > doris::snii::format::kSlimDfThreshold, "must exceed slim df"); + TempRun run; + { + TermPostings tp; + for (uint32_t d = 0; d < kDocs; ++d) { + tp.docids.push_back(d); + tp.freqs.push_back(1); + tp.positions_flat.push_back(d * 3 + 1); // distinct non-zero positions + } + RunWriter w; + ASSERT_TRUE(w.open(run.path).ok()); + ASSERT_TRUE(w.write_term(0, tp).ok()); + ASSERT_TRUE(w.close().ok()); + } + // Determine the real on-disk size, then chop the tail of the POSITIONS block: + // drop the last 100 u32 positions so the pump runs out mid-stream. docids/freqs/ + // n_pos header are untouched (they precede the positions block), so the merge + // reaches the streamed pump and only stream_positions() hits the truncation. + struct stat st {}; + ASSERT_EQ(::stat(run.path.c_str(), &st), 0); + const off_t chopped = st.st_size - static_cast(100 * sizeof(uint32_t)); + ASSERT_GT(chopped, 0); + ASSERT_EQ(::truncate(run.path.c_str(), chopped), 0); + + // Drive the merge: capture the pump, then drain it into a sentinel-poisoned + // buffer (mirrors the windowed writer's synchronous consumption). The pump + // must zero-fill the unreachable tail rather than leave the sentinel in place. + constexpr uint32_t kSentinel = 0xDEADBEEFU; + bool saw_pump = false; + std::vector drained; + const Status s = MergeRuns( + {run.path}, vocab, LexRank(vocab), /*has_positions=*/true, + [&](TermPostings&& tp) { + ASSERT_TRUE(static_cast(tp.pos_pump)); // wide term -> streamed pump + saw_pump = true; + drained.assign(static_cast(tp.pos_total), kSentinel); + if (tp.pos_total != 0) { + tp.pos_pump(drained.data(), static_cast(tp.pos_total)); + } + }, + /*allow_stream_positions=*/true); + + // The merge must surface the truncation as Corruption (latched after fn()). + EXPECT_TRUE(s.is()) << s; + ASSERT_TRUE(saw_pump); + // The crux: NOT ONE slot may still hold the sentinel. Whatever the pump could + // not fill (the chopped tail) must be deterministic zero -- never uninitialized + // / never the poison value. (The leading slots it did fill are the real data.) + for (uint32_t v : drained) { + ASSERT_NE(v, kSentinel) << "pump left an unfilled slot uninitialized"; + } + // Stronger: the truncated tail (last 100 positions) must be exactly zero. + size_t zeros = 0; + for (size_t i = drained.size(); i-- > 0 && drained[i] == 0U;) { + ++zeros; + } + EXPECT_GE(zeros, 100U) << "chopped tail was not zero-filled"; +} + +// A truncated run file is rejected by decode (anti-corruption on bytes we read). +TEST(SniiSpillRunCodec, TruncatedRunIsCorruption) { + TempRun run; + { + RunWriter w; + ASSERT_TRUE(w.open(run.path).ok()); + ASSERT_TRUE(w.write_term(0, MakeTerm({0, 1, 2}, {1, 1, 1}, {{0}, {0}, {0}})).ok()); + ASSERT_TRUE(w.write_term(1, MakeTerm({4}, {1}, {{0}})).ok()); + ASSERT_TRUE(w.close().ok()); + } + // Chop the file so the second record promises more bytes than remain. + ASSERT_EQ(::truncate(run.path.c_str(), 4), 0); + RunReader r; + Status s = r.open(run.path, /*has_positions=*/true); + while (s.ok() && !r.exhausted()) { + s = r.advance(); + } + EXPECT_FALSE(s.ok()); +} + +// Ownership contract: a streamed pos_pump captured by a STORED TermPostings (a +// violation of the synchronous-consume-once contract) must throw if invoked after +// the merge returns, not dereference the freed merge-stack/reader state (UAF). +TEST(SniiSpillRunCodec, StreamedPumpThrowsWhenCalledAfterMerge) { + TempRun run; + // Wide term (>= kSlimDfThreshold=512 docs) so the merge streams positions via a + // pos_pump instead of materializing positions_flat. + std::vector docids, freqs, flat; + for (uint32_t d = 0; d < 1000; ++d) { + docids.push_back(d); + freqs.push_back(1); + flat.push_back(d); + } + TermPostings tp; + tp.docids = docids; + tp.freqs = freqs; + tp.positions_flat = flat; + { + RunWriter w; + ASSERT_TRUE(w.open(run.path).ok()); + ASSERT_TRUE(w.write_term(0, tp).ok()); + ASSERT_TRUE(w.close().ok()); + } + const std::vector vocab = {"wide"}; + // Deliberately violate the contract: STORE the streamed TermPostings, do not pump. + TermPostings stored; + ASSERT_TRUE(MergeRuns({run.path}, vocab, LexRank(vocab), /*has_positions=*/true, + [&](TermPostings&& t) { stored = std::move(t); }) + .ok()); + ASSERT_TRUE(static_cast(stored.pos_pump)); // streaming path was taken + EXPECT_TRUE(stored.positions_flat.empty()); // positions were not materialized + // The deferred call fails loudly instead of touching freed merge state. + std::vector buf(stored.pos_total != 0 ? stored.pos_total : 1); + EXPECT_THROW(stored.pos_pump(buf.data(), buf.size()), std::logic_error); +} + +// =========================================================================== +// SniiSpillMergeTest -- T15: MergeRuns keyed on the integer string_rank array. +// +// MergeRuns now takes a precomputed `string_rank` (term-id -> lexicographic rank) +// and keys its heap/gather on that dense 4 B integer array instead of comparing +// vocab strings inline. These cases prove (a) the key is the integer rank array +// (FM-04: a deliberately NON-lexicographic rank permutation drives the emit +// order), (b) output stays byte-identical when the rank is the lexicographic one +// (FM-01..FM-03, FM-09), (c) the wide-term streamed path is unaffected (FM-05), +// (d) the error/boundary paths (FM-06..FM-08), and (e) end-to-end spill == +// in-memory through SpimiTermBuffer's production wiring (FM-10). +// =========================================================================== + +namespace { + +// Writes one run file from (term-id, postings) pairs in the given order. The caller +// supplies them sorted by the MERGE KEY (the spill writer's contract: ascending by +// the same rank MergeRuns will use). Asserts on any I/O failure. +void WriteRun(const std::string& path, const std::vector& terms) { + RunWriter w; + ASSERT_TRUE(w.open(path).ok()); + for (const auto& t : terms) { + ASSERT_TRUE(w.write_term(t.id, t.tp).ok()); + } + ASSERT_TRUE(w.close().ok()); +} + +// K-way merges `paths` under the integer `rank` key, collecting every emitted term +// into `out` with positions always materialized (any streamed pos_pump is drained), +// so callers can compare positions_flat directly. Returns the merge Status. +Status CollectMerge(const std::vector& paths, const std::vector& vocab, + const std::vector& rank, bool has_positions, + std::vector* out, bool allow_stream_positions = true) { + return MergeRuns( + paths, vocab, rank, has_positions, + [&](TermPostings&& tp) { out->push_back(DrainStreamed(std::move(tp))); }, + allow_stream_positions); +} + +} // namespace + +// FM-04 (KEY PROOF): the heap/gather key is the integer string_rank ARRAY, not the +// vocab strings. The vocab sorts lexicographically as a(id1) < b(id0) < c(id2), but +// we pass a DIFFERENT permutation (id0->0, id2->1, id1->2), so the rank order is +// b(id0), c(id2), a(id1) -- matching neither the vocab string order (a,b,c) nor the +// numeric id order. The two runs hold DISJOINT-but-overlapping term sets so the heap +// must actually interleave them; the emitted order must follow the rank array. The +// OLD vocab-string comparator, fed these rank-sorted runs, would instead emit +// b,a,c, so this sequence equality FAILS on the un-optimized code and PASSES once +// the comparator keys on string_rank. +TEST(SniiSpillMergeTest, MergeRunsOrdersByStringRankInteger) { + const std::vector vocab = {"b", "a", "c"}; + const std::vector rank = {0, 2, 1}; // id0->0, id1->2, id2->1 + TempRun r0, r1; + // Each run sorted ascending by the merge key (rank): run0 = id0(0), id1(2); + // run1 = id0(0), id2(1). id0 ("b") appears in BOTH runs (integer-id gather). + WriteRun(r0.path, {{.id = 0, .tp = MakeTerm({0}, {1})}, {.id = 1, .tp = MakeTerm({2}, {1})}}); + WriteRun(r1.path, {{.id = 0, .tp = MakeTerm({5}, {1})}, {.id = 2, .tp = MakeTerm({3}, {1})}}); + std::vector merged; + ASSERT_TRUE( + CollectMerge({r0.path, r1.path}, vocab, rank, /*has_positions=*/false, &merged).ok()); + ASSERT_EQ(merged.size(), 3U); + std::vector order; + for (const auto& m : merged) { + order.push_back(m.term); + } + EXPECT_EQ(order, (std::vector {"b", "c", "a"})); // strictly the rank order + // id0 ("b") gathered across both runs in run order -> ascending docids. + EXPECT_EQ(merged[0].term, "b"); + EXPECT_EQ(merged[0].docids, (std::vector {0, 5})); + EXPECT_EQ(merged[1].term, "c"); + EXPECT_EQ(merged[1].docids, (std::vector {3})); + EXPECT_EQ(merged[2].term, "a"); + EXPECT_EQ(merged[2].docids, (std::vector {2})); +} + +// FM-01: lexicographic rank reproduces dictionary order; an id present in several +// runs concatenates in run order (docids stay ascending). No positions. +TEST(SniiSpillMergeTest, MergeByLexRankConcatenatesNoPositions) { + const std::vector vocab = {"banana", "apple", "cherry"}; // ids 0,1,2 + const std::vector rank = LexRank(vocab); // apple(1) < banana(0) < cherry(2) + TempRun r0, r1; + // Each run sorted by lex rank: apple(1), then banana(0) / cherry(2). + WriteRun(r0.path, + {{.id = 1, .tp = MakeTerm({0, 2}, {1, 1})}, {.id = 0, .tp = MakeTerm({1}, {1})}}); + WriteRun(r1.path, + {{.id = 1, .tp = MakeTerm({5}, {1})}, {.id = 2, .tp = MakeTerm({3, 9}, {1, 1})}}); + std::vector merged; + ASSERT_TRUE( + CollectMerge({r0.path, r1.path}, vocab, rank, /*has_positions=*/false, &merged).ok()); + ASSERT_EQ(merged.size(), 3U); + EXPECT_EQ(merged[0].term, "apple"); + EXPECT_EQ(merged[0].docids, (std::vector {0, 2, 5})); // r0{0,2} ++ r1{5} + EXPECT_EQ(merged[0].freqs, (std::vector {1, 1, 1})); + EXPECT_EQ(merged[1].term, "banana"); + EXPECT_EQ(merged[1].docids, (std::vector {1})); + EXPECT_EQ(merged[2].term, "cherry"); + EXPECT_EQ(merged[2].docids, (std::vector {3, 9})); +} + +// FM-02: same shape with positions -- positions_flat materializes correctly per term +// (document order, partitioned by freqs). +TEST(SniiSpillMergeTest, MergeByLexRankWithPositions) { + const std::vector vocab = {"banana", "apple", "cherry"}; + const std::vector rank = LexRank(vocab); + TempRun r0, r1; + WriteRun(r0.path, {{.id = 1, .tp = MakeTerm({0, 2}, {2, 1}, {{3, 4}, {7}})}, + {.id = 0, .tp = MakeTerm({1}, {1}, {{5}})}}); + WriteRun(r1.path, {{.id = 1, .tp = MakeTerm({5}, {2}, {{0, 9}})}, + {.id = 2, .tp = MakeTerm({3}, {1}, {{6}})}}); + std::vector merged; + ASSERT_TRUE( + CollectMerge({r0.path, r1.path}, vocab, rank, /*has_positions=*/true, &merged).ok()); + ASSERT_EQ(merged.size(), 3U); + EXPECT_EQ(merged[0].term, "apple"); + EXPECT_EQ(merged[0].docids, (std::vector {0, 2, 5})); + EXPECT_EQ(merged[0].freqs, (std::vector {2, 1, 2})); + // doc0{3,4} doc2{7} doc5{0,9} + EXPECT_EQ(merged[0].positions_flat, (std::vector {3, 4, 7, 0, 9})); + EXPECT_EQ(merged[1].positions_flat, (std::vector {5})); + EXPECT_EQ(merged[2].positions_flat, (std::vector {6})); +} + +// FM-03: a doc split across a spill boundary (last doc of run0 == first doc of run1) +// coalesces into one entry (freqs summed, positions spliced in run order). The merge +// key is the integer rank, but the Concat boundary path is unchanged. +TEST(SniiSpillMergeTest, MergeCoalescesBoundaryDoc) { + const std::vector vocab = {"x"}; + const std::vector rank = LexRank(vocab); // {0} + TempRun r0, r1; + // doc 0, then doc 4 (first half). doc 4 continues in r1. + WriteRun(r0.path, {{.id = 0, .tp = MakeTerm({0, 4}, {1, 2}, {{1}, {2, 3}})}}); + // doc 4 (second half pos 8), then doc 7. + WriteRun(r1.path, {{.id = 0, .tp = MakeTerm({4, 7}, {1, 1}, {{8}, {9}})}}); + std::vector merged; + ASSERT_TRUE( + CollectMerge({r0.path, r1.path}, vocab, rank, /*has_positions=*/true, &merged).ok()); + ASSERT_EQ(merged.size(), 1U); + EXPECT_EQ(merged[0].docids, (std::vector {0, 4, 7})); // one entry per docid + EXPECT_EQ(merged[0].freqs, (std::vector {1, 3, 1})); // doc4: 2 + 1 = 3 + EXPECT_EQ(merged[0].positions_flat, (std::vector {1, 2, 3, 8, 9})); // doc4: 2,3,8 +} + +// FM-05: a wide term (df >= kSlimDfThreshold) split across runs streams its positions +// via pos_pump under allow_stream_positions=true, yielding the identical +// docids/freqs/positions as the materialized path -- the rank change does not touch +// the streamed path. +TEST(SniiSpillMergeTest, MergeWideTermStreamsMatchesMaterialized) { + const std::vector vocab = {"wide"}; + const std::vector rank = LexRank(vocab); + TempRun r0, r1; + { + TermPostings a; + for (uint32_t d = 0; d <= 450; ++d) { // docs 0..450, freq 2 each + a.docids.push_back(d); + a.freqs.push_back(2); + a.positions_flat.push_back(d * 5); + a.positions_flat.push_back(d * 5 + 1); + } + WriteRun(r0.path, {{.id = 0, .tp = a}}); + } + { + TermPostings b; + b.docids.push_back(450); // boundary doc continues from r0 + b.freqs.push_back(1); + b.positions_flat.push_back(450 * 5 + 2); + for (uint32_t d = 451; d <= 900; ++d) { + b.docids.push_back(d); + b.freqs.push_back(2); + b.positions_flat.push_back(d * 5); + b.positions_flat.push_back(d * 5 + 1); + } + WriteRun(r1.path, {{.id = 0, .tp = b}}); + } + const std::vector paths = {r0.path, r1.path}; + TermPostings materialized, streamed; + ASSERT_TRUE(MergeRuns( + paths, vocab, rank, /*has_positions=*/true, + [&](TermPostings&& tp) { materialized = std::move(tp); }, + /*allow_stream_positions=*/false) + .ok()); + ASSERT_TRUE(MergeRuns( + paths, vocab, rank, /*has_positions=*/true, + [&](TermPostings&& tp) { streamed = DrainStreamed(std::move(tp)); }, + /*allow_stream_positions=*/true) + .ok()); + EXPECT_GE(materialized.docids.size(), + static_cast(doris::snii::format::kSlimDfThreshold)); + EXPECT_EQ(materialized.docids, streamed.docids); + EXPECT_EQ(materialized.freqs, streamed.freqs); + EXPECT_EQ(materialized.positions_flat, streamed.positions_flat); + // Boundary doc 450 coalesced: freq 2 (r0) + 1 (r1) = 3. + const auto it = std::ranges::find(materialized.docids, 450U); + ASSERT_NE(it, materialized.docids.end()); + EXPECT_EQ(materialized.freqs[static_cast(it - materialized.docids.begin())], 3U); +} + +// FM-06: a single run passes through unchanged; an empty run and an empty run-set +// both emit nothing and return OK (degenerate inputs). +TEST(SniiSpillMergeTest, MergeSingleRunAndEmptyInputs) { + const std::vector vocab = {"a", "b"}; + const std::vector rank = LexRank(vocab); + TempRun r0; + WriteRun(r0.path, + {{.id = 0, .tp = MakeTerm({1, 2}, {1, 1})}, {.id = 1, .tp = MakeTerm({3}, {1})}}); + std::vector merged; + ASSERT_TRUE(CollectMerge({r0.path}, vocab, rank, /*has_positions=*/false, &merged).ok()); + ASSERT_EQ(merged.size(), 2U); + EXPECT_EQ(merged[0].term, "a"); + EXPECT_EQ(merged[0].docids, (std::vector {1, 2})); + EXPECT_EQ(merged[1].term, "b"); + EXPECT_EQ(merged[1].docids, (std::vector {3})); + + // Empty run (no terms) -> fn never invoked. + TempRun empty; + WriteRun(empty.path, {}); + int calls = 0; + ASSERT_TRUE(MergeRuns({empty.path}, vocab, rank, /*has_positions=*/false, [&](TermPostings&&) { + ++calls; + }).ok()); + EXPECT_EQ(calls, 0); + + // No run paths at all -> also OK, zero calls. + calls = 0; + ASSERT_TRUE(MergeRuns({}, vocab, rank, /*has_positions=*/false, [&](TermPostings&&) { + ++calls; + }).ok()); + EXPECT_EQ(calls, 0); +} + +// FM-07: a run term-id >= vocab.size() is rejected as Corruption -- the +// current_id() < vocab.size() guards remain, so string_rank[term_id] is never +// indexed out of range. +TEST(SniiSpillMergeTest, MergeOutOfRangeTermIdIsCorruption) { + const std::vector vocab = {"only"}; // valid id 0 + const std::vector rank = LexRank(vocab); // size 1 + TempRun run; + WriteRun(run.path, {{.id = 0, .tp = MakeTerm({0}, {1})}, + {.id = 3, .tp = MakeTerm({9}, {1})}}); // id 3 out of range + std::vector merged; + const Status s = MergeRuns({run.path}, vocab, rank, /*has_positions=*/false, + [&](TermPostings&& tp) { merged.push_back(std::move(tp)); }); + EXPECT_TRUE(s.is()) << s; +} + +// FM-08: string_rank sized differently from vocab is an InternalError, rejected at the +// entry guard before any run is opened or any term emitted (the T15 guard). +TEST(SniiSpillMergeTest, MergeRankVocabSizeMismatchIsInternal) { + const std::vector vocab = {"a", "b", "c"}; + const std::vector rank = {0, 1}; // size 2 != vocab size 3 + TempRun run; + WriteRun(run.path, {{.id = 0, .tp = MakeTerm({0}, {1})}}); + int calls = 0; + const Status s = MergeRuns({run.path}, vocab, rank, /*has_positions=*/false, + [&](TermPostings&&) { ++calls; }); + EXPECT_TRUE(s.is()) << s; + EXPECT_EQ(calls, 0); // rejected before emitting anything +} + +// FM-09 (equivalence baseline): a richer scenario -- multiple terms across multiple +// runs, positions, and a boundary-doc overlap -- compared field-by-field against the +// hand-computed expected merged stream. With the lexicographic rank this pins the +// byte-identical output (== the old vocab-string-keyed semantics). +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiSpillMergeTest, MergeProducesByteIdenticalOutput) { + const std::vector vocab = {"delta", "alpha", "charlie"}; // ids 0,1,2 + const std::vector rank = LexRank(vocab); // alpha(1) merged; + ASSERT_TRUE( + CollectMerge({r0.path, r1.path, r2.path}, vocab, rank, /*has_positions=*/true, &merged) + .ok()); + ASSERT_EQ(merged.size(), 3U); + // alpha (id1): r0 docs{0,3} ++ r1 doc{3} -> doc3 coalesces (freq 2 + 1 = 3). + EXPECT_EQ(merged[0].term, "alpha"); + EXPECT_EQ(merged[0].docids, (std::vector {0, 3})); + EXPECT_EQ(merged[0].freqs, (std::vector {1, 3})); + EXPECT_EQ(merged[0].positions_flat, + (std::vector {2, 0, 5, 8})); // doc0{2} doc3{0,5,8} + // charlie (id2): r1 only. + EXPECT_EQ(merged[1].term, "charlie"); + EXPECT_EQ(merged[1].docids, (std::vector {2})); + EXPECT_EQ(merged[1].freqs, (std::vector {2})); + EXPECT_EQ(merged[1].positions_flat, (std::vector {1, 4})); + // delta (id0): r0 doc{1} ++ r2 docs{6,7}. + EXPECT_EQ(merged[2].term, "delta"); + EXPECT_EQ(merged[2].docids, (std::vector {1, 6, 7})); + EXPECT_EQ(merged[2].freqs, (std::vector {1, 1, 1})); + EXPECT_EQ(merged[2].positions_flat, (std::vector {9, 0, 0})); +} + +// FM-10 (end-to-end): a borrowed-vocab SpimiTermBuffer fed the SAME tokens produces +// byte-identical merged postings whether it stays in memory (threshold 0) or spills +// to many runs (tiny threshold) and goes through the rank-keyed k-way merge. This +// drives the production wiring (SpimiTermBuffer::merge_runs -> ensure_string_rank -> +// MergeRuns(string_rank_)) and proves spill == in-memory under the new integer key. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiSpillMergeTest, SpillMergeEqualsInMemory) { + using doris::snii::writer::SpimiTermBuffer; + // 6-id vocab in a NON-lexicographic id order, so the derived rank permutation is + // non-trivial (id order != string order). + const std::vector vocab = {"m", "g", "t", "a", "p", "c"}; + auto feed = [&](SpimiTermBuffer& buf) { + // Globally ascending docids; per term ascending docids; per (term,doc) 1..3 + // consecutive tokens (freq) with ascending positions. A sparse mask leaves some + // (term,doc) cells empty so terms get varied df and the merge must interleave. + for (uint32_t d = 0; d < 9; ++d) { + for (uint32_t id = 0; id < static_cast(vocab.size()); ++id) { + if (((d * 5 + id * 3) % 4) == 1) { + continue; // sparse: skip some (term,doc) + } + const uint32_t freq = 1 + ((d + id) % 3); // 1..3 tokens in this doc + for (uint32_t k = 0; k < freq; ++k) { + buf.add_token(id, d, /*pos=*/d * 50 + id * 7 + k); + } + } + } + }; + + std::vector in_memory; + { + SpimiTermBuffer buf(&vocab, /*has_positions=*/true, /*spill_threshold_bytes=*/0); + feed(buf); + ASSERT_TRUE(buf.for_each_term_sorted([&](TermPostings&& tp) { + in_memory.push_back(DrainStreamed(std::move(tp))); + }).ok()); + EXPECT_EQ(buf.run_count_for_test(), 0U); // pure in-memory: no spill + } + + std::vector spilled; + size_t runs = 0; + { + // Tiny threshold: the first 32 KiB arena block immediately exceeds it, so a + // spill fires repeatedly -> many small runs (each id lands in several runs and a + // multi-token doc straddles run seams -> exercises boundary-doc coalesce). + SpimiTermBuffer buf(&vocab, /*has_positions=*/true, /*spill_threshold_bytes=*/1); + feed(buf); + ASSERT_TRUE(buf.for_each_term_sorted([&](TermPostings&& tp) { + spilled.push_back(DrainStreamed(std::move(tp))); + }).ok()); + runs = buf.run_count_for_test(); + } + EXPECT_GT(runs, 1U); // the spill path actually fired multiple runs + + ASSERT_EQ(in_memory.size(), spilled.size()); + for (size_t i = 0; i < in_memory.size(); ++i) { + EXPECT_EQ(in_memory[i].term, spilled[i].term) << "term index " << i; + EXPECT_EQ(in_memory[i].docids, spilled[i].docids) << "docids of " << in_memory[i].term; + EXPECT_EQ(in_memory[i].freqs, spilled[i].freqs) << "freqs of " << in_memory[i].term; + EXPECT_EQ(in_memory[i].positions_flat, spilled[i].positions_flat) + << "positions of " << in_memory[i].term; + } +} diff --git a/be/test/storage/index/snii/writer/spillable_byte_buffer_test.cpp b/be/test/storage/index/snii/writer/spillable_byte_buffer_test.cpp new file mode 100644 index 00000000000000..d2be29d640d97f --- /dev/null +++ b/be/test/storage/index/snii/writer/spillable_byte_buffer_test.cpp @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/spillable_byte_buffer.h" + +#include + +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/io/file_writer.h" + +using doris::snii::writer::SpillableByteBuffer; +using doris::snii::Slice; +using doris::Status; + +namespace { + +// In-RAM sink: collects everything appended so a test can compare stream_into's +// output against the exact bytes that were fed in. +class CollectWriter : public doris::snii::io::FileWriter { +public: + Status append(Slice s) override { + bytes_.insert(bytes_.end(), s.data(), s.data() + s.size()); + written_ += s.size(); + return Status::OK(); + } + Status finalize() override { return Status::OK(); } + uint64_t bytes_written() const override { return written_; } + const std::vector& bytes() const { return bytes_; } + +private: + std::vector bytes_; + uint64_t written_ = 0; +}; + +std::vector Pattern(size_t n, uint8_t seed) { + std::vector v(n); + for (size_t i = 0; i < n; ++i) { + v[i] = static_cast((i * 31 + seed) & 0xFF); + } + return v; +} + +// Feeds `blocks` chunks of `block` bytes through a buffer with the given cap, then +// asserts: size() == total, spilled() matches expectation, and stream_into() +// reproduces the exact concatenation (RAM-resident or read back from the temp). +void RoundTrip(uint64_t cap, size_t block, int blocks, bool expect_spill) { + SpillableByteBuffer buf(cap, "test"); + std::vector expect; + for (int b = 0; b < blocks; ++b) { + const auto chunk = Pattern(block, static_cast(b)); + ASSERT_TRUE(buf.append(Slice(chunk)).ok()); + expect.insert(expect.end(), chunk.begin(), chunk.end()); + } + EXPECT_EQ(buf.size(), expect.size()); + EXPECT_EQ(buf.spilled(), expect_spill); + ASSERT_TRUE(buf.seal().ok()); + CollectWriter out; + ASSERT_TRUE(buf.stream_into(&out).ok()); + EXPECT_EQ(out.bytes(), expect); +} + +// append_move adopts the caller's vector; verify identical bytes for RAM + spill. +void RoundTripMove(uint64_t cap, size_t block, int blocks, bool expect_spill) { + SpillableByteBuffer buf(cap, "test"); + std::vector expect; + for (int b = 0; b < blocks; ++b) { + auto chunk = Pattern(block, static_cast(b + 7)); + expect.insert(expect.end(), chunk.begin(), chunk.end()); + ASSERT_TRUE(buf.append_move(std::move(chunk)).ok()); + } + EXPECT_EQ(buf.size(), expect.size()); + EXPECT_EQ(buf.spilled(), expect_spill); + ASSERT_TRUE(buf.seal().ok()); + CollectWriter out; + ASSERT_TRUE(buf.stream_into(&out).ok()); + EXPECT_EQ(out.bytes(), expect); +} + +} // namespace + +TEST(SniiSpillableByteBuffer, StaysInRamUnderCap) { + RoundTrip(/*cap=*/1U << 20, /*block=*/4096, /*blocks=*/4, /*expect_spill=*/false); +} + +TEST(SniiSpillableByteBuffer, SpillsOverCapAndRoundTripsByteForByte) { + // 8 x 4 KiB = 32 KiB through an 8 KiB cap -> spills after the 2nd block. + RoundTrip(/*cap=*/8192, /*block=*/4096, /*blocks=*/8, /*expect_spill=*/true); +} + +TEST(SniiSpillableByteBuffer, MaxCapNeverSpills) { + RoundTrip(/*cap=*/UINT64_MAX, /*block=*/65536, /*blocks=*/8, /*expect_spill=*/false); +} + +TEST(SniiSpillableByteBuffer, MoveAppendStaysInRam) { + RoundTripMove(/*cap=*/1U << 20, /*block=*/4096, /*blocks=*/4, /*expect_spill=*/false); +} + +TEST(SniiSpillableByteBuffer, MoveAppendSpillsAndRoundTrips) { + RoundTripMove(/*cap=*/8192, /*block=*/4096, /*blocks=*/8, /*expect_spill=*/true); +} + +TEST(SniiSpillableByteBuffer, EmptyBufferStreamsNothing) { + SpillableByteBuffer buf(1U << 20, "test"); + EXPECT_EQ(buf.size(), 0U); + EXPECT_FALSE(buf.spilled()); + ASSERT_TRUE(buf.seal().ok()); + CollectWriter out; + ASSERT_TRUE(buf.stream_into(&out).ok()); + EXPECT_TRUE(out.bytes().empty()); +} diff --git a/be/test/storage/index/snii/writer/spimi_bigram_drain_df_gate_test.cpp b/be/test/storage/index/snii/writer/spimi_bigram_drain_df_gate_test.cpp new file mode 100644 index 00000000000000..b90b4ccc026f45 --- /dev/null +++ b/be/test/storage/index/snii/writer/spimi_bigram_drain_df_gate_test.cpp @@ -0,0 +1,575 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/logical_index_writer.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" +#include "storage/index/snii_query_test_util.h" + +// G06: the flush-time phrase-bigram df prune (process_term's +// `df < bigram_prune_min_df` gate) is SUNK into the SPIMI drain for PAIR-KEYED +// bigram terms: a pair term whose EXACT df (Term::ndocs, valid while +// Term::sorted holds) is below the threshold is dropped at +// prepare_pair_terms_for_drain WITHOUT composing its term string, decoding its +// postings, or leaving any trace -- the same disposition process_term would +// give it after paying for all of that (on wikipedia, the low-df tail's +// materialize+emit was ~1/3 of build CPU). These tests pin: +// (1) the final-drain drop set equals process_term's df gate exactly +// (equality vs a pair-keyed-without-G06 control), drops never bloom, +// and the strings of dropped terms are never materialized; +// (2) the threshold boundary through the REAL flush plumbing +// (LogicalIndexWriter::build_blocks -> set_bigram_drain_min_df): +// df == threshold survives, df == threshold-1 drops, and the segment +// BYTES equal a string-keyed control's (whose bigrams only process_term +// gates -- G06 never touches non-pair-map terms); +// (3) EXACTNESS: an out-of-order docid revisit feed (5,1,5) can make ndocs +// overcount the coalesced df, so such terms are NEVER dropped at drain -- +// they materialize and process_term gates the exact coalesced df; +// (4) the mid-feed spill + final drain mix: df-below-threshold pair terms +// are dropped+BLOOMED at mid-feed spills (a reappearance must die at +// flush) but dropped UNbloomed at the final drain, and the emitted +// stream equals a no-spill control's. +using doris::Status; +using doris::snii::format::make_phrase_bigram_sentinel_term; +using doris::snii::format::make_phrase_bigram_term; +using doris::snii::writer::SpimiTermBuffer; +using doris::snii::writer::TermPostings; +using namespace doris::snii; +using namespace doris::snii::snii_test; +namespace spimi_testing = doris::snii::writer::testing; + +namespace { + +// Feeds one "document" of two adjacent words into `buf` through the G05 pair +// path exactly as the production column writer does: intern the unigrams first +// (capturing their ids), then add the pair by id. +void feed_pair_doc_ids(SpimiTermBuffer* buf, uint32_t docid, std::string_view l, std::string_view r, + uint32_t pos_base = 0) { + const uint32_t lid = buf->add_token_returning_id(l, docid, pos_base); + const uint32_t rid = buf->add_token_returning_id(r, docid, pos_base + 1); + ASSERT_NE(lid, SpimiTermBuffer::kInvalidTermId); + ASSERT_NE(rid, SpimiTermBuffer::kInvalidTermId); + buf->add_bigram_token(lid, rid, docid, pos_base); +} + +// The G01 string-keyed control feed for the identical logical stream. These +// terms live in the content-keyed intern set, NOT the pair map, so the G06 +// drain gate never sees them -- process_term's df gate is their only pruner. +void feed_pair_doc_strings(SpimiTermBuffer* buf, uint32_t docid, std::string_view l, + std::string_view r, uint32_t pos_base = 0) { + buf->add_token(l, docid, pos_base); + buf->add_token(r, docid, pos_base + 1); + buf->add_bigram_token(l, r, docid, pos_base); +} + +std::vector drain_ordered(SpimiTermBuffer* buf) { + std::vector out = buf->finalize_sorted(); + EXPECT_TRUE(buf->status().ok()) << buf->status().to_string(); + return out; +} + +void expect_same_stream(const std::vector& got, const std::vector& want, + const char* label) { + ASSERT_EQ(got.size(), want.size()) << label; + for (size_t i = 0; i < got.size(); ++i) { + EXPECT_EQ(got[i].term, want[i].term) << label << " term order diverged at " << i; + EXPECT_EQ(got[i].docids, want[i].docids) << label << " " << got[i].term; + EXPECT_EQ(got[i].freqs, want[i].freqs) << label << " " << got[i].term; + EXPECT_EQ(got[i].positions_flat, want[i].positions_flat) << label << " " << got[i].term; + } +} + +bool stream_has_term(const std::vector& terms, const std::string& term) { + for (const TermPostings& tp : terms) { + if (tp.term == term) { + return true; + } + } + return false; +} + +// Threshold-straddling fixture at kGateThreshold == 3: a df==3 survivor (the +// == threshold boundary), a df==2 boundary victim (== threshold - 1), a df==1 +// tail victim, a pair-less unigram and the sentinel (fed last, as the +// production writer's finish() does). +constexpr uint32_t kGateThreshold = 3; + +template +void feed_gate_stream(SpimiTermBuffer* buf, FeedPair&& feed_pair) { + feed_pair(buf, 0, "hot", "pair", 0); + feed_pair(buf, 1, "hot", "pair", 0); + feed_pair(buf, 2, "hot", "pair", 0); // df == 3 == threshold: SURVIVES + feed_pair(buf, 3, "mid", "tier", 0); + feed_pair(buf, 4, "mid", "tier", 0); // df == 2 == threshold-1: DROPS + feed_pair(buf, 5, "rare", "once", 0); // df == 1: DROPS + buf->add_token("zonly", 6, 7); // unigram with no pair + buf->add_token(make_phrase_bigram_sentinel_term(), 0, 0); +} + +Status build_segment(MemoryFile* file, SpimiTermBuffer* buf, uint32_t threshold, uint32_t doc_count, + uint64_t max_df = 0) { + writer::SniiIndexInput input; + input.index_id = 61; + input.index_suffix = "Body"; + input.config = format::IndexConfig::kDocsPositions; + input.doc_count = doc_count; + input.bigram_prune_min_df = threshold; + input.bigram_prune_max_df = max_df; + input.term_source = buf; + input.bigram_ever_dropped = buf->bigram_dropped_filter(); + writer::SniiCompoundWriter writer(file); + RETURN_IF_ERROR(writer.add_logical_index(input)); + return writer.finish(); +} + +std::vector file_bytes(MemoryFile* file) { + std::vector bytes; + assert_ok(file->read_at(0, file->size(), &bytes)); + return bytes; +} + +bool lookup_term(const reader::LogicalIndexReader& idx, const std::string& term, + format::DictEntry* entry) { + bool found = false; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + assert_ok(idx.lookup(term, &found, entry, &frq_base, &prx_base)); + return found; +} + +} // namespace + +// (1) BUFFER-LEVEL final-drain gate: the drop set is exactly {df < threshold}, +// drops never materialize a string, never create/insert into the drop bloom, +// and the surviving stream equals a pair-keyed control WITHOUT the gate minus +// the below-threshold bigram terms. +TEST(SniiSpimiBigramDrainDfGate, FinalDrainDropsExactLowDfPairsNoBloomNoStrings) { + SpimiTermBuffer gated(/*has_positions=*/true); + SpimiTermBuffer control(/*has_positions=*/true); // pair-keyed, NO G06 gate + gated.configure_bigram_diet(0); + control.configure_bigram_diet(0); + + feed_gate_stream(&gated, feed_pair_doc_ids); + feed_gate_stream(&control, feed_pair_doc_ids); + ASSERT_TRUE(gated.status().ok()); + ASSERT_TRUE(control.status().ok()); + + // The flush plumbs the EXACT threshold right before the drain; mimic it. + gated.set_bigram_drain_min_df(kGateThreshold); + + spimi_testing::reset_bigram_drain_df_drops(); + spimi_testing::reset_vocab_string_materialization_count(); + const std::vector got = drain_ordered(&gated); + // Exactly ONE pair term survived the gate, so the drain composed exactly + // ONE bigram string ("hot pair"); the two dropped pairs composed NOTHING. + EXPECT_EQ(spimi_testing::vocab_string_materialization_count(), 1U); + EXPECT_EQ(spimi_testing::bigram_drain_df_drops(), 2U); // (mid,tier) + (rare,once) + // Final-drain drops must leave NO bloom trace: the filter is created on + // first (bloomed) eviction only, and none happened. + EXPECT_EQ(gated.bigram_dropped_filter(), nullptr); + + const std::vector full = drain_ordered(&control); + // want = the ungated control stream minus the below-threshold bigrams. + std::vector want; + for (const TermPostings& tp : full) { + const bool prunable_bigram = format::is_phrase_bigram_term(tp.term) && + !format::is_phrase_bigram_sentinel_term(tp.term) && + tp.docids.size() < kGateThreshold; + if (!prunable_bigram) { + want.push_back(tp); + } + } + ASSERT_LT(want.size(), full.size()); // the fixture really exercised the gate + expect_same_stream(got, want, "final-drain gate"); + + // Boundary pins: df==threshold survived, df==threshold-1 and df==1 dropped. + EXPECT_TRUE(stream_has_term(got, make_phrase_bigram_term("hot", "pair"))); + EXPECT_FALSE(stream_has_term(got, make_phrase_bigram_term("mid", "tier"))); + EXPECT_FALSE(stream_has_term(got, make_phrase_bigram_term("rare", "once"))); + EXPECT_TRUE(stream_has_term(got, make_phrase_bigram_sentinel_term())); +} + +// (2) SEGMENT-LEVEL boundary through the REAL plumbing (build_blocks calls +// set_bigram_drain_min_df with the effective threshold): the pair-keyed build's +// BYTES equal the string-keyed control's, df==threshold got its dict entry, +// df==threshold-1 did not. +TEST(SniiSpimiBigramDrainDfGate, SegmentBytesEqualStringControlAndBoundaryHolds) { + for (const uint32_t threshold : {kGateThreshold, kGateThreshold + 1}) { + SpimiTermBuffer pair_buf(/*has_positions=*/true); + SpimiTermBuffer str_buf(/*has_positions=*/true); + pair_buf.configure_bigram_diet(0); + str_buf.configure_bigram_diet(0); + feed_gate_stream(&pair_buf, feed_pair_doc_ids); + feed_gate_stream(&str_buf, feed_pair_doc_strings); + ASSERT_TRUE(pair_buf.status().ok()); + ASSERT_TRUE(str_buf.status().ok()); + + spimi_testing::reset_bigram_drain_df_drops(); + MemoryFile pair_file; + assert_ok(build_segment(&pair_file, &pair_buf, threshold, /*doc_count=*/7)); + // threshold 3 drops (mid,tier)+(rare,once); threshold 4 also drops + // (hot,pair) -- the every-pair-dropped edge. + EXPECT_EQ(spimi_testing::bigram_drain_df_drops(), threshold == kGateThreshold ? 2U : 3U); + + MemoryFile str_file; + assert_ok(build_segment(&str_file, &str_buf, threshold, /*doc_count=*/7)); + // String-keyed bigrams are not pair-map entries: the G06 gate never + // fires for them (process_term pruned them instead)... + EXPECT_EQ(spimi_testing::bigram_drain_df_drops(), threshold == kGateThreshold ? 2U : 3U); + // ...yet the two builds' bytes are identical: the drain gate's drop set + // is exactly process_term's. + EXPECT_EQ(file_bytes(&pair_file), file_bytes(&str_file)) + << "threshold " << threshold << " bytes diverged"; + + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(reader::SniiSegmentReader::open(&pair_file, &segment_reader)); + assert_ok(segment_reader.open_index(61, "Body", &index_reader)); + format::DictEntry entry; + if (threshold == kGateThreshold) { + ASSERT_TRUE(lookup_term(index_reader, make_phrase_bigram_term("hot", "pair"), &entry)); + EXPECT_EQ(entry.df, 3U); // df == threshold SURVIVES, postings intact + } else { + EXPECT_FALSE(lookup_term(index_reader, make_phrase_bigram_term("hot", "pair"), &entry)); + } + EXPECT_FALSE(lookup_term(index_reader, make_phrase_bigram_term("mid", "tier"), &entry)); + EXPECT_FALSE(lookup_term(index_reader, make_phrase_bigram_term("rare", "once"), &entry)); + ASSERT_TRUE(lookup_term(index_reader, make_phrase_bigram_sentinel_term(), &entry)); + } +} + +// (3) EXACTNESS: an out-of-order docid REVISIT feed (5,1,5) makes Term::ndocs +// overcount the coalesced df (3 groups, 2 distinct docs), so the drain gate +// must NOT drop such terms -- they materialize and process_term gates the +// exact coalesced df. Both revisit shapes are pinned: one whose coalesced df +// is below the threshold (process_term prunes it) and one at/above it (it +// survives with coalesced postings) -- and the segment bytes still equal the +// string-keyed control's. +TEST(SniiSpimiBigramDrainDfGate, OutOfOrderRevisitFeedNeverDroppedAtDrain) { + auto feed = [](SpimiTermBuffer* buf, auto&& feed_pair) { + // Revisit-low: docids 5,1,5 -> ndocs 3, coalesced df 2 (< threshold 3). + feed_pair(buf, 5, "twist", "back", 0); + feed_pair(buf, 1, "twist", "back", 0); + feed_pair(buf, 5, "twist", "back", 10); // revisit: pos past the doc-5 first visit + // Revisit-high: docids 9,8,9,10 -> ndocs 4, coalesced df 3 (== threshold). + feed_pair(buf, 9, "loop", "guard", 0); + feed_pair(buf, 8, "loop", "guard", 0); + feed_pair(buf, 9, "loop", "guard", 10); + feed_pair(buf, 10, "loop", "guard", 0); + buf->add_token(make_phrase_bigram_sentinel_term(), 0, 0); + }; + + // Buffer-level: with the gate armed, NEITHER revisit term is dropped at + // the drain (unsorted terms are exempt), and the drop seam stays 0. + { + SpimiTermBuffer buf(/*has_positions=*/true); + buf.configure_bigram_diet(0); + feed(&buf, feed_pair_doc_ids); + ASSERT_TRUE(buf.status().ok()); + buf.set_bigram_drain_min_df(kGateThreshold); + spimi_testing::reset_bigram_drain_df_drops(); + const std::vector got = drain_ordered(&buf); + EXPECT_EQ(spimi_testing::bigram_drain_df_drops(), 0U); + EXPECT_TRUE(stream_has_term(got, make_phrase_bigram_term("twist", "back"))); + EXPECT_TRUE(stream_has_term(got, make_phrase_bigram_term("loop", "guard"))); + for (const TermPostings& tp : got) { + if (tp.term == make_phrase_bigram_term("twist", "back")) { + // Coalesced exactly as process_term will see it: df 2. + EXPECT_EQ(tp.docids, (std::vector {1, 5})); + EXPECT_EQ(tp.freqs, (std::vector {1, 2})); + } + } + } + + // Segment-level: process_term prunes revisit-low (exact df 2 < 3) and + // keeps revisit-high (exact df 3); bytes equal the string-keyed control. + SpimiTermBuffer pair_buf(/*has_positions=*/true); + SpimiTermBuffer str_buf(/*has_positions=*/true); + pair_buf.configure_bigram_diet(0); + str_buf.configure_bigram_diet(0); + feed(&pair_buf, feed_pair_doc_ids); + feed(&str_buf, feed_pair_doc_strings); + ASSERT_TRUE(pair_buf.status().ok()); + ASSERT_TRUE(str_buf.status().ok()); + + MemoryFile pair_file; + MemoryFile str_file; + assert_ok(build_segment(&pair_file, &pair_buf, kGateThreshold, /*doc_count=*/11)); + assert_ok(build_segment(&str_file, &str_buf, kGateThreshold, /*doc_count=*/11)); + EXPECT_EQ(file_bytes(&pair_file), file_bytes(&str_file)); + + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(reader::SniiSegmentReader::open(&pair_file, &segment_reader)); + assert_ok(segment_reader.open_index(61, "Body", &index_reader)); + format::DictEntry entry; + EXPECT_FALSE(lookup_term(index_reader, make_phrase_bigram_term("twist", "back"), &entry)); + ASSERT_TRUE(lookup_term(index_reader, make_phrase_bigram_term("loop", "guard"), &entry)); + EXPECT_EQ(entry.df, 3U); +} + +// (4) MID-FEED SPILL + FINAL DRAIN MIX: a df-in-[2,threshold) pair term is +// dropped AND BLOOMED at a mid-feed spill (a reappearance must die at flush); +// a df==1 pair term takes the unchanged G04 eviction; the survivor is +// materialized, written, and boundary-coalesced across runs; the residual +// (final) drain drops nothing bloomed; and the emitted stream equals a +// no-spill control that dropped the same terms at ITS final drain (unbloomed). +TEST(SniiSpimiBigramDrainDfGate, MidFeedSpillDropsBloomAndFinalDrainDropsDoNot) { + spimi_testing::reset_bigram_vocab_cap_counters(); + // Spilled build: gate threshold delivered UP FRONT through the new + // configure_bigram_diet parameter (what the production column writer does), + // so mid-feed spill drains can gate before the flush value exists. + SpimiTermBuffer spilled(/*has_positions=*/true, /*spill_threshold_bytes=*/40 * 1024); + spilled.configure_bigram_diet(/*vocab_cap_bytes=*/64 * 1024, kGateThreshold); + // Control: no spilling; same gate applied only at its final drain. The diet + // must be ON (huge cap = never evicts) so bigram position suppression matches + // the spilled buffer -- with the diet off the control keeps accumulating + // bigram positions and the drained streams differ in positions_flat even + // though the on-disk bytes (docs-only either way) agree. + SpimiTermBuffer control(/*has_positions=*/true); + control.configure_bigram_diet(/*vocab_cap_bytes=*/uint64_t(1) << 40); + control.set_bigram_drain_min_df(kGateThreshold); + + spimi_testing::reset_bigram_drain_df_drops(); + for (SpimiTermBuffer* buf : {&spilled, &control}) { + // Survivor: df 3 == threshold BEFORE the filler can trigger any spill. + feed_pair_doc_ids(buf, 0, "hot", "pair"); + feed_pair_doc_ids(buf, 1, "hot", "pair"); + feed_pair_doc_ids(buf, 2, "hot", "pair"); + // df==2 victim: below the threshold -> the NEW mid-feed df-gate drop. + feed_pair_doc_ids(buf, 3, "gone", "away"); + feed_pair_doc_ids(buf, 4, "gone", "away"); + // df==1 victim: the unchanged G04 eviction rule. + feed_pair_doc_ids(buf, 5, "solo", "once"); + // ~40+ KiB of unigram chain bytes force at least one gate-2 spill in + // the spilled build (established spill-test pattern). + for (uint32_t d = 6; d < 10000; ++d) { + buf->add_token("word", d, 0); + buf->add_token("more", d, 1); + } + // Post-spill survivor occurrences: cross-run postings must coalesce. + for (uint32_t d = 10000; d < 10005; ++d) { + feed_pair_doc_ids(buf, d, "hot", "pair"); + } + buf->add_token(make_phrase_bigram_sentinel_term(), 0, 0); + ASSERT_TRUE(buf->status().ok()); + } + ASSERT_GE(spilled.run_count_for_test(), 1U); + ASSERT_EQ(control.run_count_for_test(), 0U); + + // Mid-feed drops BLOOMED both victims in the spilled build: the df==2 one + // via the G06 gate (seam == 1: the df==1 one took the plain G04 eviction, + // which the gate seam deliberately excludes). + EXPECT_EQ(spimi_testing::bigram_drain_df_drops(), 1U); + ASSERT_NE(spilled.bigram_dropped_filter(), nullptr); + EXPECT_TRUE(spilled.bigram_dropped_filter()->maybe_contains( + make_phrase_bigram_term("gone", "away"))); + EXPECT_TRUE(spilled.bigram_dropped_filter()->maybe_contains( + make_phrase_bigram_term("solo", "once"))); + EXPECT_FALSE(spilled.bigram_dropped_filter()->maybe_contains( + make_phrase_bigram_term("hot", "pair"))); + + // Drain both; the spilled stream (k-way merged) equals the no-spill one. + // for_each_term_sorted streams positions: a WIDE term (df >= kSlimDfThreshold; + // the ~10k-doc filler unigrams here) arrives with positions_flat EMPTY and its + // positions delivered ONLY through pos_pump, valid solely inside fn() (the + // synchronous-consume-once contract on TermPostings::pos_pump). A retaining + // collector must therefore materialize the pump INSIDE fn() -- which also pins + // that the streamed positions equal the control's materialized ones. + std::vector got; + assert_ok(spilled.for_each_term_sorted([&](TermPostings&& tp) { + if (tp.pos_pump) { + tp.positions_flat.resize(tp.pos_total); + tp.pos_pump(tp.positions_flat.data(), tp.positions_flat.size()); + tp.pos_pump = nullptr; + tp.pos_total = 0; + } + got.push_back(std::move(tp)); + })); + ASSERT_TRUE(spilled.status().ok()); + spimi_testing::reset_bigram_drain_df_drops(); + const std::vector want = drain_ordered(&control); + // The control dropped BOTH victims at its FINAL drain through the gate + // (the df==1 term is not "evicted" there -- eviction is mid-feed-only)... + EXPECT_EQ(spimi_testing::bigram_drain_df_drops(), 2U); + // ...and final-drain drops never bloom: the control never created a filter. + EXPECT_EQ(control.bigram_dropped_filter(), nullptr); + + expect_same_stream(got, want, "spill mix"); + bool saw_hot = false; + for (const TermPostings& tp : got) { + EXPECT_NE(tp.term, make_phrase_bigram_term("gone", "away")); + EXPECT_NE(tp.term, make_phrase_bigram_term("solo", "once")); + if (tp.term == make_phrase_bigram_term("hot", "pair")) { + saw_hot = true; + std::vector want_docs {0, 1, 2}; + for (uint32_t d = 10000; d < 10005; ++d) { + want_docs.push_back(d); + } + EXPECT_EQ(tp.docids, want_docs); // pre- and post-spill runs coalesced + } + } + EXPECT_TRUE(saw_hot); +} + +// (5) G15 MAX gate on the UNSPILLED direct final drain (term_source, zero +// runs): the gate lives ONLY in process_term -- the drain (whose df-drop seam +// counts min-gate drops) drops nothing -- and the strictly-greater boundary +// holds: df == max materializes, df == max + 1 is pruned by the max seam. With +// the min gate off (max-only mode, the production shape when +// snii_bigram_prune_min_df == 0: the diet is never configured), bigram +// positions were buffered, so the SURVIVOR keeps its positions -- max-only +// pruning does not switch survivors to the min-mode docs-only layout. +TEST(SniiSpimiBigramDrainDfGate, MaxGateAppliesAtUnspilledFinalDrain) { + constexpr uint64_t kMaxDf = 3; + SpimiTermBuffer buf(/*has_positions=*/true); // no spill, no diet + feed_pair_doc_ids(&buf, 0, "hot", "stop"); + feed_pair_doc_ids(&buf, 1, "hot", "stop"); + feed_pair_doc_ids(&buf, 2, "hot", "stop"); + feed_pair_doc_ids(&buf, 3, "hot", "stop"); // df 4 == max + 1: PRUNED + feed_pair_doc_ids(&buf, 4, "mid", "band"); + feed_pair_doc_ids(&buf, 5, "mid", "band"); + feed_pair_doc_ids(&buf, 6, "mid", "band"); // df 3 == max: SURVIVES + buf.add_token(make_phrase_bigram_sentinel_term(), 0, 0); + ASSERT_TRUE(buf.status().ok()); + ASSERT_EQ(buf.run_count_for_test(), 0U); + + spimi_testing::reset_bigram_drain_df_drops(); + spimi_testing::reset_bigram_prune_counters(); + MemoryFile file; + assert_ok(build_segment(&file, &buf, /*threshold=*/0, /*doc_count=*/10, kMaxDf)); + + EXPECT_EQ(spimi_testing::bigram_drain_df_drops(), 0U); // max NEVER drops at drain + EXPECT_EQ(spimi_testing::bigram_terms_max_pruned(), 1U); + EXPECT_EQ(spimi_testing::bigram_terms_pruned(), 0U); + EXPECT_EQ(spimi_testing::bigram_terms_materialized(), 1U); + + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(reader::SniiSegmentReader::open(&file, &segment_reader)); + assert_ok(segment_reader.open_index(61, "Body", &index_reader)); + format::DictEntry entry; + EXPECT_FALSE(lookup_term(index_reader, make_phrase_bigram_term("hot", "stop"), &entry)); + ASSERT_TRUE(lookup_term(index_reader, make_phrase_bigram_term("mid", "band"), &entry)); + EXPECT_EQ(entry.df, kMaxDf); + // Max-only survivor keeps its (buffered) positions: legacy layout. + EXPECT_TRUE(entry.prx_len > 0 || !entry.prx_bytes.empty()); + ASSERT_TRUE(lookup_term(index_reader, make_phrase_bigram_sentinel_term(), &entry)); + EXPECT_EQ(index_reader.bigram_prune_min_df(), 0U); + EXPECT_EQ(index_reader.bigram_prune_max_df(), kMaxDf); +} + +// (6) G15 MID-FEED SPILL SAFETY -- the partial-count trap regression. A pair +// that covers 30% of every mid-feed prefix of the first half (ratio far above +// the production 0.2) but whose FINAL df lands UNDER the flush-resolved +// absolute bound (the doc count doubles in the pair-less second half) must +// SURVIVE: the max gate is applied ONLY at process_term with the final +// coalesced df -- no spill drain may compare a partial df against any +// doc-count-scaled bound. A genuinely ubiquitous pair fed in EVERY doc is +// still max-pruned at flush from its ACROSS-RUNS coalesced df, its drop +// counted by the max seam alone (the drain's min-df seam stays 0), and the +// spilled segment's bytes equal a no-spill control's. +TEST(SniiSpimiBigramDrainDfGate, MidFeedHighRatioPairSurvivesFinalMaxGateAcrossSpill) { + constexpr uint32_t kMinGate = 2; + constexpr uint32_t kHalf = 5000; + constexpr uint32_t kFinalDocs = 10005; + constexpr uint64_t kMaxDf = kFinalDocs / 5; // 2001, as ratio 0.2 resolves at flush + constexpr uint32_t kFrontLoadDf = kHalf / 10 * 3; // 1500: kMinGate <= 1500 <= kMaxDf + + // Spilled build: production shape (diet on, min gate delivered up front for + // mid-feed drains; huge vocab cap so no G04 eviction muddies the water). + SpimiTermBuffer spilled(/*has_positions=*/true, /*spill_threshold_bytes=*/40 * 1024); + spilled.configure_bigram_diet(/*vocab_cap_bytes=*/uint64_t(1) << 40, kMinGate); + // Control: identical feed and diet, never spills. + SpimiTermBuffer control(/*has_positions=*/true); + control.configure_bigram_diet(/*vocab_cap_bytes=*/uint64_t(1) << 40, kMinGate); + + for (SpimiTermBuffer* buf : {&spilled, &control}) { + for (uint32_t d = 0; d < kFinalDocs; ++d) { + if (d < kHalf && d % 10 < 3) { + // First half only, 30% density: partial ratio ~0.3 at EVERY + // mid-feed drain point, final ratio 1500/10005 ~ 0.15. + feed_pair_doc_ids(buf, d, "front", "load", 0); + } + // Ubiquitous pair: df == kFinalDocs > kMaxDf, spread across runs. + feed_pair_doc_ids(buf, d, "ubiq", "pair", 4); + // Wide filler unigrams (df ~10k >= 512: positions ride pos_pump + // through the writer's streaming drain) force gate-2 spills. + buf->add_token("word", d, 8); + buf->add_token("more", d, 9); + } + buf->add_token(make_phrase_bigram_sentinel_term(), 0, 0); + ASSERT_TRUE(buf->status().ok()); + // No pair ever had df < kMinGate at a drain and the cap never evicted: + // nothing may be bloom-recorded (build_segment hands the writer a null + // ever-dropped filter either way, keeping the two builds comparable). + EXPECT_EQ(buf->bigram_dropped_filter(), nullptr); + } + ASSERT_GE(spilled.run_count_for_test(), 1U); + ASSERT_EQ(control.run_count_for_test(), 0U); + + spimi_testing::reset_bigram_drain_df_drops(); + spimi_testing::reset_bigram_prune_counters(); + MemoryFile spilled_file; + assert_ok(build_segment(&spilled_file, &spilled, kMinGate, kFinalDocs, kMaxDf)); + EXPECT_EQ(spimi_testing::bigram_terms_max_pruned(), 1U); // (ubiq,pair) at flush ONLY + EXPECT_EQ(spimi_testing::bigram_terms_pruned(), 0U); + EXPECT_EQ(spimi_testing::bigram_terms_materialized(), 1U); // (front,load) + EXPECT_EQ(spimi_testing::bigram_drain_df_drops(), 0U); + + spimi_testing::reset_bigram_drain_df_drops(); + spimi_testing::reset_bigram_prune_counters(); + MemoryFile control_file; + assert_ok(build_segment(&control_file, &control, kMinGate, kFinalDocs, kMaxDf)); + EXPECT_EQ(spimi_testing::bigram_terms_max_pruned(), 1U); + EXPECT_EQ(spimi_testing::bigram_terms_pruned(), 0U); + EXPECT_EQ(spimi_testing::bigram_drain_df_drops(), 0U); + + EXPECT_EQ(file_bytes(&spilled_file), file_bytes(&control_file)); + + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(reader::SniiSegmentReader::open(&spilled_file, &segment_reader)); + assert_ok(segment_reader.open_index(61, "Body", &index_reader)); + format::DictEntry entry; + // THE regression pin: the high-partial-ratio pair SURVIVED, df coalesced + // across every spill run. + ASSERT_TRUE(lookup_term(index_reader, make_phrase_bigram_term("front", "load"), &entry)); + EXPECT_EQ(entry.df, kFrontLoadDf); + // Min-mode survivor layout (min gate active): docs-only, no positions. + EXPECT_EQ(entry.prx_len, 0U); + EXPECT_TRUE(entry.prx_bytes.empty()); + // The truly ubiquitous pair was max-pruned from its final coalesced df. + EXPECT_FALSE(lookup_term(index_reader, make_phrase_bigram_term("ubiq", "pair"), &entry)); + ASSERT_TRUE(lookup_term(index_reader, make_phrase_bigram_sentinel_term(), &entry)); + // Both applied gates land in the meta (reader fallback stays armed). + EXPECT_EQ(index_reader.bigram_prune_min_df(), kMinGate); + EXPECT_EQ(index_reader.bigram_prune_max_df(), kMaxDf); +} diff --git a/be/test/storage/index/snii/writer/spimi_bigram_pair_key_test.cpp b/be/test/storage/index/snii/writer/spimi_bigram_pair_key_test.cpp new file mode 100644 index 00000000000000..a67324a30de2ff --- /dev/null +++ b/be/test/storage/index/snii/writer/spimi_bigram_pair_key_test.cpp @@ -0,0 +1,521 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/logical_index_writer.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" +#include "storage/index/snii_query_test_util.h" + +// G05: pair-keyed bigram interning. The production writer interns each unigram +// once (capturing its term-id) and feeds adjacent pairs to +// add_bigram_token(left_id, right_id, ...), which keys the hidden bigram term +// by the uint64 (left_id << 32 | right_id) pair key -- no composed-string +// hashing, comparing, or storage on the accumulation path. The composed +// on-disk term string materializes inside the buffer only at spill/flush. +// These tests pin: +// (1) the pair path's drained stream (terms, ORDER, postings) and its on-disk +// segment BYTES are identical to the G01 string-keyed path's; +// (2) the pair-map hit/miss seams and the deferred-materialization seam; +// (3) G04 cap/eviction/bloom semantics are preserved under pair keying, +// including the bloom KEY contract (pair evictions are probe-able by the +// composed term string at flush); +// (4) the spill paths: df>=2 pair survivors are materialized at spill time +// and round-trip through the k-way merge in lexicographic order; df==1 +// pair terms are evicted-not-spilled on mid-feed spills (G04 rule). +using doris::Status; +using doris::snii::format::make_phrase_bigram_sentinel_term; +using doris::snii::format::make_phrase_bigram_term; +using doris::snii::writer::SpimiTermBuffer; +using doris::snii::writer::TermPostings; +using namespace doris::snii; +using namespace doris::snii::snii_test; +namespace spimi_testing = doris::snii::writer::testing; + +namespace { + +// Deterministic distinct alpha-only pair for filler bigrams (same generator as +// the G04 cap tests). +std::pair filler_pair(uint32_t i) { + std::string l = "u"; + l += static_cast('a' + (i / 26) % 26); + l += static_cast('a' + i % 26); + l += static_cast('a' + (i / 676) % 26); + std::string r = "v"; + r += static_cast('a' + i % 26); + r += static_cast('a' + (i / 26) % 26); + r += static_cast('a' + (i / 676) % 26); + return {std::move(l), std::move(r)}; +} + +// Feeds one "document" of two adjacent words into `buf` through the G05 pair +// path exactly as the production column writer does: intern the unigrams first +// (capturing their ids), then add the pair by id. +void feed_pair_doc_ids(SpimiTermBuffer* buf, uint32_t docid, std::string_view l, std::string_view r, + uint32_t pos_base = 0) { + const uint32_t lid = buf->add_token_returning_id(l, docid, pos_base); + const uint32_t rid = buf->add_token_returning_id(r, docid, pos_base + 1); + ASSERT_NE(lid, SpimiTermBuffer::kInvalidTermId); + ASSERT_NE(rid, SpimiTermBuffer::kInvalidTermId); + buf->add_bigram_token(lid, rid, docid, pos_base); +} + +// The G01 string-keyed control feed for the identical logical stream. +void feed_pair_doc_strings(SpimiTermBuffer* buf, uint32_t docid, std::string_view l, + std::string_view r, uint32_t pos_base = 0) { + buf->add_token(l, docid, pos_base); + buf->add_token(r, docid, pos_base + 1); + buf->add_bigram_token(l, r, docid, pos_base); +} + +// Drains preserving EMISSION ORDER (the property under test: pair terms must +// interleave with unigrams exactly where their composed bytes sort). +std::vector drain_ordered(SpimiTermBuffer* buf) { + std::vector out = buf->finalize_sorted(); + EXPECT_TRUE(buf->status().ok()) << buf->status().to_string(); + return out; +} + +void expect_same_stream(const std::vector& got, const std::vector& want, + const char* label) { + ASSERT_EQ(got.size(), want.size()) << label; + for (size_t i = 0; i < got.size(); ++i) { + EXPECT_EQ(got[i].term, want[i].term) << label << " term order diverged at " << i; + EXPECT_EQ(got[i].docids, want[i].docids) << label << " " << got[i].term; + EXPECT_EQ(got[i].freqs, want[i].freqs) << label << " " << got[i].term; + EXPECT_EQ(got[i].positions_flat, want[i].positions_flat) << label << " " << got[i].term; + } +} + +// One shared logical stream: two unigram-only fillers, a hot df==3 pair, an +// ambiguous-concatenation pair set ("ab c" vs "a bc"), a repeat-in-doc pair +// (freq 2), a df==1 rare pair, and the sentinel fed LAST at docid 0 (as the +// production writer's finish() does). +template +void feed_shared_stream(SpimiTermBuffer* buf, FeedPair&& feed_pair) { + feed_pair(buf, 0, "hot", "pair", 0); + feed_pair(buf, 1, "hot", "pair", 0); + buf->add_token("zonly", 1, 7); // unigram with no pair, positions exercised + feed_pair(buf, 2, "ab", "c", 0); + feed_pair(buf, 3, "a", "bc", 0); + feed_pair(buf, 4, "hot", "pair", 0); + // Same pair twice in one doc at ASCENDING positions -> freq 2, coalesced. + feed_pair(buf, 5, "twin", "beat", 0); + feed_pair(buf, 5, "twin", "beat", 2); + feed_pair(buf, 6, "rare", "once", 0); // df==1: pruned at flush by any threshold >= 2 + buf->add_token(make_phrase_bigram_sentinel_term(), 0, 0); +} + +} // namespace + +// (1a) In-memory drained stream identical to the string-keyed control, in diet +// mode (docs-only bigrams, as production prune mode implies). +TEST(SniiSpimiBigramPairKey, DrainedStreamEqualsStringPathControlDietMode) { + SpimiTermBuffer pair_buf(/*has_positions=*/true); + SpimiTermBuffer str_buf(/*has_positions=*/true); + pair_buf.configure_bigram_diet(0); + str_buf.configure_bigram_diet(0); + + feed_shared_stream(&pair_buf, feed_pair_doc_ids); + feed_shared_stream(&str_buf, feed_pair_doc_strings); + ASSERT_TRUE(pair_buf.status().ok()); + ASSERT_TRUE(str_buf.status().ok()); + EXPECT_EQ(pair_buf.total_tokens(), str_buf.total_tokens()); + + const std::vector got = drain_ordered(&pair_buf); + const std::vector want = drain_ordered(&str_buf); + expect_same_stream(got, want, "diet"); + // The pair terms really are in the stream (docs+freq only under the diet). + bool saw_hot = false; + for (const TermPostings& tp : got) { + if (tp.term == make_phrase_bigram_term("hot", "pair")) { + saw_hot = true; + EXPECT_EQ(tp.docids, (std::vector {0, 1, 4})); + EXPECT_TRUE(tp.positions_flat.empty()); + } + } + EXPECT_TRUE(saw_hot); +} + +// (1b) Legacy mode (no diet): bigram positions are preserved and still equal +// the string-keyed control byte for byte. +TEST(SniiSpimiBigramPairKey, DrainedStreamEqualsStringPathControlLegacyMode) { + SpimiTermBuffer pair_buf(/*has_positions=*/true); + SpimiTermBuffer str_buf(/*has_positions=*/true); + + feed_shared_stream(&pair_buf, feed_pair_doc_ids); + feed_shared_stream(&str_buf, feed_pair_doc_strings); + ASSERT_TRUE(pair_buf.status().ok()); + ASSERT_TRUE(str_buf.status().ok()); + + const std::vector got = drain_ordered(&pair_buf); + const std::vector want = drain_ordered(&str_buf); + expect_same_stream(got, want, "legacy"); + for (const TermPostings& tp : got) { + if (tp.term == make_phrase_bigram_term("twin", "beat")) { + EXPECT_EQ(tp.freqs, (std::vector {2})); + EXPECT_FALSE(tp.positions_flat.empty()); // legacy keeps bigram positions + } + } +} + +// (1c) The BUILT SEGMENT BYTES are identical between the two keying schemes for +// a fixed fixture -- the pair keying is purely an in-memory representation +// change; dict/posting bytes must not move. +TEST(SniiSpimiBigramPairKey, OnDiskBytesIdenticalToStringPathControl) { + constexpr uint32_t kThreshold = 2; // prunes the df==1 "rare once" pair at flush + auto build = [&](SpimiTermBuffer* buf, MemoryFile* file) { + writer::SniiIndexInput input; + input.index_id = 31; + input.index_suffix = "Body"; + input.config = format::IndexConfig::kDocsPositions; + input.doc_count = 7; + input.bigram_prune_min_df = kThreshold; + input.term_source = buf; + input.bigram_ever_dropped = buf->bigram_dropped_filter(); // null: no evictions + writer::SniiCompoundWriter writer(file); + assert_ok(writer.add_logical_index(input)); + assert_ok(writer.finish()); + }; + + SpimiTermBuffer pair_buf(/*has_positions=*/true); + SpimiTermBuffer str_buf(/*has_positions=*/true); + pair_buf.configure_bigram_diet(0); + str_buf.configure_bigram_diet(0); + feed_shared_stream(&pair_buf, feed_pair_doc_ids); + feed_shared_stream(&str_buf, feed_pair_doc_strings); + ASSERT_TRUE(pair_buf.status().ok()); + ASSERT_TRUE(str_buf.status().ok()); + + MemoryFile pair_file; + MemoryFile str_file; + build(&pair_buf, &pair_file); + build(&str_buf, &str_file); + + ASSERT_EQ(pair_file.size(), str_file.size()); + std::vector pair_bytes; + std::vector str_bytes; + assert_ok(pair_file.read_at(0, pair_file.size(), &pair_bytes)); + assert_ok(str_file.read_at(0, str_file.size(), &str_bytes)); + EXPECT_EQ(pair_bytes, str_bytes); +} + +// (2a) Pair-map seams: one miss per DISTINCT pair, one hit per repeat. +TEST(SniiSpimiBigramPairKey, PairMapSeamsCountHitsAndMisses) { + spimi_testing::reset_bigram_pair_map_counters(); + SpimiTermBuffer buf(/*has_positions=*/true); + buf.configure_bigram_diet(0); + + const uint32_t a = buf.add_token_returning_id("alpha", 0, 0); + const uint32_t b = buf.add_token_returning_id("beta", 0, 1); + const uint32_t c = buf.add_token_returning_id("gamma", 0, 2); + // Repeat unigram: same id back (unigram ids are stable). + EXPECT_EQ(buf.add_token_returning_id("alpha", 1, 0), a); + + for (uint32_t d = 0; d < 5; ++d) { + buf.add_bigram_token(a, b, d, 0); + } + buf.add_bigram_token(b, c, 9, 0); + ASSERT_TRUE(buf.status().ok()); + + EXPECT_EQ(spimi_testing::bigram_pair_map_misses(), 2U); // (a,b) and (b,c) + EXPECT_EQ(spimi_testing::bigram_pair_map_hits(), 4U); // 4 repeats of (a,b) + EXPECT_EQ(buf.bigram_pair_terms_for_test(), 2U); +} + +// (2b) NO bigram string is composed during accumulation; the composed strings +// materialize exactly once per live pair term, at drain. +TEST(SniiSpimiBigramPairKey, BigramStringsMaterializeOnlyAtDrain) { + SpimiTermBuffer buf(/*has_positions=*/true); + buf.configure_bigram_diet(0); + + spimi_testing::reset_vocab_string_materialization_count(); + const uint32_t l = buf.add_token_returning_id("left", 0, 0); + const uint32_t r = buf.add_token_returning_id("right", 0, 1); + EXPECT_EQ(spimi_testing::vocab_string_materialization_count(), 2U); // the unigrams + + for (uint32_t d = 0; d < 100; ++d) { + buf.add_bigram_token(l, r, d, 0); + } + const auto [l2s, r2s] = filler_pair(7); + const uint32_t l2 = buf.add_token_returning_id(l2s, 100, 0); + const uint32_t r2 = buf.add_token_returning_id(r2s, 100, 1); + buf.add_bigram_token(l2, r2, 100, 0); + ASSERT_TRUE(buf.status().ok()); + // 100 pair adds of 2 distinct pairs composed NOTHING (only the 2 extra + // unigram interns count). + EXPECT_EQ(spimi_testing::vocab_string_materialization_count(), 4U); + + const std::vector terms = drain_ordered(&buf); + // Drain materialized exactly the 2 live pair terms' composed strings. + EXPECT_EQ(spimi_testing::vocab_string_materialization_count(), 6U); + bool saw = false; + for (const TermPostings& tp : terms) { + saw |= tp.term == make_phrase_bigram_term("left", "right"); + } + EXPECT_TRUE(saw); +} + +// (3) G04 cap/eviction/bloom preserved under pair keying, end-to-end through +// the real flush: only the df==1 tail is evicted (bounded intern bytes), the +// bloom is PROBE-ABLE BY THE COMPOSED TERM STRING (the pair eviction inserted +// the piecewise content hash -- the key contract), an evicted-then-reappearing +// pair is dropped at flush by the bloom, and hot/df>=2 pairs plus the sentinel +// are untouched. +TEST(SniiSpimiBigramPairKey, CapEvictionAndBloomPreservedUnderPairKeying) { + constexpr uint64_t kCap = 2 * 1024; + constexpr uint32_t kThreshold = 4; + spimi_testing::reset_bigram_vocab_cap_counters(); + doris::snii::writer::testing::reset_bigram_prune_counters(); + + SpimiTermBuffer buf(/*has_positions=*/true); + buf.configure_bigram_diet(kCap); + + // Hot survivor: df well past the threshold before any cap pressure. + const uint32_t hot_l = buf.add_token_returning_id("hot", 0, 0); + const uint32_t hot_r = buf.add_token_returning_id("pair", 0, 1); + for (uint32_t d = 0; d < 8; ++d) { + buf.add_bigram_token(hot_l, hot_r, d, 0); + } + // The victim pair: one occurrence -> df==1. + const uint32_t gone = buf.add_token_returning_id("gone", 10, 0); + const uint32_t away = buf.add_token_returning_id("away", 10, 1); + buf.add_bigram_token(gone, away, 10, 0); + const std::string victim = make_phrase_bigram_term("gone", "away"); + + // Blow the cap with unique df==1 tail pairs until the victim is provably + // evicted; the bloom probe uses the COMPOSED STRING, which must agree with + // the piecewise pair-key insertion. + uint32_t docid = 20; + for (uint32_t i = 0; i < 4000; ++i) { + if (buf.bigram_dropped_filter() != nullptr && + buf.bigram_dropped_filter()->maybe_contains(victim)) { + break; + } + const auto [l, r] = filler_pair(i); + feed_pair_doc_ids(&buf, docid++, l, r); + } + ASSERT_NE(buf.bigram_dropped_filter(), nullptr); + ASSERT_TRUE(buf.bigram_dropped_filter()->maybe_contains(victim)); + EXPECT_GT(spimi_testing::bigram_evictions(), 0U); + EXPECT_GT(spimi_testing::vocab_cap_sweeps(), 0U); + // The cap bounded the pair-map intern accounting (fixed bytes/entry). + EXPECT_LE(buf.bigram_intern_bytes(), kCap + 2048); + // The hot pair was never evicted. + EXPECT_FALSE( + buf.bigram_dropped_filter()->maybe_contains(make_phrase_bigram_term("hot", "pair"))); + + // The victim REAPPEARS far past the threshold; the bloom must still drop it + // at flush (its earlier posting for doc 10 is gone -- incomplete). + for (uint32_t d = 5000; d < 5200; ++d) { + buf.add_bigram_token(gone, away, d, 0); + } + buf.add_token(make_phrase_bigram_sentinel_term(), 0, 0); + ASSERT_TRUE(buf.status().ok()); + + MemoryFile file; + writer::SniiIndexInput input; + input.index_id = 12; + input.index_suffix = "Body"; + input.config = format::IndexConfig::kDocsPositions; + input.doc_count = 5200; + input.bigram_prune_min_df = kThreshold; + input.term_source = &buf; + input.bigram_ever_dropped = buf.bigram_dropped_filter(); + + writer::SniiCompoundWriter writer(&file); + assert_ok(writer.add_logical_index(input)); + assert_ok(writer.finish()); + + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(reader::SniiSegmentReader::open(&file, &segment_reader)); + assert_ok(segment_reader.open_index(input.index_id, input.index_suffix, &index_reader)); + + auto lookup = [&](const std::string& term, format::DictEntry* entry) { + bool found = false; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + assert_ok(index_reader.lookup(term, &found, entry, &frq_base, &prx_base)); + return found; + }; + + format::DictEntry entry; + EXPECT_FALSE(lookup(victim, &entry)); + EXPECT_EQ(doris::snii::writer::testing::bigram_bloom_dropped(), 1U); + ASSERT_TRUE(lookup(make_phrase_bigram_term("hot", "pair"), &entry)); + EXPECT_EQ(entry.df, 8U); + EXPECT_EQ(entry.prx_len, 0U); // docs-only bigram (G01 layout) + ASSERT_TRUE(lookup(make_phrase_bigram_sentinel_term(), &entry)); +} + +// (4a) Spill round trip with a pair SURVIVOR: df>=2 pair terms are materialized +// at spill time, written to runs by id, and the k-way merge re-emits the whole +// stream in lexicographic composed-string order with coalesced postings equal +// to a no-spill control. +TEST(SniiSpimiBigramPairKey, SpillWithPairSurvivorRoundTripMatchesNoSpillControl) { + // Threshold below the 32 KiB arena block size -> a spill on (nearly) every + // token (the established spill-test pattern), so the survivor's occurrences + // land in DIFFERENT runs and must boundary-coalesce at merge. + SpimiTermBuffer spilled(/*has_positions=*/true, /*spill_threshold_bytes=*/4096); + SpimiTermBuffer control(/*has_positions=*/true); + spilled.configure_bigram_diet(0); // suppression, NO cap: nothing is evicted + control.configure_bigram_diet(0); + + for (SpimiTermBuffer* buf : {&spilled, &control}) { + for (uint32_t d = 0; d < 10; ++d) { + const uint32_t w = buf->add_token_returning_id("word", d, 0); + ASSERT_NE(w, SpimiTermBuffer::kInvalidTermId); + const uint32_t aa = buf->add_token_returning_id("aa", d, 1); + const uint32_t bb = buf->add_token_returning_id("bb", d, 2); + buf->add_bigram_token(aa, bb, d, 1); + buf->add_bigram_token(aa, bb, d, 5); // same doc twice: freq 2 + } + // One df==1 pair; with NO cap it must survive the spill unharmed. + const uint32_t r1 = buf->add_token_returning_id("rare", 10, 0); + const uint32_t r2 = buf->add_token_returning_id("once", 10, 1); + buf->add_bigram_token(r1, r2, 10, 0); + ASSERT_TRUE(buf->status().ok()); + } + EXPECT_GE(spilled.run_count_for_test(), 2U); + EXPECT_EQ(control.run_count_for_test(), 0U); + + std::vector got; + assert_ok( + spilled.for_each_term_sorted([&](TermPostings&& tp) { got.push_back(std::move(tp)); })); + const std::vector want = drain_ordered(&control); + expect_same_stream(got, want, "spill"); + + // Explicit order pin: the emission is sorted by the FINAL composed bytes + // (marker byte 0x1F sorts the hidden bigrams before the ASCII unigrams). + for (size_t i = 1; i < got.size(); ++i) { + EXPECT_LT(got[i - 1].term, got[i].term) << "emission order not lexicographic at " << i; + } + bool saw_pair = false; + for (const TermPostings& tp : got) { + if (tp.term == make_phrase_bigram_term("aa", "bb")) { + saw_pair = true; + std::vector want_docs(10); + for (uint32_t d = 0; d < 10; ++d) { + want_docs[d] = d; + } + EXPECT_EQ(tp.docids, want_docs); + EXPECT_EQ(tp.freqs, std::vector(10, 2)); // boundary-coalesced + EXPECT_TRUE(tp.positions_flat.empty()); // diet suppression + } + } + EXPECT_TRUE(saw_pair); +} + +// (4b) Mid-feed spills EVICT df==1 pair terms instead of spilling them (they +// would pin their id + a composed string in the run), recording them in the +// bloom; df>=2 pair terms are materialized and written. Same G04 rule as the +// string path. +TEST(SniiSpimiBigramPairKey, MidFeedSpillEvictsDfOnePairsInsteadOfSpilling) { + spimi_testing::reset_bigram_vocab_cap_counters(); + // Threshold ABOVE one 32 KiB arena block but below two: the first spill + // fires only once the chain bytes claim a second block, i.e. AFTER the hot + // pair below has reached df==2 (df==1 at spill time would evict it). + SpimiTermBuffer buf(/*has_positions=*/true, /*spill_threshold_bytes=*/40 * 1024); + buf.configure_bigram_diet(64 * 1024); // eviction enabled; cap never binding + + // Survivor first: df==2 long before the first spill can consider it. + const uint32_t hl = buf.add_token_returning_id("hot", 0, 0); + const uint32_t hr = buf.add_token_returning_id("pair", 0, 1); + buf.add_bigram_token(hl, hr, 0, 0); + buf.add_bigram_token(hl, hr, 1, 0); + // df==1 victim, then enough unigram tokens (~40 KiB of chain bytes) to + // force at least one gate-2 spill past it. + const uint32_t vl = buf.add_token_returning_id("gone", 2, 0); + const uint32_t vr = buf.add_token_returning_id("away", 2, 1); + buf.add_bigram_token(vl, vr, 2, 0); + for (uint32_t d = 3; d < 10000; ++d) { + buf.add_token("word", d, 0); + buf.add_token("more", d, 1); + } + ASSERT_TRUE(buf.status().ok()); + ASSERT_GE(buf.run_count_for_test(), 1U); + + const std::string victim = make_phrase_bigram_term("gone", "away"); + ASSERT_NE(buf.bigram_dropped_filter(), nullptr); + EXPECT_TRUE(buf.bigram_dropped_filter()->maybe_contains(victim)); + EXPECT_GE(spimi_testing::bigram_evictions(), 1U); + + std::vector got; + assert_ok(buf.for_each_term_sorted([&](TermPostings&& tp) { got.push_back(std::move(tp)); })); + bool saw_hot = false; + for (size_t i = 0; i < got.size(); ++i) { + if (i > 0) { + EXPECT_LT(got[i - 1].term, got[i].term); + } + EXPECT_NE(got[i].term, victim); // evicted, never spilled + if (got[i].term == make_phrase_bigram_term("hot", "pair")) { + saw_hot = true; + EXPECT_EQ(got[i].docids, (std::vector {0, 1})); + } + } + EXPECT_TRUE(saw_hot); +} + +// (5) Contract violations latch InvalidArgument and drop the token: borrowed +// vocab mode, out-of-range ids, and a pair-term id as a constituent. +TEST(SniiSpimiBigramPairKey, InvalidInputsLatchStatus) { + { + const std::vector vocab {"a", "b"}; + SpimiTermBuffer borrowed(&vocab, /*has_positions=*/true); + borrowed.add_bigram_token(0U, 1U, 0, 0); + EXPECT_FALSE(borrowed.status().ok()); + EXPECT_EQ(borrowed.total_tokens(), 0U); + } + { + SpimiTermBuffer buf(/*has_positions=*/true); + const uint32_t a = buf.add_token_returning_id("a", 0, 0); + buf.add_bigram_token(a, a + 100, 0, 0); // right id never interned + EXPECT_FALSE(buf.status().ok()); + } + { + SpimiTermBuffer buf(/*has_positions=*/true); + const uint32_t a = buf.add_token_returning_id("aa", 0, 0); + const uint32_t b = buf.add_token_returning_id("bb", 0, 1); + buf.add_bigram_token(a, b, 0, 0); + ASSERT_TRUE(buf.status().ok()); + // The pair term's own id is the next fresh id (a, b, pair): using it as + // a constituent must be rejected. + const uint32_t pair_id = b + 1; + buf.add_bigram_token(pair_id, a, 1, 0); + EXPECT_FALSE(buf.status().ok()); + } + { + // add_token_returning_id on a borrowed-vocab buffer: kInvalidTermId. + const std::vector vocab {"a"}; + SpimiTermBuffer borrowed(&vocab, /*has_positions=*/true); + EXPECT_EQ(borrowed.add_token_returning_id("a", 0, 0), SpimiTermBuffer::kInvalidTermId); + EXPECT_FALSE(borrowed.status().ok()); + } +} diff --git a/be/test/storage/index/snii/writer/spimi_bigram_token_test.cpp b/be/test/storage/index/snii/writer/spimi_bigram_token_test.cpp new file mode 100644 index 00000000000000..d500c37b6db03f --- /dev/null +++ b/be/test/storage/index/snii/writer/spimi_bigram_token_test.cpp @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +// G01 part C: SpimiTermBuffer::add_bigram_token interns the synthetic +// marker+varint(len(left))+left+right term by PIECEWISE hash/compare, composing +// the owned std::string exactly once per DISTINCT pair. These tests pin (a) byte +// equivalence with the compose-then-add_token path, (b) the single-composition +// guarantee via the vocab-materialization seam, (c) that both entry points land +// on the SAME interned term, (d) that ambiguous concatenations stay distinct +// (the varint length prefix disambiguates), and (e) the owned-vocab-mode +// contract. +using doris::Status; +using doris::snii::format::make_phrase_bigram_term; +using doris::snii::writer::SpimiTermBuffer; +using doris::snii::writer::TermPostings; +namespace spimi_testing = doris::snii::writer::testing; + +namespace { + +void expect_same_postings(const std::vector& a, const std::vector& b) { + ASSERT_EQ(a.size(), b.size()); + for (size_t i = 0; i < a.size(); ++i) { + EXPECT_EQ(a[i].term, b[i].term); + EXPECT_EQ(a[i].docids, b[i].docids); + EXPECT_EQ(a[i].freqs, b[i].freqs); + EXPECT_EQ(a[i].positions_flat, b[i].positions_flat); + } +} + +} // namespace + +// The zero-alloc entry point accumulates byte-identically to composing the +// synthetic term and feeding add_token(string_view). +TEST(SniiSpimiBigramToken, MatchesComposedStringPath) { + const std::vector> pairs { + {"failed", "order"}, {"order", "ordinal"}, {"failed", "order"}, {"repeat", "repeat"}}; + + SpimiTermBuffer composed(/*has_positions=*/true); + SpimiTermBuffer piecewise(/*has_positions=*/true); + uint32_t docid = 0; + uint32_t pos = 0; + for (const auto& [left, right] : pairs) { + composed.add_token(make_phrase_bigram_term(left, right), docid, pos); + // Also mix in a real unigram so bigrams and unigrams share the intern set. + composed.add_token(left, docid, pos + 1); + piecewise.add_bigram_token(left, right, docid, pos); + piecewise.add_token(left, docid, pos + 1); + ++docid; + pos += 2; + } + ASSERT_TRUE(composed.status().ok()); + ASSERT_TRUE(piecewise.status().ok()); + EXPECT_EQ(piecewise.unique_terms(), composed.unique_terms()); + EXPECT_EQ(piecewise.total_tokens(), composed.total_tokens()); + + expect_same_postings(piecewise.finalize_sorted(), composed.finalize_sorted()); +} + +// The owned string is composed exactly once per DISTINCT pair: repeats are +// answered from the piecewise intern probe with zero materializations. +TEST(SniiSpimiBigramToken, RepeatPairMaterializesOnce) { + spimi_testing::reset_vocab_string_materialization_count(); + SpimiTermBuffer buf(/*has_positions=*/true); + for (uint32_t docid = 0; docid < 1000; ++docid) { + buf.add_bigram_token("failed", "order", docid, 0); + } + EXPECT_EQ(spimi_testing::vocab_string_materialization_count(), 1U); + + buf.add_bigram_token("order", "ordinal", 1000, 0); + EXPECT_EQ(spimi_testing::vocab_string_materialization_count(), 2U); + ASSERT_TRUE(buf.status().ok()); + EXPECT_EQ(buf.unique_terms(), 2U); + + const std::vector terms = buf.finalize_sorted(); + ASSERT_EQ(terms.size(), 2U); + // Lexicographic order: marker + varint(5) + "order"... sorts before + // marker + varint(6) + "failed"... (the length byte compares first). + EXPECT_EQ(terms[0].term, make_phrase_bigram_term("order", "ordinal")); + EXPECT_EQ(terms[0].docids, (std::vector {1000})); + EXPECT_EQ(terms[1].term, make_phrase_bigram_term("failed", "order")); + EXPECT_EQ(terms[1].docids.size(), 1000U); +} + +// Both entry points must land on the SAME interned term-id (one shared content +// hash/equality across composed-string and piecewise probes). +TEST(SniiSpimiBigramToken, MixedEntryPointsInternToOneTerm) { + spimi_testing::reset_vocab_string_materialization_count(); + SpimiTermBuffer buf(/*has_positions=*/true); + buf.add_token(make_phrase_bigram_term("failed", "order"), 1, 0); + buf.add_bigram_token("failed", "order", 2, 0); + buf.add_token(make_phrase_bigram_term("failed", "order"), 3, 0); + ASSERT_TRUE(buf.status().ok()); + EXPECT_EQ(buf.unique_terms(), 1U); + EXPECT_EQ(spimi_testing::vocab_string_materialization_count(), 1U); + + const std::vector terms = buf.finalize_sorted(); + ASSERT_EQ(terms.size(), 1U); + EXPECT_EQ(terms[0].docids, (std::vector {1, 2, 3})); +} + +// ("ab","c") and ("a","bc") concatenate identically; the varint(len(left)) +// fragment must keep them DISTINCT terms in both hash and equality. +TEST(SniiSpimiBigramToken, AmbiguousConcatenationStaysDistinct) { + SpimiTermBuffer buf(/*has_positions=*/true); + buf.add_bigram_token("ab", "c", 1, 0); + buf.add_bigram_token("a", "bc", 2, 0); + buf.add_bigram_token("ab", "c", 3, 1); + ASSERT_TRUE(buf.status().ok()); + EXPECT_EQ(buf.unique_terms(), 2U); + + const std::vector terms = buf.finalize_sorted(); + ASSERT_EQ(terms.size(), 2U); + // marker + varint(1) + "abc" sorts before marker + varint(2) + "abc". + EXPECT_EQ(terms[0].term, make_phrase_bigram_term("a", "bc")); + EXPECT_EQ(terms[0].docids, (std::vector {2})); + EXPECT_EQ(terms[1].term, make_phrase_bigram_term("ab", "c")); + EXPECT_EQ(terms[1].docids, (std::vector {1, 3})); +} + +// Same contract as add_token(string_view): interning is owned-vocab-mode only; a +// borrowed-vocab buffer latches InvalidArgument and ignores the token. +TEST(SniiSpimiBigramToken, RejectedInBorrowedVocabMode) { + const std::vector vocab {"aa", "bb"}; + SpimiTermBuffer buf(&vocab, /*has_positions=*/true); + buf.add_bigram_token("aa", "bb", 0, 0); + EXPECT_FALSE(buf.status().ok()); + EXPECT_EQ(buf.unique_terms(), 0U); + EXPECT_EQ(buf.total_tokens(), 0U); +} diff --git a/be/test/storage/index/snii/writer/spimi_global_memory_limiter_test.cpp b/be/test/storage/index/snii/writer/spimi_global_memory_limiter_test.cpp new file mode 100644 index 00000000000000..6b5a164578c584 --- /dev/null +++ b/be/test/storage/index/snii/writer/spimi_global_memory_limiter_test.cpp @@ -0,0 +1,530 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/writer/compact_posting_pool.h" +#include "storage/index/snii/writer/global_memory_limiter.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +// G09: process-wide build-RAM limiter. The per-writer gate-2 cap bounds ONE +// SPIMI accumulator; a concurrent load keeps (tablets x concurrency) writers +// alive at once, none of which may ever reach its own cap (wikipedia at +// concurrency 16: 100+ writers x 300-500 MB, ~41 GiB, zero per-writer spills). +// These tests pin the limiter's contract: +// (1) REGISTRY: register / absolute-report / unregister maintain the exact +// total and entry count; reports for unregistered flags are ignored; +// (2) SELECTION: over budget flags the largest-ARENA eligible buffers +// (arena >= the victim floor -- only the arena is reclaimable by a +// forced spill) until the flagged arena covers the overage -- counting +// already-pending flags -- and under budget (or budget 0 = off) never +// flags anything; +// (3) HONOR: the owner's next add_token observes a pending request, spills +// (run_count increments; global_forced_spills seam bumps) BYPASSING the +// G08 anti-churn floor but respecting the FORCED-SPILL FLOOR, and +// clears the flag; requests are advisory; +// (4) LIFETIME: attach registers the current resident bytes, the debounced +// report path keeps the registry equal to resident_bytes(), and the +// destructor un-registers; +// (5) THREADS: concurrent register / report / unregister (with flags dying +// right after unregister) is race-free -- the TSAN canary; +// (6) ANTI-STORM (the conc=16 wikipedia field failure): the floor makes a +// below-floor request a pending NO-OP, the cooldown exempts a +// just-spilled buffer until its arena regrows past the floor, and the +// budget-sanity check degrades to log-once-and-stop when the per-writer +// budget share cannot be met by spilling persistent-dominated buffers. +using doris::snii::writer::CompactPostingPool; +using doris::snii::writer::GlobalMemoryLimiter; +using doris::snii::writer::SpimiTermBuffer; +using doris::snii::writer::TermPostings; + +namespace snii_testing = doris::snii::writer::testing; + +namespace { + +constexpr int64_t kMiB = 1LL << 20; + +// Distinct short unigrams ("uaa", "uab", ...): SSO-sized, alpha-only, never +// bigram-marker-prefixed. +std::string unigram(uint32_t i) { + std::string s = "u"; + s += static_cast('a' + i % 26); + s += static_cast('a' + (i / 26) % 26); + s += static_cast('a' + (i / 676) % 26); + return s; +} + +// ---- (1) registry bookkeeping --------------------------------------------- + +TEST(SniiGlobalMemoryLimiter, RegistryAddRemoveTotal) { + GlobalMemoryLimiter lim; // budget defaults to 0 (off): pure tracking + std::atomic a {false}; + std::atomic b {false}; + EXPECT_EQ(lim.total_bytes(), 0); + EXPECT_EQ(lim.registered_count(), 0U); + + lim.register_buffer(&a, 100, 40); + lim.register_buffer(&b, 60, 10); + EXPECT_EQ(lim.total_bytes(), 160); + EXPECT_EQ(lim.registered_count(), 2U); + + lim.report(&a, 150, 90); // ABSOLUTE totals, not deltas + EXPECT_EQ(lim.total_bytes(), 210); + + lim.register_buffer(&a, 40, 5); // re-register updates in place, no duplicate + EXPECT_EQ(lim.total_bytes(), 100); + EXPECT_EQ(lim.registered_count(), 2U); + + lim.unregister_buffer(&a); + EXPECT_EQ(lim.total_bytes(), 60); + EXPECT_EQ(lim.registered_count(), 1U); + + lim.unregister_buffer(&a); // double-unregister is harmless + lim.report(&a, 999, 999); // a report for an unregistered flag is ignored + EXPECT_EQ(lim.total_bytes(), 60); + EXPECT_EQ(lim.registered_count(), 1U); + + lim.unregister_buffer(&b); + EXPECT_EQ(lim.total_bytes(), 0); + EXPECT_EQ(lim.registered_count(), 0U); + EXPECT_FALSE(a.load()); + EXPECT_FALSE(b.load()); +} + +// ---- (2) victim selection --------------------------------------------------- + +// MiB-scale entries with arena == resident keep the pre-floor selection shape +// visible while every entry clears the default 64 MiB victim floor AND every +// budget keeps budget/count above the 96 MiB/writer sanity minimum. +TEST(SniiGlobalMemoryLimiter, OverBudgetFlagsLargestUntilOverageCovered) { + GlobalMemoryLimiter lim; + lim.set_budget_bytes(1000 * kMiB); + std::atomic a {false}; // the largest + std::atomic b {false}; + std::atomic c {false}; + lim.register_buffer(&a, 900 * kMiB, 900 * kMiB); + lim.register_buffer(&b, 500 * kMiB, 500 * kMiB); + lim.register_buffer(&c, 100 * kMiB, 100 * kMiB); // total 1500 MiB > 1000 MiB + // Registration alone never flags: reacting belongs to the report path. + EXPECT_FALSE(a.load()); + EXPECT_FALSE(b.load()); + EXPECT_FALSE(c.load()); + + // Overage 500 MiB: the largest (a, 900 MiB of arena) alone covers it; b + // and c spared. + lim.report(&c, 100 * kMiB, 100 * kMiB); + EXPECT_TRUE(a.load()); + EXPECT_FALSE(b.load()); + EXPECT_FALSE(c.load()); + + // a's request is still pending -- its arena counts toward coverage, so a + // re-report while still over budget must NOT widen the victim set. + lim.report(&c, 100 * kMiB, 100 * kMiB); + EXPECT_FALSE(b.load()); + EXPECT_FALSE(c.load()); + + // Deeper overage (budget 300 MiB -> overage 1200 MiB): a (900) is no + // longer enough, b joins (900 + 500 >= 1200), c is still spared. + lim.set_budget_bytes(300 * kMiB); + lim.report(&c, 100 * kMiB, 100 * kMiB); + EXPECT_TRUE(a.load()); + EXPECT_TRUE(b.load()); + EXPECT_FALSE(c.load()); +} + +TEST(SniiGlobalMemoryLimiter, UnderBudgetOrDisabledNeverFlags) { + GlobalMemoryLimiter lim; + lim.set_budget_bytes(1000 * kMiB); + std::atomic a {false}; + std::atomic b {false}; + lim.register_buffer(&a, 600 * kMiB, 600 * kMiB); + lim.register_buffer(&b, 300 * kMiB, 300 * kMiB); + lim.report(&a, 700 * kMiB, 700 * kMiB); // total == budget: at (not over) budget + EXPECT_FALSE(a.load()); + EXPECT_FALSE(b.load()); + + lim.set_budget_bytes(0); // 0 = limiter off, no matter how large the total + lim.report(&a, 100000 * kMiB, 100000 * kMiB); + EXPECT_FALSE(a.load()); + EXPECT_FALSE(b.load()); +} + +// The field storm's root selection bug: victims were ranked by RESIDENT total, +// which is dominated by PERSISTENT (non-spillable) vocab/pair structures. +// Victims must be ranked by their reclaimable ARENA, and a buffer below the +// victim floor is not a victim at all. +TEST(SniiGlobalMemoryLimiter, VictimsSelectedByArenaNotResident) { + GlobalMemoryLimiter lim; + lim.set_budget_bytes(600 * kMiB); // 2 writers -> 300 MiB/writer: sane budget + std::atomic persistent_heavy {false}; + std::atomic arena_heavy {false}; + // Larger RESIDENT, tiny arena (8 MiB < the 64 MiB default floor). + lim.register_buffer(&persistent_heavy, 1000 * kMiB, 8 * kMiB); + // Smaller resident, large reclaimable arena. + lim.register_buffer(&arena_heavy, 500 * kMiB, 300 * kMiB); + + lim.report(&persistent_heavy, 1000 * kMiB, 8 * kMiB); // total 1500 > 600 + EXPECT_TRUE(arena_heavy.load()) << "the reclaimable-arena holder is the victim"; + EXPECT_FALSE(persistent_heavy.load()) + << "a below-floor arena must never be flagged, however large its resident total"; +} + +// PER-BUFFER COOLDOWN: after a victim's forced spill its arena is ~0; further +// over-budget reports must NOT re-flag it until the arena regrows past the +// floor. (No timer: eligibility IS the cooldown.) +TEST(SniiGlobalMemoryLimiter, CooldownSkipsJustSpilledBufferUntilArenaRegrows) { + GlobalMemoryLimiter lim; + lim.set_budget_bytes(200 * kMiB); // 1 writer -> 200 MiB/writer: sane budget + std::atomic flag {false}; + lim.register_buffer(&flag, 400 * kMiB, 200 * kMiB); + + lim.report(&flag, 400 * kMiB, 200 * kMiB); // over budget, arena >= floor + EXPECT_TRUE(flag.load()); + flag.store(false); // the owner honors: spill, clear the flag... + // ...and its next report shows the arena reclaimed but the PERSISTENT + // remainder still over budget (the field shape). + lim.report(&flag, 210 * kMiB, 0); + EXPECT_FALSE(flag.load()) << "cooldown: a spilled (empty-arena) buffer is exempt"; + lim.report(&flag, 240 * kMiB, 32 * kMiB); // regrown, still below the floor + EXPECT_FALSE(flag.load()) << "still exempt below the floor"; + lim.report(&flag, 280 * kMiB, 72 * kMiB); // regrown PAST the 64 MiB floor + EXPECT_TRUE(flag.load()) << "eligible again once a floor's worth is reclaimable"; +} + +// BUDGET SANITY: when budget / registered_count < the per-writer useful +// minimum, spilling cannot meet the budget (persistent structures dominate) -- +// the limiter must degrade (log once, flag nothing) and recover when the +// ratio does. +TEST(SniiGlobalMemoryLimiter, BudgetDegradationStopsFlaggingAndRecovers) { + GlobalMemoryLimiter lim; + // 3 writers on a 240 MiB budget: 80 MiB/writer < the 96 MiB minimum. + lim.set_budget_bytes(240 * kMiB); + std::atomic a {false}; + std::atomic b {false}; + std::atomic c {false}; + lim.register_buffer(&a, 200 * kMiB, 150 * kMiB); + lim.register_buffer(&b, 200 * kMiB, 150 * kMiB); + lim.register_buffer(&c, 200 * kMiB, 150 * kMiB); + EXPECT_FALSE(lim.budget_degraded()); + + lim.report(&a, 200 * kMiB, 150 * kMiB); // total 600 > 240: over budget... + EXPECT_TRUE(lim.budget_degraded()) << "80 MiB/writer < 96 MiB: degrade"; + EXPECT_FALSE(a.load()); + EXPECT_FALSE(b.load()); + EXPECT_FALSE(c.load()); + lim.report(&b, 200 * kMiB, 150 * kMiB); // sustained episode: still no flags + EXPECT_FALSE(a.load()); + EXPECT_FALSE(b.load()); + EXPECT_FALSE(c.load()); + + // One writer drains: 240 MiB / 2 = 120 MiB/writer >= 96 MiB -- recovered. + lim.unregister_buffer(&c); + lim.report(&a, 200 * kMiB, 150 * kMiB); // total 400 > 240, sane ratio + EXPECT_FALSE(lim.budget_degraded()); + EXPECT_TRUE(a.load() || b.load()) << "flagging must resume after recovery"; +} + +// ---- (3) the owner honors a pending request --------------------------------- + +TEST(SniiSpimiGlobalSpill, OwnerHonorsRequestAtNextTokenAndClears) { + snii_testing::reset_global_forced_spills(); + // Unlimited local threshold: the per-writer gate can never fire, so any + // spill below is attributable to the global request alone. The G08 + // anti-churn floor (arena >= cap/4) is bypassed by construction here -- + // the arena holds a single 32 KiB block, far below any production cap/4. + // The forced-spill floor is dropped to its one-block minimum: THIS test + // pins the honor mechanics, not the floor (see the floor tests below). + SpimiTermBuffer buf(/*has_positions=*/true, /*spill_threshold_bytes=*/0); + buf.set_forced_spill_min_arena_bytes(1); + for (uint32_t d = 0; d < 200; ++d) { + buf.add_token("hot", /*docid=*/d, /*pos=*/0); + } + ASSERT_TRUE(buf.status().ok()); + ASSERT_EQ(buf.run_count_for_test(), 0U); + + buf.request_global_spill_for_test(); // what the limiter does cross-thread + EXPECT_TRUE(buf.global_spill_requested_for_test()); + EXPECT_EQ(snii_testing::global_forced_spills(), 0U); + + buf.add_token("hot", /*docid=*/200, /*pos=*/0); // next token observes it + EXPECT_EQ(buf.run_count_for_test(), 1U) << "forced spill must cut a run"; + EXPECT_FALSE(buf.global_spill_requested_for_test()) << "honor must clear"; + EXPECT_EQ(snii_testing::global_forced_spills(), 1U); + + // Once cleared, later tokens spill nothing further (advisory, one-shot). + buf.add_token("hot", /*docid=*/201, /*pos=*/0); + EXPECT_EQ(buf.run_count_for_test(), 1U); + + // The forced spill changes WHEN a run was cut, never WHAT is emitted. + std::vector out = buf.finalize_sorted(); + ASSERT_TRUE(buf.status().ok()) << buf.status().to_string(); + ASSERT_EQ(out.size(), 1U); + EXPECT_EQ(out[0].term, "hot"); + ASSERT_EQ(out[0].docids.size(), 202U); + EXPECT_EQ(out[0].docids.front(), 0U); + EXPECT_EQ(out[0].docids.back(), 201U); +} + +TEST(SniiSpimiGlobalSpill, RequestOnEmptyArenaIsPendingUntilARunIsWritable) { + snii_testing::reset_global_forced_spills(); + SpimiTermBuffer buf(/*has_positions=*/true, /*spill_threshold_bytes=*/0); + buf.set_forced_spill_min_arena_bytes(1); // one-block minimum still applies + // A request on an empty buffer has nothing to write: purely advisory, no + // run, flag stays pending (a drained owner would simply never observe it). + buf.request_global_spill_for_test(); + EXPECT_EQ(buf.run_count_for_test(), 0U); + EXPECT_TRUE(buf.global_spill_requested_for_test()); + // The first token claims the first 32 KiB arena block -- the "arena >= one + // block" floor is now met, so the pending request is honored right there. + buf.add_token("t", /*docid=*/0, /*pos=*/0); + EXPECT_EQ(buf.run_count_for_test(), 1U); + EXPECT_FALSE(buf.global_spill_requested_for_test()); + EXPECT_EQ(snii_testing::global_forced_spills(), 1U); + ASSERT_TRUE(buf.status().ok()); +} + +// ---- (6) forced-spill floor: below-floor requests are pending no-ops -------- + +// The field storm's honor-side bug: a request was honored with a single 32 KiB +// arena block, cutting a near-empty run. With the (default 64 MiB) floor, a +// request planted while the arena is small must spill NOTHING -- not per +// token, not once -- and stay pending. +TEST(SniiSpimiGlobalSpill, RequestBelowForcedSpillFloorIsNotHonored) { + snii_testing::reset_global_forced_spills(); + SpimiTermBuffer buf(/*has_positions=*/true, /*spill_threshold_bytes=*/0); + // Production-default floor (64 MiB) -- far above this feed's arena. + ASSERT_EQ(buf.forced_spill_min_arena_bytes(), + SpimiTermBuffer::kDefaultForcedSpillMinArenaBytes); + buf.request_global_spill_for_test(); + for (uint32_t d = 0; d < 5000; ++d) { // a few arena blocks, << 64 MiB + buf.add_token(unigram(d % 300), /*docid=*/d, /*pos=*/0); + } + ASSERT_TRUE(buf.status().ok()); + EXPECT_EQ(buf.run_count_for_test(), 0U) << "below-floor request must be a no-op"; + EXPECT_EQ(snii_testing::global_forced_spills(), 0U); + EXPECT_TRUE(buf.global_spill_requested_for_test()) << "...that stays pending"; +} + +// Deferred honor: with a small floor (3 arena blocks), a pending request is a +// no-op until the arena regrows past the floor, then fires exactly once. +TEST(SniiSpimiGlobalSpill, RequestHonoredOnceArenaRegrowsPastFloor) { + snii_testing::reset_global_forced_spills(); + SpimiTermBuffer buf(/*has_positions=*/true, /*spill_threshold_bytes=*/0); + const uint64_t kFloor = 3ULL * CompactPostingPool::kBlockSize; + buf.set_forced_spill_min_arena_bytes(kFloor); + buf.request_global_spill_for_test(); + uint32_t docid = 0; + // One block is not enough: feed until the arena holds one block and check + // the request is still pending, then keep feeding until the floor is met. + while (buf.status().ok() && buf.run_count_for_test() == 0 && docid < 200000) { + buf.add_token("hot", docid, /*pos=*/docid % 7); + ++docid; + } + ASSERT_TRUE(buf.status().ok()); + ASSERT_EQ(buf.run_count_for_test(), 1U) << "must fire once the floor is met"; + EXPECT_FALSE(buf.global_spill_requested_for_test()); + EXPECT_EQ(snii_testing::global_forced_spills(), 1U); + // The run was cut with at least a floor's worth of arena accumulated: the + // feed needed more than 2 blocks' worth of tokens (each token appends + // >= 2 bytes, so 2 blocks < 64 KiB of payload < the token count here). + EXPECT_GT(docid, (2ULL * CompactPostingPool::kBlockSize) / 4) + << "honor must not fire before the floor's worth of arena existed"; +} + +// ---- (6) the storm scenario end-to-end -------------------------------------- + +// The conc=16 wikipedia field failure in miniature, with PRODUCTION-DEFAULT +// anti-storm settings: a budget far below the writers' (persistent-dominated) +// resident sum and far below (96 MiB x writers). The old limiter flagged every +// buffer on every report and each honored with one 32 KiB block -- a storm of +// tiny runs. Now: the budget-sanity check degrades (log once, stop flagging), +// the victim floor exempts every small-arena buffer anyway, so ZERO forced +// spills and ZERO runs result. +TEST(SniiSpimiGlobalSpill, StormScenarioTinyBudgetSmallArenasProducesZeroForcedSpills) { + snii_testing::reset_global_forced_spills(); + GlobalMemoryLimiter lim; // production defaults: 64 MiB floor, 96 MiB/writer sanity + lim.set_budget_bytes(256 * 1024); + + constexpr size_t kBuffers = 6; + std::vector> buffers; + buffers.reserve(kBuffers); + for (size_t i = 0; i < kBuffers; ++i) { + auto buf = std::make_unique(/*has_positions=*/true, + /*spill_threshold_bytes=*/0); + buf->attach_global_limiter(&lim); + buffers.push_back(std::move(buf)); + } + // Interleaved feed: every buffer's resident grows well past its budget + // share (the registry total exceeds 1 MiB almost immediately), reports + // fire constantly, yet nothing may be flagged or spilled. + for (uint32_t round = 0; round < 400; ++round) { + for (auto& buf : buffers) { + for (uint32_t k = 0; k < 8; ++k) { + buf->add_token(unigram((round * 8 + k) % 2000), /*docid=*/round * 8 + k, + /*pos=*/0); + } + } + } + ASSERT_GT(lim.total_bytes(), lim.budget_bytes()) << "the scenario must be over budget"; + EXPECT_TRUE(lim.budget_degraded()) << "256 KiB / 6 writers is an unmeetable budget"; + EXPECT_EQ(snii_testing::global_forced_spills(), 0U) << "no forced-spill storm"; + for (auto& buf : buffers) { + ASSERT_TRUE(buf->status().ok()); + EXPECT_EQ(buf->run_count_for_test(), 0U) << "no runs were cut"; + EXPECT_FALSE(buf->global_spill_requested_for_test()) << "no flags were planted"; + } +} + +// ---- (4) attach / report / detach lifetime ---------------------------------- + +TEST(SniiSpimiGlobalSpill, AttachRegistersReportsTrackResidentAndDtorUnregisters) { + GlobalMemoryLimiter lim; // budget 0: tracking only, no flags + { + SpimiTermBuffer buf(/*has_positions=*/true, /*spill_threshold_bytes=*/0); + buf.attach_global_limiter(&lim); + EXPECT_EQ(lim.registered_count(), 1U); + EXPECT_EQ(lim.total_bytes(), static_cast(buf.resident_bytes_for_test())); + + for (uint32_t i = 0; i < 300; ++i) { + buf.add_token(unigram(i), /*docid=*/i, /*pos=*/0); + } + ASSERT_TRUE(buf.status().ok()); + // The debounced report path forwards ABSOLUTE totals: at rest the + // registry equals the buffer's real resident bytes exactly. + EXPECT_EQ(lim.total_bytes(), static_cast(buf.resident_bytes_for_test())); + + buf.attach_global_limiter(&lim); // at-most-once: ignored + EXPECT_EQ(lim.registered_count(), 1U); + } + // Destruction un-registers: nothing leaks into the process-wide total. + EXPECT_EQ(lim.registered_count(), 0U); + EXPECT_EQ(lim.total_bytes(), 0); +} + +// End-to-end: two attached buffers, one small and one that grows past the +// budget. The limiter must flag the larger-ARENA grower once its arena clears +// the victim floor (the small buffer's single arena block never does); the +// grower's own next token honors the request (its local threshold is +// unlimited, the G08 floor bypassed); the small buffer is never flagged and +// never spills. Degradation is disabled: the KiB-scale budget is intentional. +TEST(SniiSpimiGlobalSpill, LimiterFlagsLargestOwnerSpillsSmallBufferSpared) { + snii_testing::reset_global_forced_spills(); + GlobalMemoryLimiter lim; // declared BEFORE the buffers: outlives them + lim.set_budget_bytes(256 * 1024); + const int64_t kFloor = 2LL * CompactPostingPool::kBlockSize; // 64 KiB + lim.set_min_victim_arena_bytes(kFloor); + lim.set_min_useful_budget_per_writer_bytes(0); // KiB-scale test budget + + SpimiTermBuffer small(/*has_positions=*/true, /*spill_threshold_bytes=*/0); + small.set_forced_spill_min_arena_bytes(static_cast(kFloor)); + small.attach_global_limiter(&lim); + SpimiTermBuffer big(/*has_positions=*/true, /*spill_threshold_bytes=*/0); + big.set_forced_spill_min_arena_bytes(static_cast(kFloor)); + big.attach_global_limiter(&lim); + + for (uint32_t i = 0; i < 50; ++i) { // ~1 arena block: forever below the floor + small.add_token(unigram(i), /*docid=*/i, /*pos=*/0); + } + ASSERT_TRUE(small.status().ok()); + + // Distinct terms grow big's resident (vocab + intern + slots + arena) past + // the budget and its ARENA past the floor; the over-budget report then + // flags big (the largest eligible arena), and big's own add path honors + // the request. Bounded feed with an early exit. The bound stays below + // unigram()'s 17576 distinct strings so every fed term is DISTINCT (the + // drain-count assertion relies on that); three tokens per term grow the + // arena ~3x faster than the vocab, so the floor is met well within it. + uint32_t fed = 0; + uint32_t docid = 0; + for (uint32_t k = 0; k < 16000 && big.run_count_for_test() == 0; ++k, ++fed) { + big.add_token(unigram(k), docid++, /*pos=*/0); + big.add_token(unigram(k), docid++, /*pos=*/1); + big.add_token(unigram(k), docid++, /*pos=*/2); + } + ASSERT_TRUE(big.status().ok()); + ASSERT_GT(big.run_count_for_test(), 0U) << "grower must be forced to spill"; + EXPECT_FALSE(big.global_spill_requested_for_test()) << "honor must clear"; + EXPECT_GE(snii_testing::global_forced_spills(), 1U); + EXPECT_EQ(small.run_count_for_test(), 0U) << "small buffer must be spared"; + EXPECT_FALSE(small.global_spill_requested_for_test()); + + // The forced spill released the grower's arena and reported the drop: the + // registry total is back to the exact resident sum of both buffers. + EXPECT_EQ(lim.total_bytes(), static_cast(small.resident_bytes_for_test()) + + static_cast(big.resident_bytes_for_test())); + + // Both buffers still drain cleanly (the small one in memory, the big one + // through its forced run + k-way merge). + size_t small_terms = 0; + ASSERT_TRUE(small.for_each_term_sorted([&small_terms](TermPostings&&) { ++small_terms; }).ok()); + EXPECT_EQ(small_terms, 50U); + size_t big_terms = 0; + ASSERT_TRUE(big.for_each_term_sorted([&big_terms](TermPostings&&) { ++big_terms; }).ok()); + EXPECT_EQ(big_terms, fed); +} + +// ---- (5) thread-safety canary ------------------------------------------------ + +// Concurrent register / report / unregister with a tiny budget so cross-thread +// flagging constantly targets flags that die right after their unregister -- +// the exact lifetime the mutex must protect. Run under TSAN this is the G09 +// race canary; under ASAN it still catches any touch-after-unregister. The +// floor and the sanity minimum are dropped so the byte-scale entries keep +// producing flags (the point is flag traffic, not selection policy). +TEST(SniiGlobalMemoryLimiter, ConcurrentRegisterReportUnregisterIsClean) { + GlobalMemoryLimiter lim; + lim.set_budget_bytes(1); // everything is over budget: flags fly + lim.set_min_victim_arena_bytes(1); + lim.set_min_useful_budget_per_writer_bytes(0); + constexpr int kThreads = 8; + constexpr int kIters = 400; + std::vector threads; + threads.reserve(kThreads); + for (int t = 0; t < kThreads; ++t) { + threads.emplace_back([&lim, t] { + for (int i = 0; i < kIters; ++i) { + std::atomic flag {false}; + lim.register_buffer(&flag, 1000 + t, 1000 + t); + for (int r = 0; r < 4; ++r) { + lim.report(&flag, 1000 + t + r, 1000 + t + r); + if (flag.load(std::memory_order_relaxed)) { + // The owner honoring: observe and clear on its thread. + flag.store(false, std::memory_order_relaxed); + } + } + lim.unregister_buffer(&flag); + // `flag` is destroyed here -- after unregister_buffer returned, + // the limiter must never touch it again. + } + }); + } + for (auto& th : threads) { + th.join(); + } + EXPECT_EQ(lim.total_bytes(), 0); + EXPECT_EQ(lim.registered_count(), 0U); +} + +} // namespace diff --git a/be/test/storage/index/snii/writer/spimi_locality_bench_test.cpp b/be/test/storage/index/snii/writer/spimi_locality_bench_test.cpp new file mode 100644 index 00000000000000..75210b21ca8312 --- /dev/null +++ b/be/test/storage/index/snii/writer/spimi_locality_bench_test.cpp @@ -0,0 +1,470 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/writer/memory_reporter.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +// G11 in-process locality microbenchmark for the SPIMI per-token hot path. +// +// WHY IN-PROCESS: the wikipedia import's wall time swings 2080s..7100s with the +// host's memory-bandwidth contention, so whole-import A/B numbers are +// meaningless for CPU-locality candidates. This bench feeds a synthetic +// wikipedia-shaped token stream (Zipf unigrams + adjacent-pair bigrams, +// production add order: all unigrams of a doc, then its pairs) straight through +// add_token_returning_id / add_bigram_token(left_id, right_id) in ONE process +// and reports steady_clock wall time per rep. Relative (before/after) deltas +// from interleaved reps + medians are contention-independent enough to gate a +// keep/drop decision; absolute numbers are not the point. +// +// EQUALITY EVIDENCE: every timed rep is followed (rep 0 of each arm) by a full +// for_each_term_sorted drain folded into an FNV-1a digest over every emitted +// term's bytes, docids, freqs and positions. A locality candidate must keep the +// digest bit-identical to the baseline build's. +// +// Scale knobs (env, so the default CI run stays cheap): +// SNII_BENCH_TOKENS total tokens to feed (default 300000; perf runs use 5M) +// SNII_BENCH_REPS timed reps per arm (default 3) + +namespace doris::snii::writer { + +namespace { + +uint64_t env_u64(const char* name, uint64_t def) { + const char* v = std::getenv(name); + if (v == nullptr || *v == '\0') { + return def; + } + return static_cast(std::strtoull(v, nullptr, 10)); +} + +constexpr uint64_t kFnvOffset = 1469598103934665603ULL; +constexpr uint64_t kFnvPrime = 1099511628211ULL; + +uint64_t fnv_bytes(uint64_t h, const void* data, size_t n) { + const auto* p = static_cast(data); + for (size_t i = 0; i < n; ++i) { + h ^= p[i]; + h *= kFnvPrime; + } + return h; +} + +uint64_t fnv_u64(uint64_t h, uint64_t v) { + return fnv_bytes(h, &v, sizeof(v)); +} + +// Wikipedia-shaped synthetic corpus: Zipf(s=1.07) over a fixed vocabulary of +// lowercase-ascii words (2..12 chars, all phrase-bigram-indexable so every +// adjacent pair reaches the pair-keyed bigram path, like body text), cut into +// fixed-size docs. Deterministic (fixed seed). Built once, untimed. +struct BenchCorpus { + std::vector vocab; // distinct words + std::vector tokens; // per-token vocab index + uint32_t tokens_per_doc = 200; + uint64_t total_tokens() const { return tokens.size(); } +}; + +const BenchCorpus& corpus() { + static const BenchCorpus c = [] { + BenchCorpus b; + const uint64_t n_tokens = env_u64("SNII_BENCH_TOKENS", 300000); + const uint32_t vocab_size = 262144; + std::mt19937_64 rng(0x5a11d5eedULL); + + // Distinct words, GUARANTEED unique by a prefix-free construction: the + // vocab index in base 25 over 'b'..'z', terminated by 'a', then 0..6 + // random 'b'..'z' chars (the terminator is the FIRST 'a', so equal + // strings imply equal indexes). Lengths land in 2..12 (SSO, + // bigram-indexable), Zipf-weighted below like body text. + b.vocab.reserve(vocab_size); + std::uniform_int_distribution extra_len(0, 6); + std::uniform_int_distribution letter(0, 24); + for (uint32_t i = 0; i < vocab_size; ++i) { + std::string w; + uint32_t x = i; + do { + w.push_back(static_cast('b' + (x % 25))); + x /= 25; + } while (x != 0); + w.push_back('a'); + const int extra = extra_len(rng); + for (int k = 0; k < extra; ++k) { + w.push_back(static_cast('b' + letter(rng))); + } + b.vocab.push_back(std::move(w)); + } + + // Zipf CDF over ranks 1..V, s = 1.07; token index sampled by binary + // search. Rank r maps to vocab id (r - 1) (vocab order is arbitrary). + std::vector cdf(vocab_size); + double acc = 0.0; + for (uint32_t r = 1; r <= vocab_size; ++r) { + acc += 1.0 / std::pow(static_cast(r), 1.07); + cdf[r - 1] = acc; + } + std::uniform_real_distribution uni(0.0, acc); + b.tokens.reserve(n_tokens); + for (uint64_t t = 0; t < n_tokens; ++t) { + const double u = uni(rng); + const auto it = std::lower_bound(cdf.begin(), cdf.end(), u); + b.tokens.push_back(static_cast(it - cdf.begin())); + } + return b; + }(); + return c; +} + +struct FeedResult { + uint64_t total_ns = 0; // whole feed loop (unigrams + bigrams) + uint64_t unigram_ns = 0; // add_token_returning_id portion + uint64_t bigram_ns = 0; // add_bigram_token(left_id, right_id) portion + uint64_t unique_terms = 0; + uint64_t total_tokens = 0; + uint64_t digest = 0; // 0 unless drained +}; + +// Feeds the corpus through a fresh buffer exactly the way the production SNII +// column writer does (owned vocab, positions on, bigram diet configured with +// the default 512 MiB cap and the 0-doc-floor drain df gate; per doc all +// unigrams first, then all adjacent pairs keyed by the captured unigram ids; +// pair position = left token's position). `drain_digest`: run the terminal +// for_each_term_sorted drain and fold everything emitted into an FNV digest +// (equality evidence); reps used purely for timing skip it (dtor cleans up). +FeedResult run_feed(bool drain_digest) { + const BenchCorpus& c = corpus(); + FeedResult r; + + MemoryReporter reporter(nullptr, /*cap_bytes=*/0); // production per-token + // report path, no spill + SpimiTermBuffer buf(/*has_positions=*/true, /*spill_threshold_bytes=*/0, &reporter); + buf.configure_bigram_diet(/*vocab_cap_bytes=*/512ULL << 20, + format::default_phrase_bigram_prune_min_df(0)); + + std::vector ids(c.tokens_per_doc); + const uint64_t n = c.total_tokens(); + const auto t_begin = std::chrono::steady_clock::now(); + uint32_t docid = 0; + for (uint64_t t = 0; t < n;) { + const uint64_t doc_end = std::min(n, t + c.tokens_per_doc); + const auto t_uni = std::chrono::steady_clock::now(); + uint32_t m = 0; + for (; t < doc_end; ++t, ++m) { + const std::string& w = c.vocab[c.tokens[t]]; + ids[m] = buf.add_token_returning_id(std::string_view(w), docid, m); + } + const auto t_big = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i + 1 < m; ++i) { + buf.add_bigram_token(ids[i], ids[i + 1], docid, i); + } + const auto t_end = std::chrono::steady_clock::now(); + r.unigram_ns += std::chrono::duration_cast(t_big - t_uni).count(); + r.bigram_ns += std::chrono::duration_cast(t_end - t_big).count(); + ++docid; + } + const auto t_total = std::chrono::steady_clock::now(); + r.total_ns = std::chrono::duration_cast(t_total - t_begin).count(); + r.unique_terms = buf.unique_terms(); + r.total_tokens = buf.total_tokens(); + EXPECT_TRUE(buf.status().ok()) << buf.status().to_string(); + + if (drain_digest) { + uint64_t h = kFnvOffset; + std::vector pos_scratch(4096); + const Status st = buf.for_each_term_sorted([&](TermPostings&& tp) { + h = fnv_bytes(h, tp.term.data(), tp.term.size()); + h = fnv_u64(h, tp.docids.size()); + h = fnv_bytes(h, tp.docids.data(), tp.docids.size() * sizeof(uint32_t)); + h = fnv_bytes(h, tp.freqs.data(), tp.freqs.size() * sizeof(uint32_t)); + if (tp.pos_pump) { + uint64_t remaining = tp.pos_total; + while (remaining > 0) { + const size_t k = + static_cast(std::min(remaining, pos_scratch.size())); + tp.pos_pump(pos_scratch.data(), k); + h = fnv_bytes(h, pos_scratch.data(), k * sizeof(uint32_t)); + remaining -= k; + } + } else { + h = fnv_bytes(h, tp.positions_flat.data(), + tp.positions_flat.size() * sizeof(uint32_t)); + } + }); + EXPECT_TRUE(st.ok()) << st.to_string(); + r.digest = h; + } + return r; +} + +uint64_t median_ns(std::vector v) { + std::sort(v.begin(), v.end()); + return v[v.size() / 2]; +} + +void print_result(const char* tag, size_t rep, const FeedResult& r) { + const double tokens = static_cast(2 * r.total_tokens); // uni + bigram adds + printf("[bench] %-16s rep=%zu total=%9.3f ms unigram=%9.3f ms bigram=%9.3f ms " + "(%.1f ns/add)\n", + tag, rep, r.total_ns / 1e6, r.unigram_ns / 1e6, r.bigram_ns / 1e6, r.total_ns / tokens); +} + +} // namespace + +// Baseline feed timing + drain-digest stability. The digest printed here is the +// cross-build equality reference every candidate build must reproduce. +TEST(SniiSpimiLocalityBenchTest, FeedBaseline) { + const size_t reps = static_cast(env_u64("SNII_BENCH_REPS", 3)); + const BenchCorpus& c = corpus(); + printf("[bench] corpus: %llu tokens, vocab %zu, %u tokens/doc\n", + static_cast(c.total_tokens()), c.vocab.size(), c.tokens_per_doc); + + std::vector total, uni, big; + uint64_t digest = 0; + uint64_t unique_terms = 0; + for (size_t rep = 0; rep < reps; ++rep) { + const FeedResult r = run_feed(/*drain_digest=*/rep == 0); + if (rep == 0) { + digest = r.digest; + unique_terms = r.unique_terms; + } else { + ASSERT_EQ(unique_terms, r.unique_terms); // determinism across reps + } + total.push_back(r.total_ns); + uni.push_back(r.unigram_ns); + big.push_back(r.bigram_ns); + print_result("baseline", rep, r); + } + printf("[bench] baseline median: total=%9.3f ms unigram=%9.3f ms bigram=%9.3f ms\n", + median_ns(total) / 1e6, median_ns(uni) / 1e6, median_ns(big) / 1e6); + printf("[bench] unique_terms=%llu drain_digest=%016llx\n", + static_cast(unique_terms), static_cast(digest)); + ASSERT_NE(digest, 0U); +} + +// In-process A/B for the G11 prefetch candidate: alternates prefetch-disabled / +// prefetch-enabled reps in ONE process and compares medians, so machine drift +// cancels. Compiled to a skip until the candidate (and its BE_TEST toggle seam) +// is present. +TEST(SniiSpimiLocalityBenchTest, PrefetchToggleAB) { +#ifndef SNII_G11_PREFETCH + GTEST_SKIP() << "G11 prefetch candidate not compiled in"; +#else + const size_t reps = static_cast(env_u64("SNII_BENCH_REPS", 3)); + std::vector off_total, on_total, off_big, on_big; + uint64_t digest_off = 0; + uint64_t digest_on = 0; + for (size_t rep = 0; rep < reps; ++rep) { + testing::set_bench_disable_g11_prefetch(true); + const FeedResult off = run_feed(/*drain_digest=*/rep == 0); + testing::set_bench_disable_g11_prefetch(false); + const FeedResult on = run_feed(/*drain_digest=*/rep == 0); + if (rep == 0) { + digest_off = off.digest; + digest_on = on.digest; + } + off_total.push_back(off.total_ns); + on_total.push_back(on.total_ns); + off_big.push_back(off.bigram_ns); + on_big.push_back(on.bigram_ns); + print_result("prefetch-OFF", rep, off); + print_result("prefetch-ON", rep, on); + } + testing::set_bench_disable_g11_prefetch(false); + const uint64_t mo = median_ns(off_total); + const uint64_t mn = median_ns(on_total); + const uint64_t bo = median_ns(off_big); + const uint64_t bn = median_ns(on_big); + printf("[bench] prefetch A/B: total OFF=%9.3f ms ON=%9.3f ms (%+.2f%%) " + "bigram OFF=%9.3f ms ON=%9.3f ms (%+.2f%%)\n", + mo / 1e6, mn / 1e6, 100.0 * (static_cast(mn) - mo) / mo, bo / 1e6, bn / 1e6, + 100.0 * (static_cast(bn) - bo) / bo); + // Byte-identical: prefetch has no architectural side effects. + ASSERT_EQ(digest_off, digest_on); +#endif +} + +// Candidate (d) upper bound, measured WITHOUT touching production code: replays +// the exact pair-key stream (find-or-emplace, feed order, dups included) into a +// raw phmap::flat_hash_map -- the bigram_pair_map_ type -- +// with and without a perfect up-front reserve. The delta bounds what any +// doc-count-heuristic reserve of the pair map could save the whole feed. +TEST(SniiSpimiLocalityBenchTest, PairMapReserveUpperBound) { + const size_t reps = static_cast(env_u64("SNII_BENCH_REPS", 3)); + const BenchCorpus& c = corpus(); + + // Pair-key stream in feed order (unigram ids == first-seen order, exactly + // what add_token_returning_id assigns; kNoPairKey never collides). + std::vector first_seen(c.vocab.size(), 0xFFFFFFFFU); + uint32_t next_id = 0; + std::vector pair_stream; + pair_stream.reserve(c.tokens.size()); + const uint64_t n = c.total_tokens(); + for (uint64_t t = 0; t < n;) { + const uint64_t doc_end = std::min(n, t + c.tokens_per_doc); + uint32_t prev_id = 0; + bool have_prev = false; + for (; t < doc_end; ++t) { + uint32_t& slot = first_seen[c.tokens[t]]; + if (slot == 0xFFFFFFFFU) { + slot = next_id++; + } + if (have_prev) { + pair_stream.push_back((static_cast(prev_id) << 32) | slot); + } + prev_id = slot; + have_prev = true; + } + } + + size_t distinct = 0; + { + phmap::flat_hash_map probe; + for (uint64_t k : pair_stream) { + probe.emplace(k, 0); + } + distinct = probe.size(); + } + + auto run_arm = [&](bool reserve) { + const auto t0 = std::chrono::steady_clock::now(); + phmap::flat_hash_map m; + if (reserve) { + m.reserve(distinct); + } + uint32_t id = 0; + uint64_t hits = 0; + for (uint64_t k : pair_stream) { + auto it = m.find(k); + if (it == m.end()) { + m.emplace(k, id++); + } else { + ++hits; + } + } + const auto t1 = std::chrono::steady_clock::now(); + EXPECT_EQ(m.size(), distinct); + (void)hits; + return static_cast( + std::chrono::duration_cast(t1 - t0).count()); + }; + + std::vector no_res, with_res; + for (size_t rep = 0; rep < reps; ++rep) { + const uint64_t t_no = run_arm(false); + const uint64_t t_yes = run_arm(true); + no_res.push_back(t_no); + with_res.push_back(t_yes); + printf("[bench] pair-map rep=%zu no-reserve=%9.3f ms reserve=%9.3f ms\n", rep, t_no / 1e6, + t_yes / 1e6); + } + const uint64_t mn = median_ns(no_res); + const uint64_t my = median_ns(with_res); + printf("[bench] pair-map reserve upper bound: distinct=%zu stream=%zu " + "no-reserve=%9.3f ms reserve=%9.3f ms delta=%+.2f%%\n", + distinct, pair_stream.size(), mn / 1e6, my / 1e6, + 100.0 * (static_cast(my) - mn) / mn); +} + +// Same experiment for the unigram intern set: a replica std::unordered_set +// keyed by u32 ids with vocab-backed heterogeneous string hashing (the +// production intern_ shape), fed the unigram token stream, with and without a +// perfect up-front reserve. +TEST(SniiSpimiLocalityBenchTest, InternSetReserveUpperBound) { + const size_t reps = static_cast(env_u64("SNII_BENCH_REPS", 3)); + const BenchCorpus& c = corpus(); + + struct H { + using is_transparent = void; + const std::vector* vocab; + size_t operator()(std::string_view s) const noexcept { + return fnv_bytes(kFnvOffset, s.data(), s.size()); + } + size_t operator()(uint32_t id) const noexcept { + return operator()(std::string_view((*vocab)[id])); + } + }; + struct E { + using is_transparent = void; + const std::vector* vocab; + bool operator()(uint32_t a, uint32_t b) const noexcept { return a == b; } + [[maybe_unused]] bool operator()(uint32_t a, std::string_view s) const noexcept { + return std::string_view((*vocab)[a]) == s; + } + [[maybe_unused]] bool operator()(std::string_view s, uint32_t a) const noexcept { + return std::string_view((*vocab)[a]) == s; + } + }; + + size_t distinct = 0; + { + std::unordered_set probe; + for (uint32_t t : c.tokens) { + probe.insert(t); + } + distinct = probe.size(); + } + + auto run_arm = [&](bool reserve) { + const auto t0 = std::chrono::steady_clock::now(); + std::unordered_set s(0, H {&c.vocab}, E {&c.vocab}); + if (reserve) { + s.reserve(distinct); + } + for (uint32_t t : c.tokens) { + const std::string& w = c.vocab[t]; + auto it = s.find(std::string_view(w)); + if (it == s.end()) { + s.insert(t); // id == vocab index in this replica + } + } + const auto t1 = std::chrono::steady_clock::now(); + EXPECT_EQ(s.size(), distinct); + return static_cast( + std::chrono::duration_cast(t1 - t0).count()); + }; + + std::vector no_res, with_res; + for (size_t rep = 0; rep < reps; ++rep) { + const uint64_t t_no = run_arm(false); + const uint64_t t_yes = run_arm(true); + no_res.push_back(t_no); + with_res.push_back(t_yes); + printf("[bench] intern-set rep=%zu no-reserve=%9.3f ms reserve=%9.3f ms\n", rep, + t_no / 1e6, t_yes / 1e6); + } + const uint64_t mn = median_ns(no_res); + const uint64_t my = median_ns(with_res); + printf("[bench] intern-set reserve upper bound: distinct=%zu no-reserve=%9.3f ms " + "reserve=%9.3f ms delta=%+.2f%%\n", + distinct, mn / 1e6, my / 1e6, 100.0 * (static_cast(my) - mn) / mn); +} + +} // namespace doris::snii::writer diff --git a/be/test/storage/index/snii/writer/spimi_resident_accounting_test.cpp b/be/test/storage/index/snii/writer/spimi_resident_accounting_test.cpp new file mode 100644 index 00000000000000..21d4acbf88e36f --- /dev/null +++ b/be/test/storage/index/snii/writer/spimi_resident_accounting_test.cpp @@ -0,0 +1,223 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/writer/memory_reporter.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +// G08: gate-2 resident accounting. Before G08, SpimiTermBuffer::resident_bytes() +// summed ONLY the posting arena + the vocab-sized slot index, so the entire G05 +// pair-key machinery (bigram_pair_map_, pair_of_), the owned vocabulary / +// intern set, the Term slot pool and the rank/bookkeeping arrays were INVISIBLE +// to both the gate-2 spill trigger and the MemoryReporter -- on wikipedia each +// of the 16 writers held those ~750 MiB ON TOP of the 512 MiB cap the gate +// believed it was enforcing (~20 GiB total). These tests pin: +// (1) COVERAGE + MONOTONICITY: after N unique pair interns, resident_bytes() +// is at least the externally-measured pair-map + reverse-slot footprint +// (the pre-G08 sum sits BELOW that floor at this shape, so the assertion +// fails without the G08 charges), and it never decreases while feeding +// with eviction off and no spills; +// (2) GATE TRIGGER ON THE BIGRAM PATH: a tiny spill threshold is crossed by +// the newly-charged pair structures alone (the arena stays inside its +// first 32 KiB block), and the spill fires from add_bigram_token -- +// run_count_for_test() > 0 where the pre-G08 accounting never spilled; +// (3) EQUALITY VS NO-SPILL CONTROL: the structure-triggered spill timing +// changes nothing about the emitted stream (terms, order, postings); +// (4) REPORTER NET-ZERO (credit/debit symmetry): every byte the intern / +// materialize paths credit is debited by eviction + the terminal drains, +// on both the in-memory and the spilled (+ diet eviction) paths. +using doris::snii::writer::MemoryReporter; +using doris::snii::writer::SpimiTermBuffer; +using doris::snii::writer::TermPostings; + +namespace { + +// Distinct short unigrams ("uaa", "uab", ...): SSO-sized, alpha-only, never +// bigram-marker-prefixed. +std::string unigram(uint32_t i) { + std::string s = "u"; + s += static_cast('a' + (i / 26) % 26); + s += static_cast('a' + i % 26); + s += static_cast('a' + (i / 676) % 26); + return s; +} + +// Interns `n` distinct unigrams at docid 0 (position i) and returns their ids. +std::vector intern_unigrams(SpimiTermBuffer* buf, uint32_t n) { + std::vector ids; + ids.reserve(n); + for (uint32_t i = 0; i < n; ++i) { + const uint32_t id = buf->add_token_returning_id(unigram(i), /*docid=*/0, /*pos=*/i); + EXPECT_NE(id, SpimiTermBuffer::kInvalidTermId); + ids.push_back(id); + } + return ids; +} + +void expect_same_stream(const std::vector& got, const std::vector& want, + const char* label) { + ASSERT_EQ(got.size(), want.size()) << label; + for (size_t i = 0; i < got.size(); ++i) { + EXPECT_EQ(got[i].term, want[i].term) << label << " term order diverged at " << i; + EXPECT_EQ(got[i].docids, want[i].docids) << label << " " << got[i].term; + EXPECT_EQ(got[i].freqs, want[i].freqs) << label << " " << got[i].term; + EXPECT_EQ(got[i].positions_flat, want[i].positions_flat) << label << " " << got[i].term; + } +} + +// (1) Coverage + monotonicity. 200k unique pairs, one token each, keeps the +// arena small (~1.8 MiB of 9 B single-token chains) while the pair structures +// alone measure ~5 MiB -- the pre-G08 arena + slot-index sum (~2.9 MiB) sits +// BELOW the asserted floor, so this test FAILS without the G08 charges. +TEST(SniiSpimiResidentAccounting, ResidentCoversPairStructuresAndIsMonotone) { + SpimiTermBuffer buf(/*has_positions=*/true, /*spill_threshold_bytes=*/0); + const std::vector ids = intern_unigrams(&buf, 600); + + constexpr uint32_t kPairs = 200000; + uint64_t prev = buf.resident_bytes_for_test(); + for (uint32_t k = 0; k < kPairs; ++k) { + // Distinct ordered pairs (i, j); docids globally ascending. + buf.add_bigram_token(ids[k / 600], ids[k % 600], /*docid=*/k, /*pos=*/0); + const uint64_t now = buf.resident_bytes_for_test(); + // Monotone while nothing spills and nothing evicts (no diet): capacities + // never shrink, the intern set and heap payloads only grow. + ASSERT_GE(now, prev) << "resident_bytes decreased at pair " << k; + prev = now; + } + ASSERT_TRUE(buf.status().ok()); + ASSERT_EQ(buf.bigram_pair_terms_for_test(), kPairs); + + // Externally-measured floor of the G05 pair machinery ALONE: one flat-map + // slot (16 B pair + 1 control byte; size <= capacity) plus one reverse + // pair-key slot (8 B) per live pair. resident_bytes() must cover at least + // this -- it actually also charges the owned-vocab headers, the Term slot + // pool and the rank arrays on top. + const uint64_t pair_slot_bytes = sizeof(std::pair) + 1; + const uint64_t measured_floor = static_cast(buf.bigram_pair_terms_for_test()) * + (pair_slot_bytes + sizeof(uint64_t)); + EXPECT_GE(buf.resident_bytes_for_test(), measured_floor); +} + +// (2) Gate trigger on the bigram path: threshold 64 KiB, no reporter (the local +// fallback gate). After 100 unigrams the resident is one 32 KiB arena block + +// ~13 KiB of structures -- no spill. Unique pair interns then push the CHARGED +// structures (map slots, reverse slots, vocab headers, Term slots) over the +// threshold while the chains still fit the FIRST arena block, so the spill that +// fires is attributable to the G08 charges on the add_bigram_token path. The +// pre-G08 accounting (arena + slot index < 49 KiB for this whole feed) never +// spilled here. +TEST(SniiSpimiResidentAccounting, TinyThresholdSpillsOnBigramPathFromChargedStructures) { + SpimiTermBuffer buf(/*has_positions=*/true, /*spill_threshold_bytes=*/64 * 1024); + const std::vector ids = intern_unigrams(&buf, 100); + ASSERT_TRUE(buf.status().ok()); + ASSERT_EQ(buf.run_count_for_test(), 0U) << "unigram feed alone must not spill"; + + uint32_t pairs_fed = 0; + for (uint32_t k = 0; k < 2000 && buf.run_count_for_test() == 0; ++k) { + buf.add_bigram_token(ids[(k / 100) % 100], ids[k % 100], /*docid=*/1 + k, /*pos=*/0); + ++pairs_fed; + } + ASSERT_TRUE(buf.status().ok()); + // run_count_for_test() > 0: the gate-2 spill fired from the bigram add path. + ASSERT_GT(buf.run_count_for_test(), 0U) + << "pair-structure growth must trip the gate (pre-G08 accounting never did)"; + + // The spilled buffer still drains cleanly: every unigram and every fed pair + // term is emitted exactly once (no diet -> nothing evicted or dropped). + size_t seen = 0; + ASSERT_TRUE(buf.for_each_term_sorted([&seen](TermPostings&&) { ++seen; }).ok()); + EXPECT_EQ(seen, 100U + pairs_fed); + EXPECT_TRUE(buf.status().ok()); +} + +// (3) Equality vs no-spill control: the earlier, structure-driven spills change +// WHEN runs are cut, never WHAT is emitted. Two rounds per pair (docids +// globally ascending across rounds) force cross-run boundary coalescing for +// pair terms materialized+pinned by the mid-feed spill. +TEST(SniiSpimiResidentAccounting, StructureDrivenSpillEqualsNoSpillControl) { + SpimiTermBuffer spilled(/*has_positions=*/true, /*spill_threshold_bytes=*/64 * 1024); + SpimiTermBuffer control(/*has_positions=*/true, /*spill_threshold_bytes=*/0); + + constexpr uint32_t kPairs = 400; + for (SpimiTermBuffer* buf : {&spilled, &control}) { + const std::vector ids = intern_unigrams(buf, 100); + for (uint32_t k = 0; k < kPairs; ++k) { // round 1: docids 1..400 + buf->add_bigram_token(ids[(k / 100) % 100], ids[k % 100], /*docid=*/1 + k, /*pos=*/0); + } + for (uint32_t k = 0; k < kPairs; ++k) { // round 2: docids 2001..2400 + buf->add_bigram_token(ids[(k / 100) % 100], ids[k % 100], /*docid=*/2001 + k, + /*pos=*/0); + } + ASSERT_TRUE(buf->status().ok()); + } + ASSERT_GE(spilled.run_count_for_test(), 1U); + ASSERT_EQ(control.run_count_for_test(), 0U); + + const std::vector got = spilled.finalize_sorted(); + const std::vector want = control.finalize_sorted(); + ASSERT_TRUE(spilled.status().ok()) << spilled.status().to_string(); + ASSERT_TRUE(control.status().ok()) << control.status().to_string(); + ASSERT_EQ(want.size(), 100U + kPairs); + expect_same_stream(got, want, "structure-driven spill"); +} + +// (4a) Reporter net-zero, in-memory drain: what the intern paths credit (pair +// map, reverse slots, vocab headers + heap payloads, intern nodes) the terminal +// drain debits in full -- current_bytes() returns to exactly 0. +TEST(SniiSpimiResidentAccounting, OwnedModeReporterNetsToZeroAfterInMemoryDrain) { + MemoryReporter rep; + SpimiTermBuffer buf(/*has_positions=*/true, /*spill_threshold_bytes=*/0, &rep); + const std::vector ids = intern_unigrams(&buf, 50); + for (uint32_t k = 0; k < 200; ++k) { + buf.add_bigram_token(ids[k % 50], ids[(k + 1) % 50], /*docid=*/1 + k, /*pos=*/0); + } + EXPECT_GT(rep.current_bytes(), 0); + size_t seen = 0; + ASSERT_TRUE(buf.for_each_term_sorted([&seen](TermPostings&&) { ++seen; }).ok()); + EXPECT_GT(seen, 0U); + EXPECT_EQ(rep.current_bytes(), 0) << "terminal drain must refund every charged byte"; +} + +// (4b) Reporter net-zero, spilled path WITH diet evictions: mid-feed spills +// materialize survivors, evict df==1 pair terms (debiting their charges), and +// the post-merge release returns the vocab-side remainder -- 0 at the end. +TEST(SniiSpimiResidentAccounting, OwnedModeReporterNetsToZeroAfterSpilledDrain) { + MemoryReporter rep(/*consume_release=*/nullptr, /*cap_bytes=*/64 * 1024); + SpimiTermBuffer buf(/*has_positions=*/true, /*spill_threshold_bytes=*/0, &rep); + buf.configure_bigram_diet(/*vocab_cap_bytes=*/4096); // binding: evictions fire + const std::vector ids = intern_unigrams(&buf, 100); + for (uint32_t k = 0; k < 1200 && buf.run_count_for_test() < 2; ++k) { + buf.add_bigram_token(ids[(k / 100) % 100], ids[k % 100], /*docid=*/1 + k, /*pos=*/0); + } + ASSERT_TRUE(buf.status().ok()); + ASSERT_GE(buf.run_count_for_test(), 1U); + EXPECT_GT(rep.current_bytes(), 0); + size_t seen = 0; + ASSERT_TRUE(buf.for_each_term_sorted([&seen](TermPostings&&) { ++seen; }).ok()); + EXPECT_GT(seen, 0U); + EXPECT_EQ(rep.current_bytes(), 0) + << "spilled drain + diet evictions must refund every charged byte"; +} + +} // namespace diff --git a/be/test/storage/index/snii/writer/spimi_run_cap_test.cpp b/be/test/storage/index/snii/writer/spimi_run_cap_test.cpp new file mode 100644 index 00000000000000..5a6d5a4f7cb8da --- /dev/null +++ b/be/test/storage/index/snii/writer/spimi_run_cap_test.cpp @@ -0,0 +1,201 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +// G09 run-file FD hygiene: a SPIMI buffer's spill runs are ALL (re)opened +// simultaneously -- one fd each, held for the whole k-way merge -- so an +// unbounded run count across ~100 concurrent writers exhausted the BE nofile +// rlimit in the conc=16 wikipedia field failure ('Too many open files' at run +// reopen). The cap (set_max_run_files / config +// snii_spill_max_run_files_per_buffer) merge-compacts the accumulated runs +// into ONE before a new spill would exceed it. These tests pin: +// * the cap is HONORED: the tracked run count never exceeds it, and the +// compaction seam records each collapse; +// * compaction is INVISIBLE in the output: a capped buffer drains the +// byte-identical term stream (terms, docids, freqs, positions) of an +// uncapped control fed the same tokens; +// * cap 0 disables compaction entirely (pre-cap behavior). +using doris::Status; +using doris::snii::writer::SpimiTermBuffer; +using doris::snii::writer::TermPostings; + +namespace snii_testing = doris::snii::writer::testing; + +namespace { + +// Distinct short unigrams ("uaa", "uab", ...): SSO-sized, alpha-only, never +// bigram-marker-prefixed. +std::string unigram(uint32_t i) { + std::string s = "u"; + s += static_cast('a' + i % 26); + s += static_cast('a' + (i / 26) % 26); + s += static_cast('a' + (i / 676) % 26); + return s; +} + +// One feed step at docid k: the shared term "hot" gets TWO tokens in the SAME +// doc (freq 2 -- a spill lands between them in the per-token spill regime +// below, exercising boundary-doc coalescing across run seams and through a +// compaction) and unigram(k) gets one token (a distinct term per step). With +// the tests' tiny 1 KiB local threshold, resident (>= one fresh 32 KiB arena +// block) exceeds the cap on EVERY token and the arena floor (one block) is +// met as soon as a chain claims its block -- so every token cuts a run: +// N fed tokens == N run files, densely exercising the cap machinery with tiny +// step counts (an uncapped 30-step feed already holds 90 runs). +void feed_step(SpimiTermBuffer* buf, uint32_t k) { + buf->add_token("hot", /*docid=*/k, /*pos=*/0); + buf->add_token("hot", /*docid=*/k, /*pos=*/1); + buf->add_token(unigram(k), /*docid=*/k, /*pos=*/2); +} + +// Bounds the honored-cap search loop; far below unigram()'s 17576 distinct +// strings so every step's unigram is DISTINCT (the drained-term-count +// assertions rely on that). +constexpr uint32_t kMaxSteps = 16000; + +struct DrainedTerm { + std::vector docids; + std::vector freqs; + std::vector positions; +}; + +std::map drain(SpimiTermBuffer* buf) { + std::map out; + Status s = buf->for_each_term_sorted([&out](TermPostings&& tp) { + DrainedTerm& t = out[tp.term]; + t.docids = tp.docids; + t.freqs = tp.freqs; + if (tp.pos_pump) { + // Wide streamed term: pull the pump synchronously (the contract). + t.positions.resize(static_cast(tp.pos_total)); + if (tp.pos_total != 0) { + tp.pos_pump(t.positions.data(), t.positions.size()); + } + } else { + t.positions = tp.positions_flat; + } + }); + EXPECT_TRUE(s.ok()) << s.to_string(); + return out; +} + +TEST(SniiSpimiRunCap, CapIsHonoredAndCompactionSeamFires) { + snii_testing::reset_run_compactions(); + SpimiTermBuffer buf(/*has_positions=*/true, /*spill_threshold_bytes=*/1024); + constexpr size_t kCap = 3; + buf.set_max_run_files(kCap); + + // Feed until the cap has forced a compaction (bounded); the cap must hold + // at EVERY step, so track the running maximum of the run count. + size_t max_runs_seen = 0; + uint32_t steps = 0; + while (steps < kMaxSteps && snii_testing::run_compactions() < 1) { + feed_step(&buf, steps); + ++steps; + max_runs_seen = std::max(max_runs_seen, buf.run_count_for_test()); + } + ASSERT_TRUE(buf.status().ok()) << buf.status().to_string(); + ASSERT_GE(snii_testing::run_compactions(), 1U) + << "the cap was never hit within " << steps << " steps"; + EXPECT_LE(max_runs_seen, kCap) << "run count exceeded the cap"; + + // The capped buffer still drains every term exactly once with the full + // posting content. + std::map got = drain(&buf); + ASSERT_TRUE(buf.status().ok()) << buf.status().to_string(); + ASSERT_EQ(got.size(), static_cast(steps) + 1U); // distinct unigrams + "hot" + const DrainedTerm& hot = got.at("hot"); + ASSERT_EQ(hot.docids.size(), static_cast(steps)); + EXPECT_EQ(hot.docids.front(), 0U); + EXPECT_EQ(hot.docids.back(), steps - 1); + for (uint32_t f : hot.freqs) { + ASSERT_EQ(f, 2U) << "boundary-doc coalescing must survive compaction"; + } + ASSERT_EQ(hot.positions.size(), 2ULL * steps); + EXPECT_EQ(hot.positions[0], 0U); + EXPECT_EQ(hot.positions[1], 1U); + const DrainedTerm& first = got.at(unigram(0)); + ASSERT_EQ(first.docids.size(), 1U); + EXPECT_EQ(first.docids[0], 0U); + ASSERT_EQ(first.positions.size(), 1U); + EXPECT_EQ(first.positions[0], 2U); +} + +TEST(SniiSpimiRunCap, CompactedDrainMatchesUncappedControl) { + snii_testing::reset_run_compactions(); + // 30 steps == 90 tokens == 90 runs uncapped (per-token spills, see + // feed_step): plenty of cap-2 compactions on the capped side while the + // control's 90-way merge stays trivial. + constexpr uint32_t kSteps = 30; + + SpimiTermBuffer capped(/*has_positions=*/true, /*spill_threshold_bytes=*/1024); + capped.set_max_run_files(2); // aggressive: compact on nearly every spill + for (uint32_t k = 0; k < kSteps; ++k) { + feed_step(&capped, k); + } + ASSERT_TRUE(capped.status().ok()) << capped.status().to_string(); + ASSERT_GE(snii_testing::run_compactions(), 1U); + + SpimiTermBuffer control(/*has_positions=*/true, /*spill_threshold_bytes=*/1024); + control.set_max_run_files(0); // uncapped: the pre-cap spill pipeline + for (uint32_t k = 0; k < kSteps; ++k) { + feed_step(&control, k); + } + ASSERT_TRUE(control.status().ok()) << control.status().to_string(); + ASSERT_GT(control.run_count_for_test(), 2U) << "the control must hold many runs"; + + std::map got = drain(&capped); + std::map want = drain(&control); + ASSERT_TRUE(capped.status().ok()) << capped.status().to_string(); + ASSERT_TRUE(control.status().ok()) << control.status().to_string(); + ASSERT_EQ(got.size(), want.size()); + for (const auto& [term, w] : want) { + auto it = got.find(term); + ASSERT_NE(it, got.end()) << "missing term: " << term; + EXPECT_EQ(it->second.docids, w.docids) << term; + EXPECT_EQ(it->second.freqs, w.freqs) << term; + EXPECT_EQ(it->second.positions, w.positions) << term; + } +} + +TEST(SniiSpimiRunCap, ZeroCapDisablesCompaction) { + snii_testing::reset_run_compactions(); + SpimiTermBuffer buf(/*has_positions=*/true, /*spill_threshold_bytes=*/1024); + buf.set_max_run_files(0); + constexpr uint32_t kSteps = 30; // 90 per-token spill runs, uncapped + for (uint32_t k = 0; k < kSteps; ++k) { + feed_step(&buf, k); + } + ASSERT_TRUE(buf.status().ok()) << buf.status().to_string(); + EXPECT_GT(buf.run_count_for_test(), 2U); + EXPECT_EQ(snii_testing::run_compactions(), 0U); + // Still drains cleanly through the plain multi-run merge. + std::map got = drain(&buf); + EXPECT_EQ(got.size(), static_cast(kSteps) + 1U); +} + +} // namespace diff --git a/be/test/storage/index/snii/writer/spimi_spill_writer_test.cpp b/be/test/storage/index/snii/writer/spimi_spill_writer_test.cpp new file mode 100644 index 00000000000000..a1cabc3778445d --- /dev/null +++ b/be/test/storage/index/snii/writer/spimi_spill_writer_test.cpp @@ -0,0 +1,324 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/io/metered_file_reader.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/logical_index_writer.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +using namespace doris::snii; +using namespace doris::snii::format; +using namespace doris::snii::writer; +using doris::Status; + +namespace { + +std::string TempPath() { + static int counter = 0; + return "/tmp/snii_spill_test_" + std::to_string(getpid()) + "_" + std::to_string(counter++) + + ".idx"; +} + +std::vector ReadAll(const std::string& path) { + io::LocalFileReader r; + EXPECT_TRUE(r.open(path).ok()); + std::vector out; + EXPECT_TRUE(r.read_at(0, r.size(), &out).ok()); + return out; +} + +// Deterministic (term, doc, pos) stream with globally ascending docids. Mixes +// high-df ("alpha", every doc), mid-df, multi-token docs, and a term whose docs +// straddle arbitrary spill boundaries -- so the spill path exercises a term in +// one run, a term in every run, and a spill boundary mid-term. +void Feed(SpimiTermBuffer* buf, uint32_t doc_count) { + for (uint32_t d = 0; d < doc_count; ++d) { + buf->add_token("alpha", d, 0); // every doc (spans all runs) + buf->add_token("alpha", d, 7); // freq 2 in every doc + if (d % 2 == 0) { + buf->add_token("beta", d, 1); + } + if (d % 3 == 0) { + buf->add_token("gamma", d, 2); + } + if (d % 5 == 0) { + buf->add_token("delta", d, 3); + buf->add_token("delta", d, 9); + } + if (d == 1) { + buf->add_token("singleton", d, 4); // only one doc, one run + } + } +} + +SniiIndexInput BaseInput(uint32_t doc_count) { + SniiIndexInput in; + in.index_id = 1; + in.index_suffix = "body"; + in.config = IndexConfig::kDocsPositions; + in.doc_count = doc_count; + in.target_dict_block_bytes = 512; // force several DICT blocks + return in; +} + +std::vector WriteContainer(const SniiIndexInput& in) { + const std::string path = TempPath(); + io::LocalFileWriter writer; + EXPECT_TRUE(writer.open(path).ok()); + SniiCompoundWriter compound(&writer); + EXPECT_TRUE(compound.add_logical_index(in).ok()); + EXPECT_TRUE(compound.finish().ok()); + std::vector bytes = ReadAll(path); + std::remove(path.c_str()); + return bytes; +} + +// Builds a container by STREAMING a freshly-fed buffer at the given spill +// threshold (0 == unlimited), returning the container bytes. +std::vector BuildStreamed(uint32_t docs, size_t spill_bytes) { + SpimiTermBuffer buf(/*has_positions=*/true, spill_bytes); + Feed(&buf, docs); + SniiIndexInput in = BaseInput(docs); + in.term_source = &buf; + return WriteContainer(in); +} + +// Opens a container from bytes and runs a couple of queries for a sanity match. +struct OpenedIndex { + std::string path; + io::LocalFileReader local; + std::unique_ptr metered; + reader::SniiSegmentReader segment; + reader::LogicalIndexReader index; + + ~OpenedIndex() { + if (!path.empty()) { + std::remove(path.c_str()); + } + } +}; + +void OpenFromBytes(const std::vector& bytes, OpenedIndex* out) { + out->path = TempPath(); + io::LocalFileWriter w; + ASSERT_TRUE(w.open(out->path).ok()); + ASSERT_TRUE(w.append(Slice(bytes)).ok()); + ASSERT_TRUE(w.finalize().ok()); + ASSERT_TRUE(out->local.open(out->path).ok()); + out->metered = std::make_unique(&out->local); + ASSERT_TRUE(reader::SniiSegmentReader::open(out->metered.get(), &out->segment).ok()); + ASSERT_TRUE(out->segment.open_index(1, "body", &out->index).ok()); +} + +std::vector TermDocs(OpenedIndex* idx, const std::string& term) { + std::vector docs; + const Status s = query::term_query(idx->index, term, &docs); + EXPECT_TRUE(s.ok()) << "term=" << term << " err=" << s.to_string(); + return docs; +} + +} // namespace + +// CORE GUARANTEE: building the SAME corpus with a tiny spill threshold (forcing +// many spills + a final k-way merge) yields a container that is BYTE-FOR-BYTE +// identical to the unlimited in-memory build. +TEST(SniiSpimiSpillWriter, SpilledMatchesUnlimitedBytes) { + constexpr uint32_t kDocs = 400; + const std::vector unlimited = BuildStreamed(kDocs, /*spill=*/0); + // ~4 KiB threshold forces dozens of spills across this corpus. + const std::vector spilled = BuildStreamed(kDocs, /*spill=*/4096); + + ASSERT_EQ(unlimited.size(), spilled.size()); + EXPECT_EQ(unlimited, spilled); +} + +// Regression: the wide term "alpha" (every doc, freq 2) can have a doc split across +// a spill seam, so its PRE-coalesce total_docs reaches >= kSlimDfThreshold (512) +// while its POST-coalesce df stays below it. The merge gate keyed on total_docs +// would then STREAM positions (leaving positions_flat empty) into the slim writer +// path, which reads positions_flat directly -> segfault. Sweeping docs around the +// 512 boundary x several spill thresholds exercises the band; byte-identity vs the +// unlimited build proves both no-crash and identical output. +TEST(SniiSpimiSpillWriter, WideTermDfNearThresholdAcrossSeamMatchesUnlimited) { + for (uint32_t docs = 505; docs <= 515; ++docs) { + const std::vector unlimited = BuildStreamed(docs, /*spill=*/0); + for (size_t spill : {size_t {1024}, size_t {2048}, size_t {4096}}) { + const std::vector spilled = BuildStreamed(docs, spill); + ASSERT_EQ(unlimited.size(), spilled.size()) << "docs=" << docs << " spill=" << spill; + EXPECT_EQ(unlimited, spilled) << "docs=" << docs << " spill=" << spill; + } + } +} + +// Identical bytes regardless of threshold size: a mid threshold (one or two +// spills) must also match the unlimited build exactly. +TEST(SniiSpimiSpillWriter, MidThresholdAlsoIdentical) { + constexpr uint32_t kDocs = 400; + const std::vector unlimited = BuildStreamed(kDocs, /*spill=*/0); + const std::vector mid = BuildStreamed(kDocs, /*spill=*/32 * 1024); + EXPECT_EQ(unlimited, mid); +} + +// An extremely small threshold (spill almost every token) still produces the +// identical container -- stresses many single-term runs and the merge. +TEST(SniiSpimiSpillWriter, ExtremeSpillIdentical) { + constexpr uint32_t kDocs = 200; + const std::vector unlimited = BuildStreamed(kDocs, /*spill=*/0); + const std::vector tiny = BuildStreamed(kDocs, /*spill=*/1); + EXPECT_EQ(unlimited, tiny); +} + +// No-positions config with spilling still matches the in-memory build. +TEST(SniiSpimiSpillWriter, NoPositionsSpillIdentical) { + constexpr uint32_t kDocs = 300; + auto build = [&](size_t spill) { + SpimiTermBuffer buf(/*has_positions=*/false, spill); + for (uint32_t d = 0; d < kDocs; ++d) { + buf.add_token("alpha", d, 0); + if (d % 2 == 0) { + buf.add_token("beta", d, 0); + } + if (d % 4 == 0) { + buf.add_token("gamma", d, 0); + } + } + SniiIndexInput in = BaseInput(kDocs); + in.config = IndexConfig::kDocsOnly; // docids only (no positions section) + in.term_source = &buf; + return WriteContainer(in); + }; + EXPECT_EQ(build(0), build(2048)); +} + +// Queries against the spilled and unlimited containers return identical results. +// Uses >= kSlimDfThreshold docs so the high-df term is windowed (populating the +// .prx POD) -- the segment reader infers has_positions from a non-empty prx POD. +TEST(SniiSpimiSpillWriter, QueriesMatchAcrossSpill) { + constexpr uint32_t kDocs = 700; + OpenedIndex un, sp; + OpenFromBytes(BuildStreamed(kDocs, 0), &un); + OpenFromBytes(BuildStreamed(kDocs, 4096), &sp); + + for (const char* term : {"alpha", "beta", "gamma", "delta", "singleton", "absent"}) { + EXPECT_EQ(TermDocs(&un, term), TermDocs(&sp, term)) << "term=" << term; + } + std::vector un_phrase, sp_phrase; + ASSERT_TRUE(query::phrase_query(un.index, {"alpha", "alpha"}, &un_phrase).ok()); + ASSERT_TRUE(query::phrase_query(sp.index, {"alpha", "alpha"}, &sp_phrase).ok()); + EXPECT_EQ(un_phrase, sp_phrase); +} + +// WIDE-TERM STREAMED MERGE byte-identity: with enough docs that the high-df term +// "alpha" exceeds kSlimDfThreshold, the spilled k-way merge takes the WIDE-term +// position-STREAMING path (pos_pump pulled across runs) instead of materializing +// the term's full positions_flat. The produced container MUST stay byte-for-byte +// identical to the unlimited in-memory build (which materializes positions). This +// is the direct regression guard for the merge-phase peak-RSS streaming change. +TEST(SniiSpimiSpillWriter, WideTermStreamedMergeMatchesUnlimitedBytes) { + // 1200 docs -> alpha df=1200 (>= kSlimDfThreshold 512) -> windowed + streamed. + constexpr uint32_t kDocs = 1200; + static_assert(kDocs >= doris::snii::format::kSlimDfThreshold, "alpha must be a wide term"); + const std::vector unlimited = BuildStreamed(kDocs, /*spill=*/0); + // Small threshold forces many spills, so alpha lands in EVERY run and the merge + // coalesces it across runs via the streamed pump. + const std::vector spilled = BuildStreamed(kDocs, /*spill=*/8192); + ASSERT_EQ(unlimited.size(), spilled.size()); + EXPECT_EQ(unlimited, spilled); +} + +// Single-doc corpus with spilling: an edge corpus must not crash or diverge. +TEST(SniiSpimiSpillWriter, SingleDocCorpus) { + SpimiTermBuffer un(/*has_positions=*/true, 0); + SpimiTermBuffer sp(/*has_positions=*/true, 1); + for (auto* b : {&un, &sp}) { + b->add_token("only", 0, 0); + b->add_token("only", 0, 1); + b->add_token("word", 0, 2); + } + SniiIndexInput un_in = BaseInput(1); + un_in.term_source = &un; + SniiIndexInput sp_in = BaseInput(1); + sp_in.term_source = &sp; + EXPECT_EQ(WriteContainer(un_in), WriteContainer(sp_in)); +} + +// finalize_sorted (the materialized accessor) also reflects spilled runs and is +// byte-identical to the in-memory result at the postings level. +TEST(SniiSpimiSpillWriter, FinalizeSortedMatchesAcrossSpill) { + constexpr uint32_t kDocs = 150; + SpimiTermBuffer un(/*has_positions=*/true, 0); + SpimiTermBuffer sp(/*has_positions=*/true, 256); + Feed(&un, kDocs); + Feed(&sp, kDocs); + const std::vector a = un.finalize_sorted(); + const std::vector b = sp.finalize_sorted(); + ASSERT_EQ(a.size(), b.size()); + for (size_t i = 0; i < a.size(); ++i) { + EXPECT_EQ(a[i].term, b[i].term); + EXPECT_EQ(a[i].docids, b[i].docids); + EXPECT_EQ(a[i].freqs, b[i].freqs); + EXPECT_EQ(a[i].positions_flat, b[i].positions_flat); + } + EXPECT_TRUE(un.status().ok()); + EXPECT_TRUE(sp.status().ok()); +} + +// Gate-2 trigger now compares REAL resident bytes (pool_.arena_bytes() + +// slot_of_.capacity()*4) against the configured cap, NOT the old per-token +// estimate. With a small cap, the first 32 KiB arena block immediately exceeds it, +// so at least one spill fires; the unlimited (cap=0) build never spills. +TEST(SniiSpimiSpillWriter, ArenaByteCapTriggersSpill) { + constexpr uint32_t kDocs = 400; + + SpimiTermBuffer capped(/*has_positions=*/true, /*spill=*/4096); + Feed(&capped, kDocs); + SniiIndexInput capped_in = BaseInput(kDocs); + capped_in.term_source = &capped; + const std::vector capped_bytes = WriteContainer(capped_in); + EXPECT_TRUE(capped.status().ok()); + // A spill ran: real resident (>= one 32 KiB block) crossed the 4 KiB cap. + EXPECT_GE(capped.run_count_for_test(), 1U); + + SpimiTermBuffer unlimited(/*has_positions=*/true, /*spill=*/0); + Feed(&unlimited, kDocs); + SniiIndexInput unlimited_in = BaseInput(kDocs); + unlimited_in.term_source = &unlimited; + const std::vector unlimited_bytes = WriteContainer(unlimited_in); + EXPECT_TRUE(unlimited.status().ok()); + // Unlimited never spills (corpus fits well under the arena hard-stop). + EXPECT_EQ(unlimited.run_count_for_test(), 0U); + + // And the metric switch did not change the output: byte-for-byte identical. + ASSERT_EQ(capped_bytes.size(), unlimited_bytes.size()); + EXPECT_EQ(capped_bytes, unlimited_bytes); +} diff --git a/be/test/storage/index/snii/writer/spimi_streaming_writer_test.cpp b/be/test/storage/index/snii/writer/spimi_streaming_writer_test.cpp new file mode 100644 index 00000000000000..23e5a3bdc87c41 --- /dev/null +++ b/be/test/storage/index/snii/writer/spimi_streaming_writer_test.cpp @@ -0,0 +1,208 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/io/local_file.h" +#include "storage/index/snii/writer/logical_index_writer.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +using namespace doris::snii; +using namespace doris::snii::format; +using namespace doris::snii::writer; + +namespace { + +std::string TempPath() { + static int counter = 0; + return "/tmp/snii_stream_test_" + std::to_string(getpid()) + "_" + std::to_string(counter++) + + ".idx"; +} + +std::vector ReadAll(const std::string& path) { + io::LocalFileReader r; + EXPECT_TRUE(r.open(path).ok()); + std::vector out; + EXPECT_TRUE(r.read_at(0, r.size(), &out).ok()); + return out; +} + +// Feeds a deterministic (term, doc, pos) stream into a SPIMI buffer. Docids +// arrive in ascending order per term (the normal tokenizer contract); some +// terms span many docs so both slim and (with enough docs) windowed paths and +// the DICT block splitter are exercised. +void Feed(SpimiTermBuffer* buf, uint32_t doc_count) { + for (uint32_t d = 0; d < doc_count; ++d) { + buf->add_token("alpha", d, 0); // every doc: high df + if (d % 2 == 0) { + buf->add_token("beta", d, 1); // half the docs + } + if (d % 7 == 0) { + buf->add_token("gamma", d, 2); + buf->add_token("gamma", d, 5); // freq 2 in this doc + } + if (d == 3 || d == 4) { + buf->add_token("delta", d, d); // tiny df + } + } +} + +// Writes a single-index container from a SniiIndexInput and returns the bytes. +std::vector WriteContainer(const SniiIndexInput& in) { + const std::string path = TempPath(); + io::LocalFileWriter writer; + EXPECT_TRUE(writer.open(path).ok()); + SniiCompoundWriter compound(&writer); + EXPECT_TRUE(compound.add_logical_index(in).ok()); + EXPECT_TRUE(compound.finish().ok()); + std::vector bytes = ReadAll(path); + std::remove(path.c_str()); + return bytes; +} + +SniiIndexInput BaseInput(uint32_t doc_count) { + SniiIndexInput in; + in.index_id = 1; + in.index_suffix = "body"; + in.config = IndexConfig::kDocsPositions; + in.doc_count = doc_count; + in.target_dict_block_bytes = 512; // force several DICT blocks + return in; +} + +} // namespace + +// The streaming term_source path must produce a BYTE-IDENTICAL container to the +// materialized terms vector path: the flat-array accumulator + stream-finalize +// must not change a single output byte. +TEST(SniiSpimiStreamingWriter, StreamingMatchesMaterializedBytes) { + constexpr uint32_t kDocs = 300; + + SpimiTermBuffer mat_buf(/*has_positions=*/true); + Feed(&mat_buf, kDocs); + SniiIndexInput mat_in = BaseInput(kDocs); + mat_in.terms = mat_buf.finalize_sorted(); + const std::vector mat_bytes = WriteContainer(mat_in); + + SpimiTermBuffer stream_buf(/*has_positions=*/true); + Feed(&stream_buf, kDocs); + SniiIndexInput stream_in = BaseInput(kDocs); + stream_in.term_source = &stream_buf; + const std::vector stream_bytes = WriteContainer(stream_in); + + ASSERT_EQ(mat_bytes.size(), stream_bytes.size()); + EXPECT_EQ(mat_bytes, stream_bytes); +} + +// Position STREAMING (pos_pump) path: a term whose token count exceeds the +// streaming threshold (65536) has its positions pulled per-window from the arena +// chain instead of a materialized positions_flat. The streamed (term_source) build +// must still be BYTE-IDENTICAL to the materialized (finalize_sorted) build -- the +// pump must yield the exact same positions in the same order, so every .prx byte +// (and thus the whole container) matches. This is the byte-identity guard for the +// peak-RSS optimization. +TEST(SniiSpimiStreamingWriter, StreamedPositionsMatchMaterializedBytesHighDf) { + // ~80k docs of a single high-df term gives that term > 65536 tokens, crossing the + // streaming threshold. A few extra terms exercise the slim (non-streamed) path + // alongside it so both encodings appear in one container. + constexpr uint32_t kDocs = 80000; + auto feed = [](SpimiTermBuffer* buf) { + for (uint32_t d = 0; d < kDocs; ++d) { + buf->add_token("hot", d, 0); // every doc: > 65536 tokens + if (d % 1000 == 0) { + buf->add_token("cold", d, 1); // tiny df (slim path) + } + } + }; + + SpimiTermBuffer mat_buf(/*has_positions=*/true); + feed(&mat_buf); + SniiIndexInput mat_in = BaseInput(kDocs); + mat_in.terms = mat_buf.finalize_sorted(); // materialized: positions_flat + const std::vector mat_bytes = WriteContainer(mat_in); + + SpimiTermBuffer stream_buf(/*has_positions=*/true); + feed(&stream_buf); + SniiIndexInput stream_in = BaseInput(kDocs); + stream_in.term_source = &stream_buf; // streamed: pos_pump for "hot" + const std::vector stream_bytes = WriteContainer(stream_in); + + ASSERT_EQ(mat_bytes.size(), stream_bytes.size()); + EXPECT_EQ(mat_bytes, stream_bytes); +} + +// A term repeated MANY times within FEW docs crosses the streaming token threshold +// (ntok >= 65536) yet has a LOW df (< kSlimDfThreshold == 512), so it takes the SLIM +// writer path -- which reads positions_flat directly and does NOT honor pos_pump. +// The stream candidate must therefore fall back to materializing positions for this +// term; otherwise build_slim_entry sees an empty positions_flat and reads out of +// bounds (deterministic segfault). Byte-identity vs the materialized build proves +// both that it does not crash and that the fallback produces the exact same bytes. +TEST(SniiSpimiStreamingWriter, StreamedLowDfHighNtokMatchesMaterialized) { + constexpr uint32_t kDocs = 200; // df = 200 (< 512 -> slim path) + constexpr uint32_t kReps = 400; // 200 * 400 = 80000 tokens (> 65536 -> stream) + auto feed = [](SpimiTermBuffer* buf) { + for (uint32_t d = 0; d < kDocs; ++d) { + for (uint32_t p = 0; p < kReps; ++p) { + buf->add_token("rep", d, p); + } + } + }; + + SpimiTermBuffer mat_buf(/*has_positions=*/true); + feed(&mat_buf); + SniiIndexInput mat_in = BaseInput(kDocs); + mat_in.terms = mat_buf.finalize_sorted(); + const std::vector mat_bytes = WriteContainer(mat_in); + + SpimiTermBuffer stream_buf(/*has_positions=*/true); + feed(&stream_buf); + SniiIndexInput stream_in = BaseInput(kDocs); + stream_in.term_source = &stream_buf; + const std::vector stream_bytes = WriteContainer(stream_in); + + ASSERT_EQ(mat_bytes.size(), stream_bytes.size()); + EXPECT_EQ(mat_bytes, stream_bytes); +} + +// The streaming path drains its source: after build the buffer is empty. +TEST(SniiSpimiStreamingWriter, StreamingConsumesSource) { + SpimiTermBuffer buf(/*has_positions=*/true); + Feed(&buf, 50); + EXPECT_GT(buf.unique_terms(), 0U); + + SniiIndexInput in = BaseInput(50); + in.term_source = &buf; + LogicalIndexWriter writer(in); + // build() streams the posting region straight into a FileWriter sink; this test + // only asserts the source is drained, so a throwaway temp sink suffices. + const std::string post_path = TempPath(); + io::LocalFileWriter post; + ASSERT_TRUE(post.open(post_path).ok()); + ASSERT_TRUE(writer.build(&post).ok()); + EXPECT_EQ(buf.unique_terms(), 0U); + std::remove(post_path.c_str()); +} diff --git a/be/test/storage/index/snii/writer/spimi_term_buffer_test.cpp b/be/test/storage/index/snii/writer/spimi_term_buffer_test.cpp new file mode 100644 index 00000000000000..4b6ce235cf288b --- /dev/null +++ b/be/test/storage/index/snii/writer/spimi_term_buffer_test.cpp @@ -0,0 +1,683 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/spimi_term_buffer.h" + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "common/status.h" + +using doris::snii::writer::MemoryReporter; +using doris::snii::writer::SpimiTermBuffer; +using doris::snii::writer::TermPostings; +using doris::Status; + +// Tokens accumulate into sorted terms with ascending docids and per-doc positions. +TEST(SniiSpimiTermBuffer, AccumulateAndSort) { + SpimiTermBuffer buf(/*has_positions=*/true); + // doc 0: "banana apple apple" + buf.add_token("banana", 0, 0); + buf.add_token("apple", 0, 1); + buf.add_token("apple", 0, 2); + // doc 1: "apple cherry" + buf.add_token("apple", 1, 0); + buf.add_token("cherry", 1, 1); + + EXPECT_EQ(buf.unique_terms(), 3U); + EXPECT_EQ(buf.total_tokens(), 5U); + + std::vector terms = buf.finalize_sorted(); + ASSERT_EQ(terms.size(), 3U); + // Sorted lexicographically: apple, banana, cherry. + EXPECT_EQ(terms[0].term, "apple"); + EXPECT_EQ(terms[1].term, "banana"); + EXPECT_EQ(terms[2].term, "cherry"); + + // apple: docs 0 (freq 2, pos {1,2}) and 1 (freq 1, pos {0}). + const TermPostings& apple = terms[0]; + ASSERT_EQ(apple.docids.size(), 2U); + EXPECT_EQ(apple.docids[0], 0U); + EXPECT_EQ(apple.freqs[0], 2U); + ASSERT_EQ(apple.doc_positions(0).size(), 2U); + EXPECT_EQ(apple.doc_positions(0)[0], 1U); + EXPECT_EQ(apple.doc_positions(0)[1], 2U); + EXPECT_EQ(apple.docids[1], 1U); + EXPECT_EQ(apple.freqs[1], 1U); +} + +// Without positions, freq is still counted but positions vectors stay empty. +TEST(SniiSpimiTermBuffer, DocsOnlyNoPositions) { + SpimiTermBuffer buf(/*has_positions=*/false); + buf.add_token("x", 0, 0); + buf.add_token("x", 0, 1); + buf.add_token("x", 2, 0); + + std::vector terms = buf.finalize_sorted(); + ASSERT_EQ(terms.size(), 1U); + EXPECT_EQ(terms[0].term, "x"); + ASSERT_EQ(terms[0].docids.size(), 2U); + EXPECT_EQ(terms[0].docids[0], 0U); + EXPECT_EQ(terms[0].freqs[0], 2U); + EXPECT_EQ(terms[0].docids[1], 2U); + EXPECT_EQ(terms[0].freqs[1], 1U); + EXPECT_TRUE(terms[0].positions_flat.empty()); +} + +TEST(SniiSpimiTermBuffer, Empty) { + SpimiTermBuffer buf(true); + EXPECT_EQ(buf.unique_terms(), 0U); + EXPECT_TRUE(buf.finalize_sorted().empty()); +} + +// Feeds the same token stream into two buffers and asserts the streaming +// for_each_term_sorted produces the byte-identical postings finalize_sorted +// returns (same flat-array refactor must not change observable output). +TEST(SniiSpimiTermBuffer, StreamingMatchesMaterialized) { + auto feed = [](SpimiTermBuffer& b) { + b.add_token("banana", 0, 0); + b.add_token("apple", 0, 1); + b.add_token("apple", 0, 2); + b.add_token("apple", 1, 0); + b.add_token("cherry", 1, 1); + b.add_token("apple", 5, 3); + b.add_token("apple", 5, 7); + b.add_token("banana", 9, 0); + }; + SpimiTermBuffer mat(/*has_positions=*/true); + SpimiTermBuffer strm(/*has_positions=*/true); + feed(mat); + feed(strm); + + std::vector material = mat.finalize_sorted(); + std::vector streamed; + Status st = strm.for_each_term_sorted( + [&](TermPostings&& tp) { streamed.push_back(std::move(tp)); }); + EXPECT_TRUE(st.ok()); + + ASSERT_EQ(material.size(), streamed.size()); + for (size_t i = 0; i < material.size(); ++i) { + EXPECT_EQ(material[i].term, streamed[i].term); + EXPECT_EQ(material[i].docids, streamed[i].docids); + EXPECT_EQ(material[i].freqs, streamed[i].freqs); + EXPECT_EQ(material[i].positions_flat, streamed[i].positions_flat); + } + // apple: docs {0(pos 1,2), 1(pos 0), 5(pos 3,7)} -> positions re-sliced by freq. + ASSERT_EQ(streamed[0].term, "apple"); + ASSERT_EQ(streamed[0].docids.size(), 3U); + EXPECT_EQ(streamed[0].freqs, (std::vector {2U, 1U, 2U})); + EXPECT_EQ(std::vector(streamed[0].doc_positions(2).begin(), + streamed[0].doc_positions(2).end()), + (std::vector {3U, 7U})); +} + +// Out-of-order docid GROUPS (each doc's tokens stay contiguous, but the docids +// are not non-decreasing) are tolerated and reordered once at finalize, with +// each doc carrying its own positions (defensive fallback path, e.g. a merge of +// pre-sorted runs). Tokens for a single docid are always contiguous. +TEST(SniiSpimiTermBuffer, OutOfOrderDocidsSortedAtFinalize) { + SpimiTermBuffer buf(/*has_positions=*/true); + buf.add_token("t", 5, 50); // doc 5 group (contiguous) + buf.add_token("t", 5, 51); + buf.add_token("t", 1, 10); // doc 1 group, arrives after doc 5 + buf.add_token("t", 1, 11); + buf.add_token("t", 3, 30); // doc 3 group + + std::vector terms = buf.finalize_sorted(); + ASSERT_EQ(terms.size(), 1U); + const TermPostings& t = terms[0]; + EXPECT_EQ(t.docids, (std::vector {1U, 3U, 5U})); + EXPECT_EQ(t.freqs, (std::vector {2U, 1U, 2U})); + EXPECT_EQ(t.positions_flat, (std::vector {10U, 11U, 30U, 50U, 51U})); +} + +// BORROWED-vocab id path: feeding raw term-ids (no per-token string work) +// produces the SAME lexicographically sorted postings as the string path. The +// vocab order (apple < banana < cherry) drives the emitted order, NOT the id +// order (banana=0, apple=1, cherry=2). +TEST(SniiSpimiTermBuffer, TermIdPathMatchesStringPath) { + const std::vector vocab = {"banana", "apple", "cherry"}; + SpimiTermBuffer buf(&vocab, /*has_positions=*/true); + // doc 0: "banana apple apple", doc 1: "apple cherry" -- by id. + buf.add_token(0, 0, 0); // banana + buf.add_token(1, 0, 1); // apple + buf.add_token(1, 0, 2); // apple + buf.add_token(1, 1, 0); // apple + buf.add_token(2, 1, 1); // cherry + + EXPECT_EQ(buf.unique_terms(), 3U); + EXPECT_EQ(buf.total_tokens(), 5U); + EXPECT_TRUE(buf.status().ok()); + + std::vector terms = buf.finalize_sorted(); + ASSERT_EQ(terms.size(), 3U); + EXPECT_EQ(terms[0].term, "apple"); + EXPECT_EQ(terms[1].term, "banana"); + EXPECT_EQ(terms[2].term, "cherry"); + const TermPostings& apple = terms[0]; + ASSERT_EQ(apple.docids.size(), 2U); + EXPECT_EQ(apple.freqs[0], 2U); + EXPECT_EQ(std::vector(apple.doc_positions(0).begin(), apple.doc_positions(0).end()), + (std::vector {1U, 2U})); + EXPECT_EQ(apple.docids[1], 1U); + EXPECT_EQ(apple.freqs[1], 1U); +} + +// A term-id never touched is simply skipped (no empty term emitted); an empty +// vocab yields no terms and stays valid. +TEST(SniiSpimiTermBuffer, UntouchedIdSkippedAndEmptyVocab) { + const std::vector vocab = {"a", "b", "c", "d"}; + SpimiTermBuffer buf(&vocab, /*has_positions=*/false); + buf.add_token(0, 0, 0); // a + buf.add_token(2, 1, 0); // c -- ids 1 (b) and 3 (d) never touched + std::vector terms = buf.finalize_sorted(); + ASSERT_EQ(terms.size(), 2U); + EXPECT_EQ(terms[0].term, "a"); + EXPECT_EQ(terms[1].term, "c"); + + const std::vector empty; + SpimiTermBuffer empty_buf(&empty, /*has_positions=*/false); + EXPECT_EQ(empty_buf.unique_terms(), 0U); + EXPECT_TRUE(empty_buf.finalize_sorted().empty()); + EXPECT_TRUE(empty_buf.status().ok()); +} + +// An out-of-range term-id is rejected: the token is ignored and an +// InvalidArgument is latched into status(). +TEST(SniiSpimiTermBuffer, OutOfRangeTermIdRejected) { + const std::vector vocab = {"x", "y"}; + SpimiTermBuffer buf(&vocab, /*has_positions=*/true); + buf.add_token(0, 0, 0); // valid + buf.add_token(5, 0, 1); // out of range -> ignored + latched + EXPECT_FALSE(buf.status().ok()); + EXPECT_EQ(buf.unique_terms(), 1U); + EXPECT_EQ(buf.total_tokens(), 1U); // the rejected token was not counted +} + +// The borrowed-vocab id path is byte-identical across a spill: a tiny threshold +// (many spills + k-way merge over term-id runs) must match the unlimited build. +TEST(SniiSpimiTermBuffer, TermIdSpillMatchesUnlimited) { + const std::vector vocab = {"alpha", "beta", "gamma", "delta"}; + auto feed = [&](SpimiTermBuffer& b) { + for (uint32_t d = 0; d < 300; ++d) { + b.add_token(0, d, 0); // alpha: every doc + b.add_token(0, d, 9); // freq 2 + if (d % 2 == 0) { + b.add_token(1, d, 1); // beta + } + if (d % 3 == 0) { + b.add_token(2, d, 2); // gamma + } + if (d % 5 == 0) { + b.add_token(3, d, 3); // delta + } + } + }; + SpimiTermBuffer un(&vocab, /*has_positions=*/true, /*spill=*/0); + SpimiTermBuffer sp(&vocab, /*has_positions=*/true, /*spill=*/256); + feed(un); + feed(sp); + const std::vector a = un.finalize_sorted(); + const std::vector b = sp.finalize_sorted(); + ASSERT_EQ(a.size(), b.size()); + for (size_t i = 0; i < a.size(); ++i) { + EXPECT_EQ(a[i].term, b[i].term); + EXPECT_EQ(a[i].docids, b[i].docids); + EXPECT_EQ(a[i].freqs, b[i].freqs); + EXPECT_EQ(a[i].positions_flat, b[i].positions_flat); + } + EXPECT_TRUE(un.status().ok()); + EXPECT_TRUE(sp.status().ok()); +} + +// for_each_term_sorted drains the buffer term-by-term: after each callback the +// consumed term is gone, so at most one term's arrays remain materialized. +TEST(SniiSpimiTermBuffer, StreamingDrainsAndShrinks) { + SpimiTermBuffer buf(/*has_positions=*/false); + for (uint32_t d = 0; d < 100; ++d) { + buf.add_token("a", d, 0); + buf.add_token("b", d, 0); + buf.add_token("c", d, 0); + } + EXPECT_EQ(buf.unique_terms(), 3U); + std::vector remaining_after_each; + size_t seen = 0; + EXPECT_TRUE(buf.for_each_term_sorted([&](TermPostings&& tp) { + ++seen; + EXPECT_EQ(tp.docids.size(), 100U); + remaining_after_each.push_back(buf.unique_terms()); + }).ok()); + EXPECT_EQ(seen, 3U); + // After consuming each of the 3 terms, the live count drops 2,1,0. + EXPECT_EQ(remaining_after_each, (std::vector {2U, 1U, 0U})); + EXPECT_EQ(buf.unique_terms(), 0U); +} + +// A REVISITED docid (the out-of-order defensive path actually re-touches a doc: +// feed 5,1,5) MUST coalesce into ONE entry per docid -- summed freq, positions +// concatenated in document order -- matching the k-way merge path and the writer's +// strictly-ascending precondition. Without coalescing this yielded docids +// {1,5,5} (duplicate, unsorted) that the writer later rejects. +TEST(SniiSpimiTermBuffer, RevisitedDocidCoalescesWithPositions) { + SpimiTermBuffer buf(/*has_positions=*/true); + buf.add_token("t", 5, 50); // doc 5, first visit + buf.add_token("t", 5, 51); + buf.add_token("t", 1, 10); // doc 1 + buf.add_token("t", 5, 52); // doc 5 REVISITED (a fresh doc-group, same docid) + + std::vector terms = buf.finalize_sorted(); + ASSERT_EQ(terms.size(), 1U); + const TermPostings& t = terms[0]; + // Exactly one entry per docid, strictly ascending. + EXPECT_EQ(t.docids, (std::vector {1U, 5U})); + // doc 5 freq = 2 (first visit) + 1 (revisit) = 3; doc 1 freq = 1. + EXPECT_EQ(t.freqs, (std::vector {1U, 3U})); + // Positions in document order: doc 1 {10}, then doc 5's two visits in arrival + // order {50,51} then {52}. + EXPECT_EQ(t.positions_flat, (std::vector {10U, 50U, 51U, 52U})); + // doc_positions slices stay consistent with the merged freqs. + EXPECT_EQ(std::vector(t.doc_positions(1).begin(), t.doc_positions(1).end()), + (std::vector {50U, 51U, 52U})); + EXPECT_TRUE(buf.status().ok()); +} + +// Same revisit, positions disabled: freqs still sum and docids stay unique. +TEST(SniiSpimiTermBuffer, RevisitedDocidCoalescesNoPositions) { + SpimiTermBuffer buf(/*has_positions=*/false); + buf.add_token("t", 5, 0); + buf.add_token("t", 1, 0); + buf.add_token("t", 5, 0); // revisit doc 5 + + std::vector terms = buf.finalize_sorted(); + ASSERT_EQ(terms.size(), 1U); + EXPECT_EQ(terms[0].docids, (std::vector {1U, 5U})); + EXPECT_EQ(terms[0].freqs, (std::vector {1U, 2U})); + EXPECT_TRUE(terms[0].positions_flat.empty()); +} + +// The coalesced out-of-order output satisfies the writer's strictly-ascending +// docid precondition: docids are unique AND strictly increasing (the exact check +// LogicalIndexWriter::validate_term enforces). This is the contract the fix +// restores -- previously a revisited docid produced a non-ascending list. +TEST(SniiSpimiTermBuffer, RevisitedDocidProducesStrictlyAscending) { + SpimiTermBuffer buf(/*has_positions=*/true); + // A messy revisit pattern: 9,3,9,3,1. + buf.add_token("w", 9, 0); + buf.add_token("w", 3, 0); + buf.add_token("w", 9, 1); + buf.add_token("w", 3, 1); + buf.add_token("w", 1, 0); + + std::vector terms = buf.finalize_sorted(); + ASSERT_EQ(terms.size(), 1U); + const TermPostings& t = terms[0]; + ASSERT_FALSE(t.docids.empty()); + for (size_t i = 1; i < t.docids.size(); ++i) { + EXPECT_LT(t.docids[i - 1], t.docids[i]) << "docids must be strictly ascending"; + } + EXPECT_EQ(t.docids, (std::vector {1U, 3U, 9U})); + EXPECT_EQ(t.freqs, (std::vector {1U, 2U, 2U})); + // Total positions equals total tokens (sum of freqs) -- nothing dropped. + uint64_t total_freq = 0; + for (uint32_t f : t.freqs) { + total_freq += f; + } + EXPECT_EQ(t.positions_flat.size(), total_freq); +} + +// Hardening: add_token(string_view) on a BORROWED-vocab buffer is rejected (it +// would otherwise grow the owned vocab out of step with the borrowed one and +// corrupt the build). The token is ignored and an error is latched. +TEST(SniiSpimiTermBuffer, AddTokenStringViewRejectedInBorrowedMode) { + const std::vector vocab = {"a", "b"}; + SpimiTermBuffer buf(&vocab, /*has_positions=*/false); + buf.add_token(0, 0, 0); // valid id-path token + buf.add_token(std::string_view("a"), 1, 0); // illegal on a borrowed-vocab buffer + EXPECT_FALSE(buf.status().ok()); + // The string-view token was ignored: only the one id-path token counts. + EXPECT_EQ(buf.total_tokens(), 1U); + EXPECT_EQ(buf.unique_terms(), 1U); +} + +// A spill's open() I/O failure surfaces as a LATCHED error in status() (the +// streaming for_each_term_sorted swallows the failure, so callers must check +// status()). We force open() to fail deterministically by EXHAUSTING every free +// file descriptor below the soft limit, so the spill's ::open returns EMFILE. +TEST(SniiSpimiTermBuffer, SpillOpenIoFailureLatched) { + // Tiny threshold so the very first token triggers a spill_to_run(). + SpimiTermBuffer buf(/*has_positions=*/false, /*spill_threshold_bytes=*/1); + + // Cap the soft limit low so we can exhaust the fd table cheaply, then hold it. + struct rlimit saved {}; + ASSERT_EQ(getrlimit(RLIMIT_NOFILE, &saved), 0); + struct rlimit tight = saved; + tight.rlim_cur = 64; // small, but >= the few gtest/std fds already open + tight.rlim_cur = std::min(tight.rlim_cur, saved.rlim_max); + ASSERT_EQ(setrlimit(RLIMIT_NOFILE, &tight), 0); + + // Open /dev/null until the table is full: every free fd below the limit is now + // taken, so the next ::open (the spill's) cannot get one -> EMFILE. + std::vector hogs; + for (;;) { + int fd = ::open("/dev/null", O_RDONLY); + if (fd < 0) { + break; // table exhausted + } + hogs.push_back(fd); + } + ASSERT_FALSE(hogs.empty()); + + buf.add_token("z", 0, 0); // triggers a spill whose RunWriter::open must fail + + // Release the hog fds and restore the limit before asserting (so gtest I/O works). + for (int fd : hogs) { + ::close(fd); + } + ASSERT_EQ(setrlimit(RLIMIT_NOFILE, &saved), 0); + + EXPECT_FALSE(buf.status().ok()) << "spill open() failure must latch an error"; +} + +// Double-drain safety: a SECOND drain (finalize_sorted after for_each_term_sorted, +// or a second finalize_sorted) must NOT silently re-emit or emit a wrong stream; +// it returns empty / no callbacks AND latches an error. +TEST(SniiSpimiTermBuffer, DoubleDrainIsRejected) { + SpimiTermBuffer buf(/*has_positions=*/true); + buf.add_token("a", 0, 0); + buf.add_token("b", 0, 0); + + std::vector first = buf.finalize_sorted(); + ASSERT_EQ(first.size(), 2U); + EXPECT_TRUE(buf.status().ok()); + + // Second finalize_sorted: empty result + latched error. + std::vector second = buf.finalize_sorted(); + EXPECT_TRUE(second.empty()); + EXPECT_FALSE(buf.status().ok()); + + // for_each_term_sorted after a drain also emits nothing; now returns an error. + size_t seen = 0; + EXPECT_FALSE(buf.for_each_term_sorted([&](TermPostings&&) { ++seen; }).ok()); + EXPECT_EQ(seen, 0U); +} + +// Double-drain via for_each_term_sorted first, then finalize_sorted: same guard. +TEST(SniiSpimiTermBuffer, DoubleDrainStreamingThenMaterialized) { + SpimiTermBuffer buf(/*has_positions=*/false); + buf.add_token("a", 0, 0); + + size_t seen = 0; + EXPECT_TRUE(buf.for_each_term_sorted([&](TermPostings&&) { ++seen; }).ok()); + EXPECT_EQ(seen, 1U); + EXPECT_TRUE(buf.status().ok()); + + std::vector again = buf.finalize_sorted(); + EXPECT_TRUE(again.empty()); + EXPECT_FALSE(buf.status().ok()); +} + +// BYTE-IDENTICAL guard for normal ascending input: the streaming and materialized +// drains over an ASCENDING feed (the common, valid path -- NOT the out-of-order +// path the coalescing fix touches) must produce identical docids/freqs/positions. +// This asserts the fix did not perturb the normal path's produced postings. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiSpimiTermBuffer, AscendingInputByteIdenticalAcrossDrains) { + auto feed = [](SpimiTermBuffer& b) { + for (uint32_t d = 0; d < 50; ++d) { + b.add_token("apple", d, d * 2); + b.add_token("apple", d, d * 2 + 1); // freq 2 per doc + if (d % 2 == 0) { + b.add_token("banana", d, d); + } + if (d % 3 == 0) { + b.add_token("cherry", d, d + 100); + } + } + }; + SpimiTermBuffer mat(/*has_positions=*/true); + SpimiTermBuffer strm(/*has_positions=*/true); + feed(mat); + feed(strm); + + std::vector material = mat.finalize_sorted(); + std::vector streamed; + Status st = strm.for_each_term_sorted( + [&](TermPostings&& tp) { streamed.push_back(std::move(tp)); }); + EXPECT_TRUE(st.ok()); + + ASSERT_EQ(material.size(), streamed.size()); + for (size_t i = 0; i < material.size(); ++i) { + EXPECT_EQ(material[i].term, streamed[i].term); + EXPECT_EQ(material[i].docids, streamed[i].docids); + EXPECT_EQ(material[i].freqs, streamed[i].freqs); + EXPECT_EQ(material[i].positions_flat, streamed[i].positions_flat); + } + // Spot-check apple stayed exactly one entry per ascending docid (no coalescing + // path was taken for this valid feed). + ASSERT_EQ(material[0].term, "apple"); + EXPECT_EQ(material[0].docids.size(), 50U); + for (uint32_t f : material[0].freqs) { + EXPECT_EQ(f, 2U); + } + EXPECT_TRUE(mat.status().ok()); + EXPECT_TRUE(strm.status().ok()); +} + +// --------------------------------------------------------------------------- +// T17: MemoryReporter per-token zero-delta debounce. +// +// accumulate() reports its REAL resident-byte delta (posting arena + the +// vocab-sized slot index) to the writer-level MemoryReporter once per token. The +// arena grows only ~every 32 KiB block and the borrowed-vocab slot index is +// fixed-capacity, so the vast majority of tokens see delta==0. report_arena_delta() +// now SKIPS the locked fetch_add for those (debounce). These tests pin the +// deterministic op-count (report() calls == arena-growth events, never per token), +// the byte-level equivalence (current_bytes() unchanged), and the REDLINE: over_cap() +// is still evaluated UNCONDITIONALLY every token (never gated on the local delta), so +// a dict-side push over the unified cap still triggers a spill when this buffer's own +// delta is 0. A MemoryReporter built with a counting consume_release lambda exposes +// the exact per-token report() count / delta values as a deterministic seam. +// --------------------------------------------------------------------------- + +// FV-1 (deterministic op-count + functional): feeding 100 same-doc tokens issues +// exactly TWO report() calls -- one ctor delta (the resident slot index) and one for +// the first token (its 32 KiB arena block plus the G08-charged first-touch Term-slot / +// touched-list growth, a few dozen bytes) -- and NEVER a zero-delta report. Before the +// debounce, tokens 2..100 each issued report(0): 101 calls, 99 of them zero. +TEST(SniiSpimiTermBufferTest, AccumulateIssuesNoZeroDeltaReport) { + std::vector deltas; + MemoryReporter rep([&deltas](int64_t d) { deltas.push_back(d); }, /*cap_bytes=*/0); + const std::vector vocab = {"a"}; + SpimiTermBuffer buf(&vocab, /*has_positions=*/false, /*spill=*/0, &rep); + + for (int i = 0; i < 100; ++i) { + buf.add_token(0, /*docid=*/1, /*pos=*/0); // same term, same doc + } + + // 1 ctor report (slot index) + 1 first-token report (first 32 KiB arena block + + // the one-term slot-pool/touched-list first-touch growth). The other 99 same-doc + // tokens leave resident unchanged -> debounced away. + ASSERT_EQ(deltas.size(), 2U); + EXPECT_GT(deltas[0], 0); // slot index resident bytes (vocab-sized) + // One CompactPostingPool block (1 << 15) dominates; the G08 slot-pool charge for + // the single touched term adds well under 128 B on top. + EXPECT_GE(deltas[1], 32768); + EXPECT_LT(deltas[1], 32768 + 128); + for (int64_t d : deltas) { + EXPECT_NE(d, 0) << "no zero-delta report() may be issued on the hot path"; + } + EXPECT_TRUE(buf.status().ok()); +} + +// FV-2 (equivalence + count stability): the report() COUNT is independent of the +// token count (100 vs 500 same-doc tokens both issue exactly 2 reports), and the +// resulting unified total is byte-identical (resident = first arena block + slot +// index + first-touch slot-pool growth, not a function of token count). The sum +// of issued deltas equals +// current_bytes() -- the MemoryReporter self-balancing invariant the debounce +// preserves. Snapshots are taken WHILE each buffer is live (before its dtor reports +// the final balancing negative). +TEST(SniiSpimiTermBufferTest, ReportedTotalMatchesResidentRegardlessOfTokenCount) { + const std::vector vocab = {"a"}; + + std::vector d100; + std::vector d500; + int64_t cur100 = 0; + int64_t cur500 = 0; + size_t count100 = 0; + size_t count500 = 0; + + { + MemoryReporter rep([&d100](int64_t d) { d100.push_back(d); }, /*cap_bytes=*/0); + SpimiTermBuffer buf(&vocab, /*has_positions=*/false, /*spill=*/0, &rep); + for (int i = 0; i < 100; ++i) { + buf.add_token(0, /*docid=*/1, /*pos=*/0); + } + count100 = d100.size(); // snapshot before the dtor's balancing report + cur100 = rep.current_bytes(); + } + { + MemoryReporter rep([&d500](int64_t d) { d500.push_back(d); }, /*cap_bytes=*/0); + SpimiTermBuffer buf(&vocab, /*has_positions=*/false, /*spill=*/0, &rep); + for (int i = 0; i < 500; ++i) { + buf.add_token(0, /*docid=*/1, /*pos=*/0); + } + count500 = d500.size(); + cur500 = rep.current_bytes(); + } + + // Count stability: neither buffer's report count scales with token count. + EXPECT_EQ(count100, 2U); + EXPECT_EQ(count500, 2U); + // Byte-level equivalence: identical unified total despite 5x the tokens. + EXPECT_EQ(cur100, cur500); + ASSERT_GE(d100.size(), 2U); + ASSERT_GE(d500.size(), 2U); + // Both: identical first-token delta -- one 32 KiB arena block plus the same + // G08 first-touch slot-pool/touched-list growth (the slot index cancels in + // this delta; the growth is token-count-independent, hence the equality). + EXPECT_EQ(d100[1], d500[1]); + EXPECT_GE(d100[1], 32768); + EXPECT_LT(d100[1], 32768 + 128); + // Self-balancing: live current_bytes() == sum of every delta issued so far. + int64_t sum100 = 0; + for (size_t i = 0; i < count100; ++i) { + sum100 += d100[i]; + } + EXPECT_EQ(cur100, sum100); + EXPECT_EQ(cur100, d100[0] + d100[1]); // slot index + first-token resident +} + +// FV-3 (REDLINE guard): the debounce skips report() ONLY -- it must NOT gate +// over_cap(). over_cap() reads the writer-level UNIFIED total (shared with the dict +// buffer), so a dict-side allocation can push the total over the cap while THIS +// buffer's local arena delta is 0. accumulate() must still evaluate over_cap() every +// token and spill. The dict-side growth is simulated with an external rep.report(). +TEST(SniiSpimiTermBufferTest, OverCapStillFiresWhenLocalArenaDeltaIsZero) { + // Cap just above one arena block + slot index, so token1 alone does NOT spill. + MemoryReporter rep(/*consume_release=*/nullptr, /*cap_bytes=*/32768 + 1000); + const std::vector vocab = {"a"}; + SpimiTermBuffer buf(&vocab, /*has_positions=*/false, /*spill=*/0, &rep); + + // token1: arena grows to one 32 KiB block; unified total < cap -> no spill. + buf.add_token(0, /*docid=*/0, /*pos=*/0); + ASSERT_EQ(buf.run_count_for_test(), 0U); + + // Simulate dict-side growth pushing the UNIFIED total over the cap. The buffer's + // own reported_resident_ is unchanged by this external report. + rep.report(2000); + ASSERT_TRUE(rep.over_cap()); + + // token2: same doc -> this buffer's local arena delta is 0 (report() debounced), + // but over_cap() is still evaluated unconditionally and is now true -> spill. + buf.add_token(0, /*docid=*/0, /*pos=*/0); + EXPECT_EQ(buf.run_count_for_test(), 1U) + << "over_cap() must NOT be gated on the local arena delta"; + EXPECT_TRUE(buf.status().ok()); +} + +// FV-4 (boundary): an EMPTY borrowed vocab has a zero-capacity slot index, so the +// ctor's resident delta is 0 and is debounced away -- no report() is issued and +// construction does not crash. Before the debounce the ctor issued report(0). +TEST(SniiSpimiTermBufferTest, EmptyVocabReportsNoDelta) { + std::vector deltas; + MemoryReporter rep([&deltas](int64_t d) { deltas.push_back(d); }, /*cap_bytes=*/0); + const std::vector empty_vocab; + SpimiTermBuffer buf(&empty_vocab, /*has_positions=*/false, /*spill=*/0, &rep); + + // Zero-capacity slot index + empty arena -> resident 0 -> ctor delta 0 -> skipped. + EXPECT_TRUE(deltas.empty()); + EXPECT_EQ(buf.unique_terms(), 0U); + EXPECT_TRUE(buf.finalize_sorted().empty()); + EXPECT_TRUE(buf.status().ok()); +} + +// FV-5 (no-reporter path unaffected): with a null reporter, report_arena_delta() is a +// no-op and finalize_sorted() still produces the correct postings. Confirms the +// debounce change did not perturb the off-Doris path. +TEST(SniiSpimiTermBufferTest, NullReporterFinalizeIsCorrect) { + const std::vector vocab = {"a"}; + SpimiTermBuffer buf(&vocab, /*has_positions=*/false, /*spill=*/0, /*reporter=*/nullptr); + + for (int i = 0; i < 100; ++i) { + buf.add_token(0, /*docid=*/1, /*pos=*/0); // same term, same doc + } + + std::vector terms = buf.finalize_sorted(); + ASSERT_EQ(terms.size(), 1U); + EXPECT_EQ(terms[0].term, "a"); + EXPECT_EQ(terms[0].docids, (std::vector {1U})); + EXPECT_EQ(terms[0].freqs, (std::vector {100U})); + EXPECT_TRUE(terms[0].positions_flat.empty()); + EXPECT_TRUE(buf.status().ok()); +} + +// FV-6 (spill/drain negative path + self-balance): a forced spill emits NONZERO +// negative deltas (arena reset, then slot-index free), never a zero. After a full +// drain the reporter's unified total returns to 0 -- the debounce preserves the +// self-balancing invariant (no leaked positive). The merged postings are correct. +TEST(SniiSpimiTermBufferTest, SpillNegativeDeltasHaveNoZeroAndBalance) { + std::vector deltas; + // Cap below one arena block, so the first token's block forces a spill. + MemoryReporter rep([&deltas](int64_t d) { deltas.push_back(d); }, /*cap_bytes=*/1000); + const std::vector vocab = {"a"}; + SpimiTermBuffer buf(&vocab, /*has_positions=*/false, /*spill=*/0, &rep); + + buf.add_token(0, /*docid=*/0, /*pos=*/0); // grows one block, then spills + ASSERT_EQ(buf.run_count_for_test(), 1U) << "the over-cap token must spill"; + + std::vector terms = buf.finalize_sorted(); + ASSERT_EQ(terms.size(), 1U); + EXPECT_EQ(terms[0].term, "a"); + EXPECT_EQ(terms[0].docids, (std::vector {0U})); + EXPECT_EQ(terms[0].freqs, (std::vector {1U})); + EXPECT_TRUE(buf.status().ok()); + + // No zero-delta report on any path (grow, spill-negative, or merge-free). + for (int64_t d : deltas) { + EXPECT_NE(d, 0) << "spill/drain deltas must all be nonzero"; + } + // Self-balancing: after a full drain every reported byte has been returned. + EXPECT_EQ(rep.current_bytes(), 0); +} diff --git a/be/test/storage/index/snii/writer/temp_dir_test.cpp b/be/test/storage/index/snii/writer/temp_dir_test.cpp new file mode 100644 index 00000000000000..5c0d42e1f332e2 --- /dev/null +++ b/be/test/storage/index/snii/writer/temp_dir_test.cpp @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/writer/temp_dir.h" + +#include + +#include +#include + +namespace doris::snii::writer { +namespace { + +// ExecEnv's tmp_file_dirs are initialized by the global SniiTmpDirEnvironment +// (be/test/storage/index/snii/snii_test_env.cpp). +TEST(SniiTempDir, ResolvesIntoSniiSubdirOfConfiguredTmpDir) { + const std::string dir = resolve_temp_dir(); + ASSERT_FALSE(dir.empty()); + // SNII gets its own "snii" subdirectory so it does not crowd the tmp root. + ASSERT_GE(dir.size(), 5U); + EXPECT_EQ(0, dir.compare(dir.size() - 5, 5, "/snii")) << dir; + // resolve_temp_dir() creates the directory, so it stats successfully. + EXPECT_NE(temp_dir_available_bytes(dir), UINT64_MAX) << dir; +} + +TEST(SniiTempDir, AvailableBytesStatsRealDirAndSentinelsOnFailure) { + EXPECT_NE(temp_dir_available_bytes("/tmp"), UINT64_MAX); // real dir -> some free + EXPECT_EQ(temp_dir_available_bytes("/snii_no_such_dir_xyzzy"), UINT64_MAX); +} + +} // namespace +} // namespace doris::snii::writer diff --git a/be/test/storage/index/snii_b2_or_read_test.cpp b/be/test/storage/index/snii_b2_or_read_test.cpp new file mode 100644 index 00000000000000..5d2337d0a2fdcb --- /dev/null +++ b/be/test/storage/index/snii_b2_or_read_test.cpp @@ -0,0 +1,347 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// SNII Batch 2 -- multi-term OR read-amplification + streaming docid-union (T09). +// +// Covers three reader-only, byte-identical changes: +// (1) emit_docid_union streams each posting into a dedup-capable sink (Roaring) +// over ONE shared fetch round -- dense-full windows stay runs via +// append_range -- instead of materializing a per-term vector + K-way merge. +// (2) union_sorted_many reserves by summed input size (single allocation), +// capped by reserve_cap so heavily-overlapping inputs do not over-reserve. +// (3) the OR resolve path threads one request-scoped DictBlockCache through its +// per-term lookups, so terms sharing a DICT block read+decode it once. +// +// All assertions are deterministic (op-counts, capacities, set equality, I/O round +// counts through MeteredFileReader / MemoryFile). No wall-clock gates. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/per_index_meta.h" +#include "storage/index/snii/io/metered_file_reader.h" +#include "storage/index/snii/query/boolean_query.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/internal/docid_set_ops.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii_query_test_util.h" + +using namespace doris::snii; +using namespace doris::snii::snii_test; +using doris::Status; + +namespace qi = doris::snii::query::internal; + +namespace { + +// A dedup-capable sink (dedups()==true) that records how each posting was handed +// off: append_range for run-preserving dense windows, append_sorted otherwise. The +// collected docids land in a std::set so equality checks are order-independent +// (a real Roaring sink also dedups/orders internally). +class CountingDedupSink final : public query::DocIdSink { +public: + Status append_sorted(std::span docids) override { + ++sorted_calls; + for (uint32_t docid : docids) { + ids.insert(docid); + } + return Status::OK(); + } + + Status append_range(uint32_t first, uint64_t last_exclusive) override { + ++range_calls; + for (uint64_t docid = first; docid < last_exclusive; ++docid) { + ids.insert(static_cast(docid)); + } + return Status::OK(); + } + + bool dedups() const override { return true; } + + std::set ids; + size_t sorted_calls = 0; + size_t range_calls = 0; +}; + +std::vector closed_range(uint32_t begin, uint32_t end_exclusive) { + std::vector out; + out.reserve(end_exclusive - begin); + for (uint32_t docid = begin; docid < end_exclusive; ++docid) { + out.push_back(docid); + } + return out; +} + +// Opens the standard 9000-doc fixture over a MeteredFileReader so I/O rounds and +// remote GETs can be measured. build_reader() writes the index bytes into `file`; +// a fresh segment/index is then re-opened over a metered wrapper of those bytes. +struct MeteredIndex { + MemoryFile file; + std::unique_ptr metered; + reader::SniiSegmentReader segment; + reader::LogicalIndexReader idx; +}; + +void OpenMeteredFixture(MeteredIndex* fx, size_t block_size) { + reader::SniiSegmentReader seg0; + reader::LogicalIndexReader idx0; + assert_ok(build_reader(&fx->file, &seg0, &idx0)); + fx->metered = std::make_unique(&fx->file, block_size); + assert_ok(reader::SniiSegmentReader::open(fx->metered.get(), &fx->segment)); + assert_ok(fx->segment.open_index(7, "Body", &fx->idx)); +} + +} // namespace + +// --------------------------------------------------------------------------- +// T09 F16 -- union_sorted_many reserves by total, capped by reserve_cap. +// --------------------------------------------------------------------------- + +TEST(SniiB2OrRead, UnionReservesByTotalForDisjointLists) { + // Three disjoint sorted lists (sum == 12); union == sum. + const std::vector> lists = {{0, 3, 6, 9}, {1, 4, 7, 10}, {2, 5, 8, 11}}; + const size_t total = 12; + + const std::vector got = qi::union_sorted_many(lists); + + EXPECT_EQ(got.size(), total); + // reserve(total) + exactly `total` pushes -> a single allocation, so capacity is + // exactly total (was geometric growth off reserve(largest)). + EXPECT_EQ(got.capacity(), total); + EXPECT_TRUE(std::ranges::is_sorted(got)); +} + +TEST(SniiB2OrRead, UnionRespectsReserveCapOnHeavyOverlap) { + // >8 identical lists -> heap path; total == 40 but the union is only 4 elements. + const std::vector> lists(10, std::vector {1, 2, 3, 4}); + const size_t union_size = 4; + + const std::vector got = qi::union_sorted_many(lists, /*reserve_cap=*/union_size); + + EXPECT_EQ(got.size(), union_size); + // Capped at reserve_cap rather than over-reserving 10x (= 40) on overlap. + EXPECT_EQ(got.capacity(), union_size); + EXPECT_EQ(got, (std::vector {1, 2, 3, 4})); +} + +TEST(SniiB2OrRead, UnionContentUnchangedByReserveFix) { + const std::vector> lists = {{0, 2, 4, 6, 8}, {1, 3, 5}, {4, 5, 6, 7}}; + std::set expected_set; + for (const std::vector& list : lists) { + expected_set.insert(list.begin(), list.end()); + } + const std::vector want(expected_set.begin(), expected_set.end()); + + EXPECT_EQ(qi::union_sorted_many(lists), want); +} + +// --------------------------------------------------------------------------- +// T09 F15 -- dedups() capability gate + streaming OR. +// --------------------------------------------------------------------------- + +TEST(SniiB2OrRead, DedupCapabilityGate) { + std::vector backing; + query::VectorDocIdSink vector_sink(backing); + EXPECT_FALSE(vector_sink.dedups()) << "plain vector sink needs materialize+merge"; + + CountingDedupSink dedup_sink; + EXPECT_TRUE(dedup_sink.dedups()) << "Roaring-style sink dedups/orders natively"; +} + +TEST(SniiB2OrRead, MultiTermOrPreservesDenseRangeToDedupSink) { + MemoryFile file; + reader::SniiSegmentReader segment; + reader::LogicalIndexReader idx; + assert_ok(build_reader(&file, &segment, &idx)); + + CountingDedupSink sink; + assert_ok(query::boolean_or(idx, {"failed", "sparse_left"}, &sink)); + + // "failed" is a dense full posting (docids 0..8999): its dense-full windows must + // stream in via append_range (run-preserving), not be expanded element-by-element + // through a merge accumulator -- the old path issued only append_sorted(acc). + EXPECT_GE(sink.range_calls, 1u); + + // failed covers every doc, so the union is the full doc range. + const std::vector got(sink.ids.begin(), sink.ids.end()); + EXPECT_EQ(got, closed_range(0, 9000)); +} + +TEST(SniiB2OrRead, MultiTermOrStreamingMatchesMergePath) { + MemoryFile file; + reader::SniiSegmentReader segment; + reader::LogicalIndexReader idx; + assert_ok(build_reader(&file, &segment, &idx)); + + const std::vector terms = {"needle", "sparse_left", "driver"}; + + // Streaming path: dedup-capable sink -> emit_docid_postings_streamed. + CountingDedupSink sink; + assert_ok(query::boolean_or(idx, terms, &sink)); + const std::vector streamed(sink.ids.begin(), sink.ids.end()); + + // Merge path: vector out -> build_docid_union + union_sorted_many. + std::vector merged; + assert_ok(query::boolean_or(idx, terms, &merged)); + + EXPECT_TRUE(std::ranges::is_sorted(merged)); + EXPECT_FALSE(merged.empty()); + EXPECT_EQ(streamed, merged) << "streaming OR must yield the identical docid set"; +} + +TEST(SniiB2OrRead, NonDedupSinkFallsBackToMergeContract) { + MemoryFile file; + reader::SniiSegmentReader segment; + reader::LogicalIndexReader idx; + assert_ok(build_reader(&file, &segment, &idx)); + + const std::vector terms = {"needle", "sparse_left"}; + + // A non-dedup sink (VectorDocIdSink) routed through the sink overload must keep + // the merge path so its single globally-sorted-deduplicated span contract holds. + std::vector via_sink; + query::VectorDocIdSink vector_sink(via_sink); + assert_ok(query::boolean_or(idx, terms, &vector_sink)); + + std::vector via_vector; + assert_ok(query::boolean_or(idx, terms, &via_vector)); + + EXPECT_TRUE(std::ranges::is_sorted(via_sink)); + EXPECT_EQ(via_sink, via_vector); +} + +TEST(SniiB2OrRead, SingleTermAndBoundaryInputs) { + MemoryFile file; + reader::SniiSegmentReader segment; + reader::LogicalIndexReader idx; + assert_ok(build_reader(&file, &segment, &idx)); + + // Single resolved term -> exact known docids. + CountingDedupSink single; + assert_ok(query::boolean_or(idx, {"needle"}, &single)); + EXPECT_EQ((std::vector(single.ids.begin(), single.ids.end())), + (std::vector {100, 101, 102, 6000})); + + // Empty terms -> OK, sink untouched. + CountingDedupSink empty; + assert_ok(query::boolean_or(idx, std::vector {}, &empty)); + EXPECT_TRUE(empty.ids.empty()); + + // All-miss term -> OK, sink untouched (resolve skips absent terms). + CountingDedupSink miss; + assert_ok(query::boolean_or(idx, {"zzz_absent_term"}, &miss)); + EXPECT_TRUE(miss.ids.empty()); + + // Null sink -> InvalidArgument. + query::DocIdSink* null_sink = nullptr; + const Status st = query::boolean_or(idx, {"failed"}, null_sink); + EXPECT_FALSE(st.ok()); + EXPECT_TRUE(st.is()) << st.to_string(); +} + +// --------------------------------------------------------------------------- +// Read amplification -- one shared fetch round for the multi-term OR postings. +// --------------------------------------------------------------------------- + +TEST(SniiB2OrRead, MultiTermOrIssuesSingleSerialRound) { + MeteredIndex fx; + OpenMeteredFixture(&fx, /*block_size=*/256); + const std::vector terms = {"failed", "driver", "sparse_left"}; + + // Per-term baseline: each term resolved + read on its own -> its own fetch round. + fx.metered->reset_metrics(); + for (const std::string& term : terms) { + std::vector docs; + assert_ok(query::term_query(fx.idx, term, &docs)); + } + const io::IoMetrics per_term = fx.metered->metrics(); + + // Batched OR: one shared fetch round for all postings' docid reads. + fx.metered->reset_metrics(); + std::vector got; + assert_ok(query::boolean_or(fx.idx, terms, &got)); + const io::IoMetrics batched = fx.metered->metrics(); + + EXPECT_EQ(batched.serial_rounds, 1u) << "multi-term OR must read all postings in one round"; + EXPECT_GE(per_term.serial_rounds, 2u); + EXPECT_GT(per_term.serial_rounds, batched.serial_rounds); + // Coalescing never increases physical GETs or bytes vs the per-term path. + EXPECT_LE(batched.range_gets, per_term.range_gets); + EXPECT_LE(batched.remote_bytes, per_term.remote_bytes); + + // Result set is unchanged: failed(0..8999) covers driver(0..7999) and sparse_left. + EXPECT_EQ(got, closed_range(0, 9000)); +} + +TEST(SniiB2OrRead, MultiTermOrDedupsSharedDictBlockReads) { + // Force on-demand DICT blocks so each lookup reads its block from the file + // (resident DICT blocks would serve lookups from memory and hide the effect). + ScopedEnv on_demand("SNII_DICT_RESIDENT_MAX", "0"); + + MemoryFile file; + reader::SniiSegmentReader segment; + reader::LogicalIndexReader idx; + assert_ok(build_reader(&file, &segment, &idx)); + + const format::RegionRef dict = idx.section_refs().dict_region; + const auto dict_reads = [&]() { + size_t count = 0; + for (const MemoryFile::Read& r : file.reads()) { + if (r.offset >= dict.offset && r.offset < dict.offset + dict.length) { + ++count; + } + } + return count; + }; + + const std::vector terms = {"123", "almost", "driver", "failed", + "needle", "order", "ordinal", "repeat", + "sparse_left", "sparse_right", "trace"}; + + // Per-term baseline: term_query resolves each DICT block on its own (cache=null) + // -> one DICT-region read per term. + file.clear_reads(); + for (const std::string& term : terms) { + std::vector docs; + assert_ok(query::term_query(idx, term, &docs)); + } + const size_t per_term_dict_reads = dict_reads(); + + // Multi-term OR threads one request-scoped DictBlockCache through resolve, so each + // unique DICT block is read + decoded once for the whole OR. + file.clear_reads(); + std::vector got; + assert_ok(query::boolean_or(idx, terms, &got)); + const size_t or_dict_reads = dict_reads(); + + EXPECT_EQ(per_term_dict_reads, terms.size()); + EXPECT_GE(or_dict_reads, 1u); + EXPECT_LT(or_dict_reads, per_term_dict_reads) + << "OR must not re-read a DICT block already decoded for another term"; + EXPECT_FALSE(got.empty()); +} diff --git a/be/test/storage/index/snii_b2_t07_dict_test.cpp b/be/test/storage/index/snii_b2_t07_dict_test.cpp new file mode 100644 index 00000000000000..03cbd88de59db5 --- /dev/null +++ b/be/test/storage/index/snii_b2_t07_dict_test.cpp @@ -0,0 +1,634 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// T07 -- DICT entry key-first decode (exact find_term + prefix streaming +// early-stop). These tests assert three things the optimization promises: +// 1. byte-identical decode: decode_dict_entry == decode_dict_entry_key + +// decode_dict_entry_rest, and visit_prefix_range streams the same entries +// decode_all materializes; +// 2. body-decode op-count drops: find_term materializes 1 (hit) / 0 (miss) +// bodies instead of every scanned entry; visit_prefix_range materializes +// only the accepted/emitted bodies (anchor-jump + early-stop); +// 3. results unchanged at the reader/query layer (prefix_terms / prefix_query +// route through the rewired streaming path). + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/format/dict_block.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii_query_test_util.h" + +namespace doris::snii { +namespace { + +using namespace doris::snii::format; // NOLINT(google-build-using-namespace) +using namespace doris::snii::snii_test; // NOLINT(google-build-using-namespace) +using doris::Status; + +// ---- entry builders (valid, self-consistent locators so they round-trip) ---- + +DictEntry MakePodRef(std::string term, uint32_t df, uint64_t frq_off, uint64_t prx_off = 0) { + DictEntry e; + e.term = std::move(term); + e.kind = DictEntryKind::kPodRef; + e.enc = DictEntryEnc::kSlim; + e.df = df; + e.ttf_delta = df * 2; + e.max_freq = 9; + e.frq_off_delta = frq_off; + e.frq_len = 128; + e.frq_docs_len = 64; // dd region on-disk length (<= frq_len) + e.dd_meta.uncomp_len = 70; + e.dd_meta.crc = 0xABCD1234U; // pod_ref regions keep their per-region crc + e.freq_meta.uncomp_len = 40; + e.freq_meta.crc = 0x55AA00FFU; + e.prx_off_delta = prx_off; + e.prx_len = 64; + return e; +} + +DictEntry MakePodRefWindowed(std::string term, uint32_t df, uint64_t frq_off) { + DictEntry e; + e.term = std::move(term); + e.kind = DictEntryKind::kPodRef; + e.enc = DictEntryEnc::kWindowed; + e.df = df; + e.ttf_delta = df * 2; + e.max_freq = 5; + e.frq_off_delta = frq_off; + e.frq_len = 200; + e.prelude_len = 10; // 0 < prelude_len <= frq_docs_len + e.frq_docs_len = 120; // <= frq_len + e.prx_off_delta = 0; + e.prx_len = 50; + return e; +} + +DictEntry MakeInline(std::string term, uint32_t df) { + DictEntry e; + e.term = std::move(term); + e.kind = DictEntryKind::kInline; + e.enc = DictEntryEnc::kSlim; + e.df = df; + e.ttf_delta = df * 3; + e.max_freq = 7; + e.frq_bytes = {0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80}; + e.inline_dd_disk_len = 5; // <= frq_bytes.size() + e.dd_meta.uncomp_len = 12; + e.freq_meta.uncomp_len = 6; + e.prx_bytes = {0xAA, 0xBB, 0xCC}; + return e; +} + +// Full field comparison of two DECODED entries (used to prove the key-first path +// produces byte-identical output to the full / decode_all path). +void ExpectEntryEq(const DictEntry& a, const DictEntry& b) { + EXPECT_EQ(a.term, b.term); + EXPECT_EQ(a.kind, b.kind); + EXPECT_EQ(a.enc, b.enc); + EXPECT_EQ(a.has_sb, b.has_sb); + EXPECT_EQ(a.df, b.df); + EXPECT_EQ(a.ttf_delta, b.ttf_delta); + EXPECT_EQ(a.max_freq, b.max_freq); + EXPECT_EQ(a.frq_off_delta, b.frq_off_delta); + EXPECT_EQ(a.frq_len, b.frq_len); + EXPECT_EQ(a.prelude_len, b.prelude_len); + EXPECT_EQ(a.frq_docs_len, b.frq_docs_len); + EXPECT_EQ(a.prx_off_delta, b.prx_off_delta); + EXPECT_EQ(a.prx_len, b.prx_len); + EXPECT_EQ(a.inline_dd_disk_len, b.inline_dd_disk_len); + EXPECT_EQ(a.frq_bytes, b.frq_bytes); + EXPECT_EQ(a.prx_bytes, b.prx_bytes); + EXPECT_EQ(a.dd_meta.zstd, b.dd_meta.zstd); + EXPECT_EQ(a.dd_meta.uncomp_len, b.dd_meta.uncomp_len); + EXPECT_EQ(a.dd_meta.disk_len, b.dd_meta.disk_len); + EXPECT_EQ(a.dd_meta.crc, b.dd_meta.crc); + EXPECT_EQ(a.dd_meta.verify_crc, b.dd_meta.verify_crc); + EXPECT_EQ(a.freq_meta.zstd, b.freq_meta.zstd); + EXPECT_EQ(a.freq_meta.uncomp_len, b.freq_meta.uncomp_len); + EXPECT_EQ(a.freq_meta.disk_len, b.freq_meta.disk_len); + EXPECT_EQ(a.freq_meta.crc, b.freq_meta.crc); + EXPECT_EQ(a.freq_meta.verify_crc, b.freq_meta.verify_crc); +} + +std::vector BuildBlock(const std::vector& entries, IndexTier tier, + bool has_positions, uint64_t frq_base, uint64_t prx_base, + uint32_t anchor_interval) { + DictBlockBuilder builder(tier, has_positions, frq_base, prx_base, anchor_interval); + for (const auto& e : entries) { + builder.add_entry(e); + } + ByteSink sink; + builder.finish(&sink); + return sink.buffer(); +} + +// A no-op key predicate (accept all); passing an empty std::function exercises +// the accept-all short-circuit in visit_prefix_range. +const std::function kAcceptAll {}; + +// Collects the streamed entries' terms; never stops early. +std::function collect_terms(std::vector* out) { + return [out](DictEntry&& e, bool* stop) -> Status { + out->push_back(e.term); + *stop = false; + return Status::OK(); + }; +} + +// ---- F-1: key/rest split and decode_all are byte-identical ---- + +// Entry level: decode_dict_entry (full) vs decode_dict_entry_key + _rest, over a +// front-coded sequence mixing pod_ref(slim/windowed) and inline entries. +TEST(SniiB2T07Dict, KeyRestSplitDecodesByteIdenticalToFull) { + const IndexTier tier = IndexTier::kT2; + const std::vector entries = { + MakePodRef("alpha", 3, 0, 0), + MakeInline("alphabet", 5), + MakePodRefWindowed("beta", 7, 100), + MakePodRef("betas", 9, 300, 80), + MakeInline("gamma", 2), + }; + + ByteSink sink; + std::string prev; + for (const auto& e : entries) { + assert_ok(encode_dict_entry(e, prev, tier, &sink)); + prev = e.term; + } + const std::vector buf = sink.buffer(); + + // Baseline: full decode. + std::vector full; + { + ByteSource src {Slice(buf)}; + std::string p; + for (size_t i = 0; i < entries.size(); ++i) { + DictEntry e; + assert_ok(decode_dict_entry(&src, p, tier, &e)); + p = e.term; + full.push_back(std::move(e)); + } + EXPECT_TRUE(src.eof()); + } + + // key + rest decode: byte-identical fields, exactly one body decode per entry. + reset_dict_entry_counters(); + std::vector split; + { + ByteSource src {Slice(buf)}; + std::string p; + for (size_t i = 0; i < entries.size(); ++i) { + DictEntry e; + size_t body_start = 0; + uint64_t total = 0; + assert_ok(decode_dict_entry_key(&src, p, &e, &body_start, &total)); + assert_ok(decode_dict_entry_rest(&src, tier, body_start, total, &e)); + p = e.term; + split.push_back(std::move(e)); + } + EXPECT_TRUE(src.eof()); + } + EXPECT_EQ(dict_entry_body_decode_count(), entries.size()); + + ASSERT_EQ(full.size(), split.size()); + for (size_t i = 0; i < full.size(); ++i) { + ExpectEntryEq(full[i], split[i]); + } +} + +// skip_dict_entry_body advances across entries (front coding still rebuilds each +// term) without materializing any body. +TEST(SniiB2T07Dict, SkipDictEntryBodyAdvancesAndCountsNoBody) { + const IndexTier tier = IndexTier::kT2; + const std::vector entries = {MakePodRef("aa", 1, 0, 0), MakeInline("ab", 2), + MakePodRefWindowed("ac", 3, 50)}; + ByteSink sink; + std::string prev; + for (const auto& e : entries) { + assert_ok(encode_dict_entry(e, prev, tier, &sink)); + prev = e.term; + } + const std::vector buf = sink.buffer(); + + reset_dict_entry_counters(); + ByteSource src {Slice(buf)}; + std::string p; + std::vector terms; + for (size_t i = 0; i < entries.size(); ++i) { + DictEntry e; + size_t body_start = 0; + uint64_t total = 0; + assert_ok(decode_dict_entry_key(&src, p, &e, &body_start, &total)); + terms.push_back(e.term); + assert_ok(skip_dict_entry_body(&src, body_start, total)); + p = e.term; + } + EXPECT_TRUE(src.eof()); + EXPECT_EQ(dict_entry_body_decode_count(), 0U); // keys only + ASSERT_EQ(terms.size(), 3U); + EXPECT_EQ(terms[0], "aa"); + EXPECT_EQ(terms[1], "ab"); + EXPECT_EQ(terms[2], "ac"); +} + +// Block level: visit_prefix_range("") streams every entry, byte-identical to +// decode_all, across multiple anchor segments and mixed entry kinds. +TEST(SniiB2T07Dict, VisitPrefixRangeEmptyPrefixMatchesDecodeAll) { + std::vector entries; + for (int i = 0; i < 40; ++i) { + char buf[16]; + std::snprintf(buf, sizeof(buf), "term_%02d", i); + if (i % 3 == 0) { + entries.push_back(MakeInline(buf, static_cast(i + 1))); + } else if (i % 3 == 1) { + entries.push_back(MakePodRef(buf, static_cast(i + 1), + static_cast(i) * 10, + static_cast(i) * 4)); + } else { + entries.push_back(MakePodRefWindowed(buf, static_cast(i + 1), + static_cast(i) * 10)); + } + } + const std::vector bytes = BuildBlock(entries, IndexTier::kT2, true, 8192, 16384, 16); + DictBlockReader reader; + assert_ok(DictBlockReader::open(Slice(bytes), IndexTier::kT2, true, &reader)); + + std::vector golden; + assert_ok(reader.decode_all(&golden)); + + std::vector streamed; + bool exhausted = false; + assert_ok(reader.visit_prefix_range( + "", kAcceptAll, + [&](DictEntry&& e, bool* stop) -> Status { + streamed.push_back(std::move(e)); + *stop = false; + return Status::OK(); + }, + &exhausted)); + EXPECT_FALSE(exhausted); // the empty prefix never leaves the range + ASSERT_EQ(streamed.size(), golden.size()); + for (size_t i = 0; i < golden.size(); ++i) { + ExpectEntryEq(golden[i], streamed[i]); + } +} + +// ---- F-2 / F-3: find_term materializes only the matched body ---- + +TEST(SniiB2T07Dict, FindTermDecodesOnlyMatchedBody) { + std::vector entries; + for (int i = 0; i < 16; ++i) { + char buf[16]; + std::snprintf(buf, sizeof(buf), "k%02d", i); + entries.push_back( + MakePodRef(buf, static_cast(i + 1), static_cast(i) * 8)); + } + const std::vector bytes = + BuildBlock(entries, IndexTier::kT1, false, 0, 0, 16); // 1 anchor + DictBlockReader reader; + assert_ok(DictBlockReader::open(Slice(bytes), IndexTier::kT1, false, &reader)); + + // First / middle / last entry: each materializes exactly one body even though + // the scan walks up to 16 keys. + for (const char* t : {"k00", "k07", "k15"}) { + reset_dict_entry_counters(); + bool found = false; + DictEntry out; + assert_ok(reader.find_term(t, &found, &out)); + EXPECT_TRUE(found) << t; + EXPECT_EQ(out.term, t) << t; + EXPECT_EQ(dict_entry_body_decode_count(), 1U) << t; + } +} + +TEST(SniiB2T07Dict, FindTermMissDecodesNoBody) { + std::vector entries; // k00, k02, ... k30 -> gaps between keys + for (int i = 0; i < 32; i += 2) { + char buf[16]; + std::snprintf(buf, sizeof(buf), "k%02d", i); + entries.push_back(MakePodRef(buf, 1, static_cast(i) * 4)); + } + ASSERT_EQ(entries.size(), 16U); + const std::vector bytes = BuildBlock(entries, IndexTier::kT1, false, 0, 0, 16); + DictBlockReader reader; + assert_ok(DictBlockReader::open(Slice(bytes), IndexTier::kT1, false, &reader)); + + bool found = true; + DictEntry out; + + // Gap inside the range: stop at the first key past target, no body decode. + reset_dict_entry_counters(); + assert_ok(reader.find_term("k15", &found, &out)); + EXPECT_FALSE(found); + EXPECT_EQ(dict_entry_body_decode_count(), 0U); + + // Below the first anchor term: rejected by the anchor search, no scan at all. + reset_dict_entry_counters(); + found = true; + assert_ok(reader.find_term("a", &found, &out)); + EXPECT_FALSE(found); + EXPECT_EQ(dict_entry_body_decode_count(), 0U); + + // Above the last term: scan keys to eof, still no body decode. + reset_dict_entry_counters(); + found = true; + assert_ok(reader.find_term("z", &found, &out)); + EXPECT_FALSE(found); + EXPECT_EQ(dict_entry_body_decode_count(), 0U); +} + +// find_term's matched body is byte-identical to the decode_all entry, across +// anchor boundaries (mixed kinds). +TEST(SniiB2T07Dict, FindTermReturnsEntryMatchingDecodeAll) { + std::vector entries; + for (int i = 0; i < 40; ++i) { + char buf[16]; + std::snprintf(buf, sizeof(buf), "term_%02d", i); + if (i % 2 == 0) { + entries.push_back(MakeInline(buf, static_cast(i + 1))); + } else { + entries.push_back(MakePodRef(buf, static_cast(i + 1), + static_cast(i) * 10, + static_cast(i) * 4)); + } + } + const std::vector bytes = BuildBlock(entries, IndexTier::kT2, true, 4096, 8192, 16); + DictBlockReader reader; + assert_ok(DictBlockReader::open(Slice(bytes), IndexTier::kT2, true, &reader)); + + std::vector golden; + assert_ok(reader.decode_all(&golden)); + for (const auto& g : golden) { + bool found = false; + DictEntry out; + assert_ok(reader.find_term(g.term, &found, &out)); + ASSERT_TRUE(found) << g.term; + ExpectEntryEq(g, out); + } +} + +// ---- F-6 / anchor-jump / accept_key / early-stop on visit_prefix_range ---- + +TEST(SniiB2T07Dict, VisitPrefixRangeStopsAtBoundaryDecodingOnlyMatches) { + const std::vector entries = { + MakePodRef("aa1", 1, 0), MakePodRef("aa2", 1, 10), MakePodRef("ab1", 1, 20), + MakeInline("ab2", 2), MakePodRef("ab3", 1, 40), MakePodRef("ac1", 1, 50), + MakePodRef("ac2", 1, 60), + }; + const std::vector bytes = BuildBlock(entries, IndexTier::kT1, false, 0, 0, 16); + DictBlockReader reader; + assert_ok(DictBlockReader::open(Slice(bytes), IndexTier::kT1, false, &reader)); + + reset_dict_entry_counters(); + std::vector got; + bool exhausted = false; + assert_ok(reader.visit_prefix_range("ab", kAcceptAll, collect_terms(&got), &exhausted)); + EXPECT_TRUE(exhausted); // saw "ac1" past the range + ASSERT_EQ(got.size(), 3U); + EXPECT_EQ(got[0], "ab1"); + EXPECT_EQ(got[1], "ab2"); + EXPECT_EQ(got[2], "ab3"); + // aa* skipped (key-only), ac* never reached -> exactly the 3 ab* bodies. + EXPECT_EQ(dict_entry_body_decode_count(), 3U); +} + +TEST(SniiB2T07Dict, VisitPrefixRangeAnchorJumpSkipsPrePrefixSegments) { + std::vector entries; + for (int i = 0; i < 32; ++i) { + char buf[16]; + std::snprintf(buf, sizeof(buf), "aa%03d", i); + entries.push_back(MakePodRef(buf, 1, static_cast(i))); + } + for (int i = 0; i < 4; ++i) { + char buf[16]; + std::snprintf(buf, sizeof(buf), "zz%03d", i); + entries.push_back(MakePodRef(buf, 1, 1000 + static_cast(i))); + } + // anchor_interval=8 -> several whole "aa*" anchor segments are jumped, never + // decoded; the segment straddling the prefix is scanned key-only. + const std::vector bytes = BuildBlock(entries, IndexTier::kT1, false, 0, 0, 8); + DictBlockReader reader; + assert_ok(DictBlockReader::open(Slice(bytes), IndexTier::kT1, false, &reader)); + + reset_dict_entry_counters(); + std::vector got; + bool exhausted = false; + assert_ok(reader.visit_prefix_range("zz", kAcceptAll, collect_terms(&got), &exhausted)); + ASSERT_EQ(got.size(), 4U); + EXPECT_EQ(got[0], "zz000"); + EXPECT_EQ(got[3], "zz003"); + EXPECT_FALSE(exhausted); // zz003 is the last entry -> block ended in range + EXPECT_EQ(dict_entry_body_decode_count(), 4U); // only the zz* bodies +} + +TEST(SniiB2T07Dict, VisitPrefixRangeAcceptKeySkipsRejectedBodies) { + std::vector entries; + for (int i = 0; i < 10; ++i) { + char buf[16]; + std::snprintf(buf, sizeof(buf), "ab%d", i); + entries.push_back(MakePodRef(buf, 1, static_cast(i) * 5)); + } + const std::vector bytes = BuildBlock(entries, IndexTier::kT1, false, 0, 0, 16); + DictBlockReader reader; + assert_ok(DictBlockReader::open(Slice(bytes), IndexTier::kT1, false, &reader)); + + reset_dict_entry_counters(); + std::vector got; + bool exhausted = false; + const std::function accept_even = [](std::string_view t) { + return ((t.back() - '0') % 2) == 0; + }; + assert_ok(reader.visit_prefix_range("ab", accept_even, collect_terms(&got), &exhausted)); + ASSERT_EQ(got.size(), 5U); // ab0, ab2, ab4, ab6, ab8 + EXPECT_EQ(got[0], "ab0"); + EXPECT_EQ(got[4], "ab8"); + // Rejected keys skip their body: 5 bodies, not 10. + EXPECT_EQ(dict_entry_body_decode_count(), 5U); +} + +TEST(SniiB2T07Dict, VisitPrefixRangeEarlyStopAfterK) { + std::vector entries; + for (int i = 0; i < 10; ++i) { + char buf[16]; + std::snprintf(buf, sizeof(buf), "ab%d", i); + entries.push_back(MakePodRef(buf, 1, static_cast(i) * 5)); + } + const std::vector bytes = BuildBlock(entries, IndexTier::kT1, false, 0, 0, 16); + DictBlockReader reader; + assert_ok(DictBlockReader::open(Slice(bytes), IndexTier::kT1, false, &reader)); + + reset_dict_entry_counters(); + std::vector got; + bool exhausted = false; + int n = 0; + assert_ok(reader.visit_prefix_range( + "ab", kAcceptAll, + [&](DictEntry&& e, bool* stop) -> Status { + got.push_back(e.term); + ++n; + *stop = (n >= 3); + return Status::OK(); + }, + &exhausted)); + ASSERT_EQ(got.size(), 3U); + EXPECT_FALSE(exhausted); // stopped by the visitor, not a boundary + EXPECT_EQ(dict_entry_body_decode_count(), 3U); // no bodies past the stop +} + +// ---- F-5 / F-7: single-entry block + corrupt entry_len ---- + +TEST(SniiB2T07Dict, SingleEntryBlockVisitAndFind) { + const std::vector entries = {MakePodRef("solo", 7, 0)}; + const std::vector bytes = BuildBlock(entries, IndexTier::kT1, false, 4096, 0, 16); + DictBlockReader reader; + assert_ok(DictBlockReader::open(Slice(bytes), IndexTier::kT1, false, &reader)); + + reset_dict_entry_counters(); + bool found = false; + DictEntry out; + assert_ok(reader.find_term("solo", &found, &out)); + EXPECT_TRUE(found); + EXPECT_EQ(out.term, "solo"); + EXPECT_EQ(dict_entry_body_decode_count(), 1U); + + reset_dict_entry_counters(); + std::vector got; + bool exhausted = false; + assert_ok(reader.visit_prefix_range("so", kAcceptAll, collect_terms(&got), &exhausted)); + ASSERT_EQ(got.size(), 1U); + EXPECT_EQ(got[0], "solo"); + EXPECT_FALSE(exhausted); + EXPECT_EQ(dict_entry_body_decode_count(), 1U); +} + +TEST(SniiB2T07Dict, DecodeDictEntryKeyRejectsCorruptEntryLen) { + // entry_len varint decodes to 127 but only 3 bytes remain after it. + const std::vector buf = {0x7F, 0x00, 0x00, 0x00}; + ByteSource src {Slice(buf)}; + DictEntry e; + size_t body_start = 0; + uint64_t total = 0; + const Status s = decode_dict_entry_key(&src, "", &e, &body_start, &total); + EXPECT_FALSE(s.ok()); +} + +// ---- reader / query level: results unchanged + body-decode bounded ---- + +// prefix_terms routes through the rewired visit_prefix_range; it returns the same +// ordered hits and materializes only the matched bodies. +TEST(SniiB2T07DictReader, PrefixTermsResultsUnchangedAndStreamBounded) { + ScopedEnv resident_bsbf("SNII_BSBF_RESIDENT_MAX", "1048576"); + ScopedEnv resident_dict("SNII_DICT_RESIDENT_MAX", "1048576"); + MemoryFile file; + reader::SniiSegmentReader seg; + reader::LogicalIndexReader idx; + assert_ok(build_reader(&file, &seg, &idx)); + + std::vector hits; + reset_dict_entry_counters(); + assert_ok(idx.prefix_terms("ord", &hits, 10)); + ASSERT_EQ(hits.size(), 2U); + EXPECT_EQ(hits[0].term, "order"); + EXPECT_EQ(hits[1].term, "ordinal"); + // Resident block: only the two prefix matches' bodies are materialized. + EXPECT_EQ(dict_entry_body_decode_count(), 2U); +} + +TEST(SniiB2T07DictReader, PrefixTermsEmptyPrefixEnumeratesAllSorted) { + ScopedEnv resident_bsbf("SNII_BSBF_RESIDENT_MAX", "1048576"); + ScopedEnv resident_dict("SNII_DICT_RESIDENT_MAX", "1048576"); + MemoryFile file; + reader::SniiSegmentReader seg; + reader::LogicalIndexReader idx; + assert_ok(build_reader(&file, &seg, &idx)); + + std::vector hits; + assert_ok(idx.prefix_terms("", &hits, 0)); + std::vector terms; + terms.reserve(hits.size()); + for (const auto& h : hits) { + terms.push_back(h.term); + } + const std::vector expected = {"123", "almost", "driver", "failed", + "needle", "order", "ordinal", "repeat", + "sparse_left", "sparse_right", "trace"}; + EXPECT_EQ(terms, expected); +} + +TEST(SniiB2T07DictReader, LookupDecodesOnlyMatchedBody) { + ScopedEnv resident_bsbf("SNII_BSBF_RESIDENT_MAX", "1048576"); + ScopedEnv resident_dict("SNII_DICT_RESIDENT_MAX", "1048576"); + MemoryFile file; + reader::SniiSegmentReader seg; + reader::LogicalIndexReader idx; + assert_ok(build_reader(&file, &seg, &idx)); + + bool found = false; + DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + reset_dict_entry_counters(); + assert_ok(idx.lookup("trace", &found, &entry, &frq_base, &prx_base)); + EXPECT_TRUE(found); + EXPECT_EQ(entry.term, "trace"); + EXPECT_EQ(dict_entry_body_decode_count(), 1U); // only "trace" body materialized +} + +// End-to-end: a prefix query over the rewired streaming path equals the union of +// the expanded terms' postings. +TEST(SniiB2T07DictReader, PrefixQueryMatchesTermUnion) { + MemoryFile file; + reader::SniiSegmentReader seg; + reader::LogicalIndexReader idx; + assert_ok(build_reader(&file, &seg, &idx)); + + std::vector prefix_docids; + assert_ok(query::prefix_query(idx, "ord", &prefix_docids)); + + std::vector order_docids; + std::vector ordinal_docids; + assert_ok(query::term_query(idx, "order", &order_docids)); + assert_ok(query::term_query(idx, "ordinal", &ordinal_docids)); + + std::vector expected; + std::set_union(order_docids.begin(), order_docids.end(), ordinal_docids.begin(), + ordinal_docids.end(), std::back_inserter(expected)); + EXPECT_EQ(prefix_docids, expected); +} + +} // namespace +} // namespace doris::snii diff --git a/be/test/storage/index/snii_b2_t08_wildcard_test.cpp b/be/test/storage/index/snii_b2_t08_wildcard_test.cpp new file mode 100644 index 00000000000000..1c6c96df37ca6b --- /dev/null +++ b/be/test/storage/index/snii_b2_t08_wildcard_test.cpp @@ -0,0 +1,373 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// T08 -- wildcard matcher scratch reuse. +// +// Proves the request-scoped internal::WildcardMatcher (a) matches bit-for-bit +// identically to the former per-call DP in wildcard_query.cpp, and (b) reuses its +// two DP scratch rows across every visited term so a whole-dictionary scan +// performs O(1) heap allocations (<= 2) instead of O(2N). A header-only +// CountingAllocator gives the deterministic allocation counts; a byte-for-byte +// copy of the original DP serves as the equivalence oracle. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/internal/wildcard_matcher.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/query/wildcard_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii_query_test_util.h" + +namespace doris::snii::query { +using doris::Status; // RETURN_IF_ERROR / Status::OK() expand to a bare Status. +namespace { + +// Shared reader/writer fixtures live in snii_query_test_util.h; pull the ones +// this suite needs into scope unqualified. +using snii_test::assert_ok; +using snii_test::build_reader; +using snii_test::MemoryFile; + +// Minimal counting allocator: every allocate() bumps a per-type static counter so +// tests can assert exact heap-allocation counts without overriding global new. +// Stateless (all instances compare equal), so std::vector::swap stays a pointer +// swap that performs no allocation -- exactly what the matcher relies on. +template +struct CountingAllocator { + using value_type = T; + + CountingAllocator() noexcept = default; + // Converting (rebind) constructor: intentionally non-explicit, as required by + // the Allocator named requirement / std::allocator_traits. + template + CountingAllocator(const CountingAllocator& /*other*/) noexcept {} // NOLINT(*-explicit-*) + + T* allocate(std::size_t n) { + ++s_total_allocs; + ++s_live; + return static_cast(::operator new(n * sizeof(T))); + } + void deallocate(T* p, std::size_t /*n*/) noexcept { + --s_live; + ::operator delete(p); + } + + template + bool operator==(const CountingAllocator& /*other*/) const noexcept { + return true; + } + template + bool operator!=(const CountingAllocator& /*other*/) const noexcept { + return false; + } + + static void reset() { + s_total_allocs = 0; + s_live = 0; + } + static std::size_t total_allocs() { return s_total_allocs; } + static std::size_t live() { return s_live; } + + static inline std::size_t s_total_allocs = 0; + static inline std::size_t s_live = 0; +}; + +// Byte-for-byte copy of the former wildcard_query.cpp DP (templated only so the +// baseline-characterization test can count its per-call allocations). This is the +// equivalence oracle the optimized matcher must reproduce exactly. +template > +bool wildcard_match_dp_reference(std::string_view pattern, std::string_view text) { + std::vector prev(text.size() + 1, 0); + std::vector curr(text.size() + 1, 0); + prev[0] = 1; + + for (char p : pattern) { + std::fill(curr.begin(), curr.end(), 0); + if (p == '*') { + curr[0] = prev[0]; + for (size_t i = 1; i <= text.size(); ++i) { + curr[i] = prev[i] || curr[i - 1]; + } + } else { + for (size_t i = 1; i <= text.size(); ++i) { + curr[i] = prev[i - 1] && (p == '?' || p == text[i - 1]); + } + } + prev.swap(curr); + } + return prev[text.size()] != 0; +} + +// All strings of length 0..max_len over `alphabet`, for exhaustive equivalence. +std::vector all_strings_up_to(std::string_view alphabet, size_t max_len) { + std::vector out; + out.emplace_back(); + size_t level_begin = 0; + for (size_t len = 1; len <= max_len; ++len) { + const size_t level_end = out.size(); + for (size_t i = level_begin; i < level_end; ++i) { + for (char c : alphabet) { + out.push_back(out[i] + c); + } + } + level_begin = level_end; + } + return out; +} + +// `count` terms whose first (warmup) entry is the longest (`max_len`); every later +// term is <= max_len, so a matcher that reuses scratch reallocates only on the +// first call. The lengths still vary across terms (cycling 0..max_len). +std::vector make_varied_length_terms(size_t count, size_t max_len) { + std::vector terms; + terms.reserve(count); + terms.push_back(std::string(max_len, 'a')); + for (size_t i = 1; i < count; ++i) { + const size_t len = i % (max_len + 1); + terms.push_back(std::string(len, static_cast('a' + (i % 26)))); + } + return terms; +} + +// W-EQ-DP: the optimized matcher reproduces the reference DP bit-for-bit over an +// exhaustive small-alphabet battery (covers "", leading/trailing '*'/'?', +// consecutive "**", '?' interplay) plus realistic dictionary patterns/terms. One +// matcher is reused across all terms of a pattern, so this also proves scratch +// reuse never corrupts a result. +TEST(SniiWildcardQueryTest, MatcherEquivalentToReferenceDp) { + const std::vector exhaustive_patterns = all_strings_up_to("ab*?", 4); + const std::vector exhaustive_texts = all_strings_up_to("ab", 4); + for (const std::string& pattern : exhaustive_patterns) { + internal::WildcardMatcher<> matcher(pattern); + for (const std::string& text : exhaustive_texts) { + EXPECT_EQ(matcher(text), wildcard_match_dp_reference(pattern, text)) + << "pattern=\"" << pattern << "\" text=\"" << text << "\""; + } + } + + const std::vector patterns = { + "", "*", "?", "ord*", "?rder", "*failed*order*", "ordinal", + "order", "**a**", "a?b", "*a*", "?ailed", "sparse_*", "*_left"}; + const std::vector terms = {"", + "order", + "ordinal", + "failed", + "needle", + "driver", + "almost", + "123", + "repeat", + "sparse_left", + "sparse_right", + "trace", + "ordering", + std::string(40, 'a')}; + for (const std::string& pattern : patterns) { + internal::WildcardMatcher<> matcher(pattern); + for (const std::string& text : terms) { + EXPECT_EQ(matcher(text), wildcard_match_dp_reference(pattern, text)) + << "pattern=\"" << pattern << "\" text=\"" << text << "\""; + } + } +} + +// W-EMPTY-PAT: an empty pattern matches only the empty string. +TEST(SniiWildcardQueryTest, EmptyPatternMatchesOnlyEmptyText) { + internal::WildcardMatcher<> matcher(""); + EXPECT_TRUE(matcher("")); + EXPECT_FALSE(matcher("a")); +} + +// W-STAR-ONLY: "*" matches the empty string and any non-empty string. +TEST(SniiWildcardQueryTest, StarMatchesEverything) { + internal::WildcardMatcher<> matcher("*"); + EXPECT_TRUE(matcher("")); + EXPECT_TRUE(matcher("x")); + EXPECT_TRUE(matcher("xyz")); +} + +// W-QMARK: "?" matches exactly one byte. +TEST(SniiWildcardQueryTest, QuestionMarkMatchesExactlyOneByte) { + internal::WildcardMatcher<> matcher("?"); + EXPECT_FALSE(matcher("")); + EXPECT_TRUE(matcher("a")); + EXPECT_FALSE(matcher("ab")); +} + +// W-CONSEC-STAR: consecutive '*' degrade gracefully. +TEST(SniiWildcardQueryTest, ConsecutiveStars) { + internal::WildcardMatcher<> matcher("**a**"); + EXPECT_TRUE(matcher("a")); + EXPECT_TRUE(matcher("xax")); + EXPECT_FALSE(matcher("b")); +} + +// W-ANCHOR: a literal pattern is anchored at both ends (full match only). +TEST(SniiWildcardQueryTest, LiteralIsFullyAnchored) { + internal::WildcardMatcher<> matcher("ab"); + EXPECT_TRUE(matcher("ab")); + EXPECT_FALSE(matcher("abc")); + EXPECT_FALSE(matcher("xab")); +} + +// Perf (deterministic): the matcher allocates its two scratch rows once and reuses +// them across every term -- total heap allocations stay <= 2 and are independent +// of the term count N. +TEST(SniiWildcardQueryTest, MatcherReusesScratchAcrossTerms) { + using Alloc = CountingAllocator; + + auto allocs_for_n = [](size_t n) { + Alloc::reset(); + internal::WildcardMatcher matcher("*a*"); + for (const std::string& term : make_varied_length_terms(n, /*max_len=*/64)) { + matcher(term); + } + return Alloc::total_allocs(); + }; + + EXPECT_LE(allocs_for_n(1000), 2U); + // N-independent: exactly the two scratch rows, whether N=10 or N=1000. + EXPECT_EQ(allocs_for_n(10), 2U); + EXPECT_EQ(allocs_for_n(1000), 2U); + EXPECT_EQ(Alloc::live(), 0U); // matcher destroyed each lambda call: no leak. +} + +// Perf (deterministic): once warmed up with the longest term, the scratch capacity +// never changes across subsequent shorter/equal-length terms (no realloc). +TEST(SniiWildcardQueryTest, MatcherScratchCapacityStable) { + internal::WildcardMatcher<> matcher("*x*"); + matcher(std::string(64, 'a')); // warmup with the longest term + const size_t cap_after_warmup = matcher.scratch_capacity(); + EXPECT_GE(cap_after_warmup, 65U); + for (size_t len : {size_t {0}, size_t {1}, size_t {7}, size_t {63}, size_t {64}}) { + matcher(std::string(len, 'b')); + EXPECT_EQ(matcher.scratch_capacity(), cap_after_warmup); + } +} + +// Perf baseline (deterministic, contrast): the former per-call DP constructs two +// std::vectors every call, so N terms cost exactly 2N allocations -- the cost the +// reused matcher above eliminates. +TEST(SniiWildcardQueryTest, PerCallDpReferenceAllocatesTwicePerTerm) { + using Alloc = CountingAllocator; + Alloc::reset(); + const std::vector terms = make_varied_length_terms(/*count=*/100, /*max_len=*/64); + for (const std::string& term : terms) { + wildcard_match_dp_reference("*a*", term); + } + EXPECT_EQ(Alloc::total_allocs(), 2U * terms.size()); + EXPECT_EQ(Alloc::live(), 0U); +} + +// W-RESULT: end-to-end, "ord*" returns the sorted deduplicated union of the +// "order" and "ordinal" docid sets (independently computed expected set). +TEST(SniiWildcardQueryTest, WildcardResultIsTermUnion) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + std::vector order_docids; + std::vector ordinal_docids; + assert_ok(term_query(index_reader, "order", &order_docids)); + assert_ok(term_query(index_reader, "ordinal", &ordinal_docids)); + std::vector expected; + std::set_union(order_docids.begin(), order_docids.end(), ordinal_docids.begin(), + ordinal_docids.end(), std::back_inserter(expected)); + + std::vector docids; + assert_ok(wildcard_query(index_reader, "ord*", &docids)); + EXPECT_EQ(docids, expected); +} + +// W-QMARK-FULL: a leading '?' forces a full-dictionary scan; "?rder" matches only +// the 5-byte "order" term (not 7-byte "ordinal"). +TEST(SniiWildcardQueryTest, LeadingQuestionMarkResult) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + std::vector order_docids; + assert_ok(term_query(index_reader, "order", &order_docids)); + + std::vector docids; + assert_ok(wildcard_query(index_reader, "?rder", &docids)); + EXPECT_EQ(docids, order_docids); +} + +// W-HIDDEN-BIGRAM: hidden phrase-bigram terms must never leak through a leading +// wildcard scan (the bigram filter in term_expansion hides them). +TEST(SniiWildcardQueryTest, DoesNotExposeHiddenBigramTerms) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader, + /*include_phrase_bigrams=*/true)); + + std::vector docids; + assert_ok(wildcard_query(index_reader, "*failed*order*", &docids)); + EXPECT_TRUE(docids.empty()); +} + +// W-MAXEXP: max_expansions caps the number of expanded terms; terms enumerate in +// sorted order, so "*" with max_expansions=1 yields only the first term "123". +TEST(SniiWildcardQueryTest, MaxExpansionsCapsExpansion) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + std::vector first_term_docids; + assert_ok(term_query(index_reader, "123", &first_term_docids)); + + std::vector docids; + assert_ok(wildcard_query(index_reader, "*", &docids, /*max_expansions=*/1)); + EXPECT_EQ(docids, first_term_docids); +} + +// W-NULL-OUT / W-NULL-SINK: null output and null sink return InvalidArgument +// (no crash, no throw). +TEST(SniiWildcardQueryTest, NullArgumentsReturnInvalidArgument) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + std::vector* const null_docids = nullptr; + EXPECT_TRUE(wildcard_query(index_reader, "a*", null_docids) + .is()); + + DocIdSink* const null_sink = nullptr; + EXPECT_TRUE( + wildcard_query(index_reader, "a*", null_sink).is()); +} + +} // namespace +} // namespace doris::snii::query diff --git a/be/test/storage/index/snii_b2_t10_cover_test.cpp b/be/test/storage/index/snii_b2_t10_cover_test.cpp new file mode 100644 index 00000000000000..94f1ecf2a8fdb8 --- /dev/null +++ b/be/test/storage/index/snii_b2_t10_cover_test.cpp @@ -0,0 +1,376 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// T10 -- select_covering_windows as a monotonic two-pointer cursor (O(C + N)). +// Covers: (a) the isolated cursor core select_covering_windows_cursor and the +// FrqPreludeReader::select_covering_windows member produce results element-for-element +// identical to the legacy per-candidate locate_window + run-collapse oracle; (b) the +// window_probe_count() seam shows the cursor is O(C + N) (not O(C * N)) and bounded by +// C + N regardless of group_size, while the legacy in-block rescan grows with G; (c) the +// packed win_last_docid_ catalogue is byte-identical to WindowMeta.last_docid; and (d) the +// wired phrase query path is unchanged. + +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii_query_test_util.h" + +namespace doris::snii { +namespace { + +using namespace snii_test; // assert_ok, build_reader, MemoryFile, ... + +using format::build_frq_prelude; +using format::FrqPreludeColumns; +using format::FrqPreludeReader; +using format::select_covering_windows_cursor; +using format::WindowMeta; + +// Trusted oracle: the legacy locate_window-per-candidate + run-collapse semantics, +// computed with a dumb per-candidate linear scan (first window whose last_docid >= d). +std::vector oracle_select(const std::vector& win_last_docid, + const std::vector& candidates) { + std::vector out; + uint32_t last = UINT32_MAX; + for (uint32_t d : candidates) { + bool found = false; + uint32_t hit = 0; + for (uint32_t w = 0; w < win_last_docid.size(); ++w) { + if (d <= win_last_docid[w]) { + found = true; + hit = w; + break; + } + } + if (!found) continue; + if (hit != last) { + out.push_back(hit); + last = hit; + } + } + return out; +} + +// Derives sb_last_docid exactly as FrqPreludeReader::open does: the absolute last docid +// of each super-block's last window. +std::vector derive_sb_last_docid(const std::vector& win_last_docid, + uint32_t group_size) { + std::vector sb; + const uint32_t n = static_cast(win_last_docid.size()); + if (n == 0) return sb; + const uint32_t n_super = (n + group_size - 1) / group_size; + sb.reserve(n_super); + for (uint32_t s = 0; s < n_super; ++s) { + const uint32_t last_win = std::min((s + 1) * group_size, n) - 1; + sb.push_back(win_last_docid[last_win]); + } + return sb; +} + +// Runs the pure cursor core over directory arrays derived from win_last_docid. +std::vector run_cursor(const std::vector& win_last_docid, uint32_t group_size, + const std::vector& candidates) { + const std::vector sb = derive_sb_last_docid(win_last_docid, group_size); + std::vector out; + select_covering_windows_cursor(win_last_docid.data(), + static_cast(win_last_docid.size()), sb.data(), + static_cast(sb.size()), group_size, candidates, &out); + return out; +} + +// Builds a real prelude from strictly-increasing absolute last docids (doc_count=1 and +// contiguous 1-byte dd/freq regions satisfy the prelude width / layout validators). +void make_test_prelude(const std::vector& last_docids, uint32_t group_size, + FrqPreludeReader* reader) { + FrqPreludeColumns cols; + cols.has_freq = true; + cols.has_prx = false; + cols.group_size = group_size; + uint64_t dd_running = 0; + uint64_t freq_running = 0; + for (uint32_t v : last_docids) { + WindowMeta m; + m.last_docid = v; + m.doc_count = 1; + m.dd_off = dd_running; + m.dd_disk_len = 1; + m.dd_uncomp_len = 1; + m.crc_dd = v; + dd_running += 1; + m.freq_off = freq_running; + m.freq_disk_len = 1; + m.freq_uncomp_len = 1; + m.crc_freq = v + 1; + freq_running += 1; + cols.windows.push_back(m); + } + ByteSink sink; + assert_ok(build_frq_prelude(cols, &sink)); + assert_ok(FrqPreludeReader::open(sink.view(), reader)); +} + +// Production oracle: per-candidate locate_window + run-collapse over a real prelude. +std::vector locate_window_select(const FrqPreludeReader& prelude, + const std::vector& candidates) { + std::vector out; + uint32_t last = UINT32_MAX; + for (uint32_t d : candidates) { + bool found = false; + uint32_t w = 0; + assert_ok(prelude.locate_window(d, &found, &w)); + if (!found) continue; + if (w != last) { + out.push_back(w); + last = w; + } + } + return out; +} + +// FV-01: randomized equivalence of the cursor with the oracle across N and group_size. +TEST(SniiB2T10CoverTest, CursorMatchesOracleOnRandomAscendingSets) { + std::mt19937 rng(0x7710u); + const uint32_t kNs[] = {1, 2, 8, 64, 200, 4000}; + const uint32_t kGs[] = {1, 8, 64}; + std::uniform_int_distribution step(1, 5); // strictly increasing last_docid + std::uniform_int_distribution keep(0, 3); // ~1/4 of docids become candidates + for (uint32_t n : kNs) { + std::vector win_last(n); + uint32_t acc = 0; + for (uint32_t w = 0; w < n; ++w) { + acc += step(rng); + win_last[w] = acc; + } + const uint32_t max_docid = acc + 3; // a few candidates land past the last window + std::vector cands; + for (uint32_t d = 0; d <= max_docid; ++d) { + if (keep(rng) == 0) cands.push_back(d); // ascending, distinct, some same-window runs + } + const std::vector expected = oracle_select(win_last, cands); + for (uint32_t g : kGs) { + EXPECT_EQ(run_cursor(win_last, g, cands), expected) << "n=" << n << " g=" << g; + } + } +} + +// FV-02: the reader member (real prelude, production path) agrees with per-candidate +// locate_window over the same prelude. +TEST(SniiB2T10CoverTest, MemberMatchesRealLocateWindow) { + std::vector last_docids; + uint32_t acc = 0; + for (uint32_t w = 0; w < 300; ++w) { + acc += 1 + (w % 7); + last_docids.push_back(acc); + } + FrqPreludeReader prelude; + make_test_prelude(last_docids, /*group_size=*/64, &prelude); + + std::vector cands; + for (uint32_t d = 0; d <= acc + 5; d += 3) cands.push_back(d); + + std::vector got; + prelude.select_covering_windows(cands, &got); + EXPECT_EQ(got, locate_window_select(prelude, cands)); + EXPECT_EQ(got, oracle_select(last_docids, cands)); +} + +// FV-03: single window -- candidates inside emit {0}, candidates past it emit {}. +TEST(SniiB2T10CoverTest, SingleWindow) { + const std::vector win_last = {100}; + EXPECT_EQ(run_cursor(win_last, 64, {0, 50, 100}), (std::vector {0})); + EXPECT_EQ(run_cursor(win_last, 64, {0, 50, 100}), oracle_select(win_last, {0, 50, 100})); + EXPECT_TRUE(run_cursor(win_last, 64, {101, 200}).empty()); +} + +// FV-04: a candidate at docid 0 / at the first window's last docid still emits window 0. +TEST(SniiB2T10CoverTest, CandidateInFirstWindow) { + const std::vector win_last = {10, 20, 30}; + EXPECT_EQ(run_cursor(win_last, 2, {0, 10}), (std::vector {0})); + EXPECT_EQ(run_cursor(win_last, 2, {0, 10}), oracle_select(win_last, {0, 10})); +} + +// FV-05: candidates past the last window are dropped (early break == locate_window miss). +TEST(SniiB2T10CoverTest, CandidatesPastLastWindowAreDropped) { + const std::vector win_last = {10, 20, 30}; + const std::vector cands = {5, 25, 31, 40, 100}; // 5->w0, 25->w2, rest miss + EXPECT_EQ(run_cursor(win_last, 2, cands), (std::vector {0, 2})); + EXPECT_EQ(run_cursor(win_last, 2, cands), oracle_select(win_last, cands)); +} + +// FV-06: empty candidate list -> empty result (output is cleared first). +TEST(SniiB2T10CoverTest, EmptyCandidates) { + const std::vector win_last = {10, 20}; + const std::vector sb = derive_sb_last_docid(win_last, 64); + std::vector out = {99, 98, 97}; // pre-filled: must be cleared + select_covering_windows_cursor(win_last.data(), 2, sb.data(), static_cast(sb.size()), + 64, {}, &out); + EXPECT_TRUE(out.empty()); +} + +// FV-07: zero windows -> empty result, no crash (mirrors locate_window's empty early-out). +TEST(SniiB2T10CoverTest, EmptyWindows) { + const std::vector win_last; + EXPECT_TRUE(run_cursor(win_last, 64, {0, 5, 100}).empty()); +} + +// FV-08: sparse candidates crossing several super-block boundaries; cursor, member, and +// real locate_window all agree (boundary jump correctness). +TEST(SniiB2T10CoverTest, SuperBlockBoundaryCrossingSparse) { + std::vector last_docids; // 10 windows, stride 100 -> window w covers [w*100, ..+99] + for (uint32_t w = 0; w < 10; ++w) last_docids.push_back((w + 1) * 100 - 1); + FrqPreludeReader prelude; + make_test_prelude(last_docids, /*group_size=*/4, &prelude); // 3 super-blocks + const std::vector cands = {50, 450, 850, 999}; // windows 0, 4, 8, 9 + std::vector got; + prelude.select_covering_windows(cands, &got); + EXPECT_EQ(got, (std::vector {0, 4, 8, 9})); + EXPECT_EQ(got, locate_window_select(prelude, cands)); + EXPECT_EQ(run_cursor(last_docids, 4, cands), got); +} + +// FV-09: every window is covered -> the full {0..N-1} set, de-duplicated. +TEST(SniiB2T10CoverTest, DenseEveryWindowCovered) { + std::vector win_last; + std::vector cands; + std::vector expected; + for (uint32_t w = 0; w < 16; ++w) { + win_last.push_back(w); // window w covers exactly docid w + cands.push_back(w); + expected.push_back(w); + } + EXPECT_EQ(run_cursor(win_last, 4, cands), expected); + EXPECT_EQ(run_cursor(win_last, 4, cands), oracle_select(win_last, cands)); +} + +// FV-10: the packed win_last_docid_ catalogue is byte-identical to WindowMeta.last_docid. +TEST(SniiB2T10CoverTest, PackedLastDocidMatchesWindowMeta) { + std::vector last_docids; + uint32_t acc = 0; + for (uint32_t w = 0; w < 130; ++w) { // spans >1 super-block at G=64 + acc += 1 + (w % 3); + last_docids.push_back(acc); + } + FrqPreludeReader prelude; + make_test_prelude(last_docids, /*group_size=*/64, &prelude); + ASSERT_EQ(prelude.n_windows(), 130u); + for (uint32_t w = 0; w < prelude.n_windows(); ++w) { + WindowMeta m; + assert_ok(prelude.window(w, &m)); + EXPECT_EQ(prelude.window_last_docid(w), m.last_docid) << "w=" << w; + } +} + +// Deterministic complexity invariant: probe_count <= C + N for both dense and sparse +// candidate sets (i.e. O(C + N), far below O(C * N)); the sparse case additionally stays +// below N, proving the super-block jump avoids a full linear scan. +TEST(SniiB2T10CoverTest, CursorProbeCountStaysLinearInCandidatesPlusWindows) { + { + const uint32_t n = 256; // dense: C == N * stride + std::vector win_last(n); + for (uint32_t w = 0; w < n; ++w) win_last[w] = w * 4 + 3; + std::vector cands; + for (uint32_t d = 0; d < n * 4; ++d) cands.push_back(d); + format::testing::reset_window_probe_count(); + const std::vector got = run_cursor(win_last, 64, cands); + const uint64_t probes = format::testing::window_probe_count(); + EXPECT_LE(probes, static_cast(cands.size()) + n); + EXPECT_EQ(got, oracle_select(win_last, cands)); + } + { + const uint32_t n = 4000; // sparse: C << N + std::vector win_last(n); + for (uint32_t w = 0; w < n; ++w) win_last[w] = w; + const std::vector cands = {0, 1000, 2000, 3000, 3999}; + format::testing::reset_window_probe_count(); + const std::vector got = run_cursor(win_last, 64, cands); + const uint64_t probes = format::testing::window_probe_count(); + EXPECT_LE(probes, static_cast(cands.size()) + n); + EXPECT_LT(probes, n); // super-block jump prevents an O(N) linear walk + EXPECT_EQ(got, oracle_select(win_last, cands)); + } +} + +// Deterministic before/after contrast: the legacy per-candidate locate_window probe count +// grows with group_size and exceeds C + N, whereas the cursor stays <= C + N for every G +// and yields the same covering-window set regardless of G. +TEST(SniiB2T10CoverTest, LegacyProbeGrowsWithGroupSizeButCursorStaysLinear) { + const uint32_t n = 200; + std::vector last_docids(n); + std::vector cands(n); + for (uint32_t w = 0; w < n; ++w) { + last_docids[w] = w; // window w covers exactly docid w + cands[w] = w; // one candidate per window + } + const uint64_t cn = static_cast(cands.size()) + n; // C + N + + FrqPreludeReader prelude_g8; + FrqPreludeReader prelude_g64; + make_test_prelude(last_docids, 8, &prelude_g8); + make_test_prelude(last_docids, 64, &prelude_g64); + + format::testing::reset_window_probe_count(); + (void)locate_window_select(prelude_g8, cands); + const uint64_t legacy_g8 = format::testing::window_probe_count(); + + format::testing::reset_window_probe_count(); + (void)locate_window_select(prelude_g64, cands); + const uint64_t legacy_g64 = format::testing::window_probe_count(); + + EXPECT_GT(legacy_g64, legacy_g8); // deeper in-block rescans with larger G + EXPECT_GT(legacy_g8, cn); // even G=8 already exceeds C + N + EXPECT_GT(legacy_g64, cn); + + format::testing::reset_window_probe_count(); + const std::vector cur_g8 = run_cursor(last_docids, 8, cands); + const uint64_t cursor_g8 = format::testing::window_probe_count(); + + format::testing::reset_window_probe_count(); + const std::vector cur_g64 = run_cursor(last_docids, 64, cands); + const uint64_t cursor_g64 = format::testing::window_probe_count(); + + EXPECT_LE(cursor_g8, cn); // bounded by C + N regardless of G + EXPECT_LE(cursor_g64, cn); + EXPECT_LT(cursor_g64, legacy_g64); // cursor crushes the legacy probe count + EXPECT_EQ(cur_g8, cur_g64); // identical covering-window set across G + EXPECT_EQ(cur_g8, oracle_select(last_docids, cands)); +} + +// FV-11: the wired phrase query result is unchanged (no regression through the windowed +// docid-conjunction path that now uses the cursor member). +TEST(SniiB2T10CoverTest, WiredPhraseQueryResultUnchanged) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + std::vector docids; + assert_ok(query::phrase_query(index_reader, {"failed", "order"}, &docids)); + EXPECT_EQ(docids, (std::vector {5000, 7000, 8000})); +} + +} // namespace +} // namespace doris::snii diff --git a/be/test/storage/index/snii_clucene_guard_test.cpp b/be/test/storage/index/snii_clucene_guard_test.cpp new file mode 100644 index 00000000000000..45cf9ae1fe1d95 --- /dev/null +++ b/be/test/storage/index/snii_clucene_guard_test.cpp @@ -0,0 +1,165 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +namespace fs = std::filesystem; + +const std::regex kCluceneRefPattern(R"(clucene|lucene::|CL_NS| locate_snii_core_dir() { + std::vector candidates; +#ifdef SNII_CORE_SOURCE_DIR + // Optional exact path injected by the build system (highest priority). If injected + // by CMake it must be a quoted string literal, e.g. + // add_definitions(-DSNII_CORE_SOURCE_DIR="${CMAKE_SOURCE_DIR}/src/storage/index/snii"). + candidates.emplace_back(SNII_CORE_SOURCE_DIR); +#endif + + std::error_code ec; + fs::path source_path(__FILE__); + if (source_path.is_relative()) { + source_path = fs::absolute(source_path, ec); + } + // Walk up from this test's source directory; matches either /be/... or + // /... so it is robust to layout changes without hardcoding the depth. + for (fs::path dir = source_path.parent_path(); !dir.empty();) { + candidates.push_back(dir / "be" / "src" / "storage" / "index" / "snii"); + candidates.push_back(dir / "src" / "storage" / "index" / "snii"); + if (dir == dir.root_path()) { + break; + } + dir = dir.parent_path(); + } + + // Fallback: the DORIS_HOME env var (consistent with existing test utilities). + if (const char* doris_home = std::getenv("DORIS_HOME"); doris_home != nullptr) { + candidates.push_back(fs::path(doris_home) / "be" / "src" / "storage" / "index" / "snii"); + } + + for (const auto& candidate : candidates) { + if (fs::is_directory(candidate, ec)) { + auto canonical = fs::weakly_canonical(candidate, ec); + return ec ? candidate : canonical; + } + } + return std::nullopt; +} + +struct CluceneHit { + std::string file; + int line = 0; + std::string text; +}; + +// Reads one file and collects matching lines; returns false if it cannot be opened +// (treated as a scan gap). +bool scan_file_for_clucene(const fs::path& file, std::vector* hits) { + std::ifstream input(file); + if (!input.is_open()) { + return false; + } + std::string line; + int line_no = 0; + while (std::getline(input, line)) { + ++line_no; + if (std::regex_search(line, kCluceneRefPattern)) { + hits->push_back({file.string(), line_no, line}); + } + } + return true; +} + +} // namespace + +// The SNII core engine (the be/src/storage/index/snii module subdirs) must contain +// zero CLucene references; CLucene is allowed only in the top-level Doris integration +// layer (snii_doris_adapter / snii_index_*). +TEST(SniiCluceneDecouplingGuardTest, CoreHasNoCluceneRef) { + const auto snii_core_dir = locate_snii_core_dir(); + ASSERT_TRUE(snii_core_dir.has_value()) + << "could not locate the SNII core source directory be/src/storage/index/snii. " + << "Run the test inside the source tree, or set DORIS_HOME / -DSNII_CORE_SOURCE_DIR. " + << "(__FILE__=" << __FILE__ << ")"; + + std::error_code ec; + fs::recursive_directory_iterator it(*snii_core_dir, ec); + ASSERT_TRUE(!ec) << "cannot iterate " << snii_core_dir->string() << ": " << ec.message(); + const fs::recursive_directory_iterator end; + + std::vector hits; + std::vector unreadable; + int scanned_files = 0; + for (; it != end; it.increment(ec)) { + ASSERT_TRUE(!ec) << "error iterating " << snii_core_dir->string() << ": " << ec.message(); + const fs::directory_entry& entry = *it; + std::error_code stat_ec; + if (!entry.is_regular_file(stat_ec) || stat_ec) { + continue; + } + // Guard only the SNII core engine (the module subdirs), which must stay + // CLucene-free. Skip the top-level Doris integration files (snii_doris_adapter + // / snii_index_*), which legitimately bridge to CLucene, and non-source files + // (docs/*.md). + const std::string ext = entry.path().extension().string(); + if (ext != ".h" && ext != ".cpp" && ext != ".cc") { + continue; + } + if (entry.path().parent_path() == *snii_core_dir) { + continue; + } + ++scanned_files; + if (!scan_file_for_clucene(entry.path(), &hits)) { + unreadable.push_back(entry.path().string()); + } + } + + // Sanity: we must actually scan some files; otherwise a broken locate would make + // the guard silently pass. + ASSERT_GT(scanned_files, 0) << "scanned no files under " << snii_core_dir->string() + << "; the guard is ineffective."; + + // Unreadable files create scan gaps (possible missed detections); surface them + // non-fatally. + EXPECT_TRUE(unreadable.empty()) + << "some files could not be read (possible missed detections): " << unreadable.size() + << " total, first is " << (unreadable.empty() ? std::string {} : unreadable.front()); + + if (!hits.empty()) { + std::ostringstream oss; + oss << "the SNII core " << snii_core_dir->string() << " has " << hits.size() + << " CLucene reference(s) (must be 0; violates the SNII/CLucene decoupling " + "constraint). CLucene is allowed only in the integration layer:\n"; + for (const auto& hit : hits) { + oss << " " << hit.file << ":" << hit.line << ": " << hit.text << "\n"; + } + FAIL() << oss.str(); + } +} diff --git a/be/test/storage/index/snii_crc32c_equiv_test.cpp b/be/test/storage/index/snii_crc32c_equiv_test.cpp new file mode 100644 index 00000000000000..b17c8fe6e96ff2 --- /dev/null +++ b/be/test/storage/index/snii_crc32c_equiv_test.cpp @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// R05-crc32c: proves the snii crc32c header (now a thin inline wrapper over the +// Doris/Google crc32c thirdparty) yields byte-identical checksums to both (a) the +// well-known canonical CRC32C test vectors and (b) the Doris ::crc32c library it +// delegates to. (a) guards the on-disk format value; (b) guards the delegation +// contract for every call site (one-shot, seeded extend, and segmented chaining). +// These checks are deterministic and need no pre-captured golden fixture. + +#include +#include + +#include +#include +#include +#include +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/crc32c.h" + +namespace { + +using doris::snii::Slice; + +// Fills n deterministic bytes from a seeded engine. The exact bytes are +// irrelevant to the equivalence asserts (both sides hash the same buffer); the +// fixed seed only keeps runs reproducible. +std::vector make_bytes(std::mt19937_64& rng, size_t n) { + std::vector v(n); + for (size_t i = 0; i < n; ++i) { + v[i] = static_cast(rng() & 0xFFU); + } + return v; +} + +// Lengths exercising the slice-by-8 main loop plus every <8-byte tail remainder, +// the 4-byte step, sub-block sizes, and several >1KB buffers. +constexpr size_t kLengths[] = {0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, + 33, 63, 64, 65, 127, 128, 255, 256, 1023, 1024, 1025, 4096, 5000}; + +} // namespace + +// (a) Hardcoded canonical CRC32C vectors. A reuse that altered the value -- and +// therefore the on-disk format -- would break these. +TEST(SniiCrc32cEquiv, StandardVectors) { + // Classic CRC-32C / iSCSI check value for the ASCII string "123456789". + { + const char* s = "123456789"; + EXPECT_EQ(0xE3069283U, doris::snii::crc32c(Slice(reinterpret_cast(s), 9))); + } + // Empty input conditions to 0 (Extend(0, ., 0)). + EXPECT_EQ(0x00000000U, doris::snii::crc32c(Slice(nullptr, 0))); + + // RFC 3720 section B.4 vectors, matching be/test/util/crc32c_test.cpp. + uint8_t buf[32]; + std::memset(buf, 0x00, sizeof(buf)); + EXPECT_EQ(0x8A9136AAU, doris::snii::crc32c(Slice(buf, sizeof(buf)))); + + std::memset(buf, 0xFF, sizeof(buf)); + EXPECT_EQ(0x62A8AB43U, doris::snii::crc32c(Slice(buf, sizeof(buf)))); + + for (int i = 0; i < 32; ++i) { + buf[i] = static_cast(i); + } + EXPECT_EQ(0x46DD794EU, doris::snii::crc32c(Slice(buf, sizeof(buf)))); + + for (int i = 0; i < 32; ++i) { + buf[i] = static_cast(31 - i); + } + EXPECT_EQ(0x113FDB5CU, doris::snii::crc32c(Slice(buf, sizeof(buf)))); +} + +// (b1) One-shot equivalence: doris::snii::crc32c(Slice(buf, n)) == ::crc32c::Crc32c(buf, n). +TEST(SniiCrc32cEquiv, OneShotMatchesDorisLib) { + std::mt19937_64 rng(0xC0FFEEULL); + for (size_t n : kLengths) { + const std::vector buf = make_bytes(rng, n); + const uint8_t* p = buf.data(); + EXPECT_EQ(doris::snii::crc32c(Slice(p, n)), ::crc32c::Crc32c(p, n)) << "len=" << n; + } +} + +// (b2) Seeded extend equivalence: doris::snii::crc32c_extend(prev, Slice) == ::crc32c::Extend(prev, ...), +// covering prev == 0 and several non-zero priors. +TEST(SniiCrc32cEquiv, ExtendMatchesDorisLib) { + std::mt19937_64 rng(0x12345678ULL); + const uint32_t priors[] = {0U, 1U, 0xFFFFFFFFU, 0xDEADBEEFU, 0x80000000U}; + for (uint32_t prev : priors) { + for (size_t n : kLengths) { + const std::vector buf = make_bytes(rng, n); + const uint8_t* p = buf.data(); + EXPECT_EQ(doris::snii::crc32c_extend(prev, Slice(p, n)), ::crc32c::Extend(prev, p, n)) + << "prev=" << prev << " len=" << n; + } + } +} + +// (b3) Segmented chaining: extend over a split equals the one-shot over the whole, +// and matches the Doris lib's Extend(Crc32c(prefix), suffix) -- i.e. the +// Extend(Extend(0, a), b) == Crc32c(a ++ b) property every framed section relies on. +TEST(SniiCrc32cEquiv, ExtendSplitEqualsWhole) { + std::mt19937_64 rng(0xABCDEFULL); + const size_t totals[] = {1, 2, 8, 9, 16, 17, 100, 1024, 4096}; + for (size_t n : totals) { + const std::vector buf = make_bytes(rng, n); + const uint8_t* p = buf.data(); + const uint32_t whole = doris::snii::crc32c(Slice(p, n)); + for (size_t k = 0; k <= n; ++k) { + const uint32_t chained = doris::snii::crc32c_extend(doris::snii::crc32c(Slice(p, k)), + Slice(p + k, n - k)); + EXPECT_EQ(whole, chained) << "n=" << n << " k=" << k; + EXPECT_EQ(chained, ::crc32c::Extend(::crc32c::Crc32c(p, k), p + k, n - k)) + << "n=" << n << " k=" << k; + } + } +} + +// (b4) Cross-decode both directions: a crc stamped by the snii wrapper verifies +// under the Doris lib, and a crc stamped by the Doris lib verifies under the snii +// wrapper -- the bidirectional guarantee that old and new on-disk bytes interop. +TEST(SniiCrc32cEquiv, CrossDecodeBidirectional) { + std::mt19937_64 rng(0x5A5A5A5AULL); + for (size_t n : kLengths) { + const std::vector buf = make_bytes(rng, n); + const uint8_t* p = buf.data(); + EXPECT_EQ(doris::snii::crc32c(Slice(p, n)), ::crc32c::Crc32c(p, n)) << "len=" << n; + EXPECT_EQ(::crc32c::Crc32c(p, n), doris::snii::crc32c(Slice(p, n))) << "len=" << n; + } +} diff --git a/be/test/storage/index/snii_doris_adapter_test.cpp b/be/test/storage/index/snii_doris_adapter_test.cpp new file mode 100644 index 00000000000000..44262c5bfb6825 --- /dev/null +++ b/be/test/storage/index/snii_doris_adapter_test.cpp @@ -0,0 +1,465 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/index/snii/snii_doris_adapter.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "io/fs/file_reader.h" +#include "io/fs/path.h" +#include "io/io_common.h" +#include "storage/index/snii/format/per_index_meta.h" +#include "storage/index/snii/io/file_reader.h" +#include "util/slice.h" +#include "util/threadpool.h" + +namespace doris::segment_v2::snii_doris { +namespace { + +struct CapturedIOContext { + bool has_ctx = false; + bool is_inverted_index = false; + bool is_index_data = false; + bool read_file_cache = true; + bool is_disposable = false; + int64_t expiration_time = 0; + io::FileCacheStatistics* file_cache_stats = nullptr; +}; + +struct CapturedRead { + size_t offset = 0; + size_t len = 0; + CapturedIOContext io_ctx; + // Destination buffer the read wrote into; used to prove single-segment reads + // land directly in the caller's output (no temp/double buffer). + const void* dst = nullptr; + // Worker thread that served the read; supports parallel-path assertions. + std::thread::id thread_id; +}; + +class RecordingFileReader final : public io::FileReader { +public: + explicit RecordingFileReader(std::string data) : _data(std::move(data)) {} + + Status close() override { + _closed = true; + return Status::OK(); + } + + const io::Path& path() const override { return _path; } + size_t size() const override { return _data.size(); } + bool closed() const override { return _closed; } + int64_t mtime() const override { return 0; } + + // Safe to read after read_batch returns: all worker reads happen-before the + // caller (latch join) and writes are guarded by _reads_mu. + const std::vector& reads() const { return _reads; } + +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const io::IOContext* io_ctx) override { + CapturedRead read; + read.offset = offset; + read.len = result.size; + read.dst = result.data; + read.thread_id = std::this_thread::get_id(); + if (io_ctx != nullptr) { + read.io_ctx.has_ctx = true; + read.io_ctx.is_inverted_index = io_ctx->is_inverted_index; + read.io_ctx.is_index_data = io_ctx->is_index_data; + read.io_ctx.read_file_cache = io_ctx->read_file_cache; + read.io_ctx.is_disposable = io_ctx->is_disposable; + read.io_ctx.expiration_time = io_ctx->expiration_time; + read.io_ctx.file_cache_stats = io_ctx->file_cache_stats; + } + + if (result.size > 0) { + std::memcpy(result.data, _data.data() + offset, result.size); + } + *bytes_read = result.size; + + // Parallel batch reads may invoke this from several pool threads at once; + // guard the capture log so the test infrastructure is race-free. + std::lock_guard guard(_reads_mu); + _reads.push_back(read); + return Status::OK(); + } + +private: + std::string _data; + io::Path _path = "/tmp/snii_doris_adapter_test.idx"; + bool _closed = false; + mutable std::mutex _reads_mu; + std::vector _reads; +}; + +// Deterministic, position-dependent byte pattern so every range's expected bytes +// are just `pattern.substr(offset, len)`. +std::string make_pattern(size_t n) { + std::string s(n, '\0'); + for (size_t i = 0; i < n; ++i) { + s[i] = static_cast('a' + static_cast(i % 26)); + } + return s; +} + +std::string as_string(const std::vector& v) { + return std::string(v.begin(), v.end()); +} + +// Restores the batch-read executor seam on scope exit so an injected pool never +// leaks into sibling tests, even if an assertion returns early. +struct IoPoolSeamGuard { + explicit IoPoolSeamGuard(ThreadPool* pool) { + DorisSniiFileReader::set_io_thread_pool_for_test(pool); + } + ~IoPoolSeamGuard() { DorisSniiFileReader::set_io_thread_pool_for_test(nullptr); } + IoPoolSeamGuard(const IoPoolSeamGuard&) = delete; + IoPoolSeamGuard& operator=(const IoPoolSeamGuard&) = delete; +}; + +} // namespace + +TEST(DorisSniiFileReaderTest, ReadAtPropagatesIndexIOContextAndRecordsStats) { + auto recording_reader = std::make_shared("0123456789abcdef"); + DorisSniiFileReader reader(recording_reader); + + io::FileCacheStatistics stats; + io::IOContext io_ctx; + io_ctx.is_disposable = true; + io_ctx.is_index_data = true; + io_ctx.read_file_cache = false; + io_ctx.file_cache_stats = &stats; + + std::vector out; + { + DorisSniiFileReader::ScopedIOContext scope(&io_ctx); + auto status = reader.read_at(2, 5, &out); + ASSERT_TRUE(status.ok()) << status.to_string(); + } + + ASSERT_EQ(out.size(), 5); + EXPECT_EQ(std::string(out.begin(), out.end()), "23456"); + ASSERT_EQ(recording_reader->reads().size(), 1); + const auto& captured = recording_reader->reads()[0].io_ctx; + EXPECT_TRUE(captured.has_ctx); + EXPECT_TRUE(captured.is_inverted_index); + EXPECT_TRUE(captured.is_index_data); + EXPECT_FALSE(captured.read_file_cache); + EXPECT_TRUE(captured.is_disposable); + EXPECT_EQ(captured.file_cache_stats, &stats); + + EXPECT_EQ(stats.inverted_index_request_bytes, 5); + EXPECT_EQ(stats.inverted_index_read_bytes, 5); + EXPECT_EQ(stats.inverted_index_range_read_count, 1); + EXPECT_EQ(stats.inverted_index_serial_read_rounds, 1); +} + +TEST(DorisSniiFileReaderTest, ReadBatchRecordsLogicalAndCoalescedPhysicalIO) { + auto recording_reader = + std::make_shared("0123456789abcdefghijklmnopqrstuvwxyz"); + DorisSniiFileReader reader(recording_reader); + + io::FileCacheStatistics stats; + io::IOContext io_ctx; + io_ctx.file_cache_stats = &stats; + + std::vector> outs; + { + DorisSniiFileReader::ScopedIOContext scope(&io_ctx); + std::vector<::doris::snii::io::Range> ranges {{0, 4}, {6, 3}, {20, 2}}; + auto status = reader.read_batch(ranges, &outs); + ASSERT_TRUE(status.ok()) << status.to_string(); + } + + ASSERT_EQ(outs.size(), 3); + EXPECT_EQ(std::string(outs[0].begin(), outs[0].end()), "0123"); + EXPECT_EQ(std::string(outs[1].begin(), outs[1].end()), "678"); + EXPECT_EQ(std::string(outs[2].begin(), outs[2].end()), "kl"); + + ASSERT_EQ(recording_reader->reads().size(), 1); + EXPECT_EQ(recording_reader->reads()[0].offset, 0); + EXPECT_EQ(recording_reader->reads()[0].len, 22); + + EXPECT_EQ(stats.inverted_index_request_bytes, 9); + EXPECT_EQ(stats.inverted_index_read_bytes, 22); + EXPECT_EQ(stats.inverted_index_range_read_count, 1); + EXPECT_EQ(stats.inverted_index_serial_read_rounds, 1); +} + +// FB-02: three ranges spaced >4096 apart form three disjoint physical segments. +// Each is still a separate physical read, but the batch is one concurrent round +// (F19): serial_read_rounds drops from K(==3) to 1. +TEST(DorisSniiFileReaderTest, ReadBatchIssuesSingleSerialRoundForDisjointSegments) { + const std::string data = make_pattern(20000); + auto recording_reader = std::make_shared(data); + DorisSniiFileReader reader(recording_reader); + + io::FileCacheStatistics stats; + io::IOContext io_ctx; + io_ctx.file_cache_stats = &stats; + + std::vector> outs; + { + DorisSniiFileReader::ScopedIOContext scope(&io_ctx); + std::vector<::doris::snii::io::Range> ranges {{0, 4}, {8192, 4}, {16384, 4}}; + auto status = reader.read_batch(ranges, &outs); + ASSERT_TRUE(status.ok()) << status.to_string(); + } + + ASSERT_EQ(outs.size(), 3); + EXPECT_EQ(as_string(outs[0]), data.substr(0, 4)); + EXPECT_EQ(as_string(outs[1]), data.substr(8192, 4)); + EXPECT_EQ(as_string(outs[2]), data.substr(16384, 4)); + + EXPECT_EQ(recording_reader->reads().size(), 3); + EXPECT_EQ(stats.inverted_index_request_bytes, 12); + EXPECT_EQ(stats.inverted_index_read_bytes, 12); + EXPECT_EQ(stats.inverted_index_range_read_count, 3); + EXPECT_EQ(stats.inverted_index_serial_read_rounds, 1); +} + +// F27: a single-range group reads straight into the caller's output slot, with no +// temporary buffer and no second memcpy. Proven by destination-pointer identity. +TEST(DorisSniiFileReaderTest, ReadBatchSingleSegmentReadsInPlace) { + const std::string data = make_pattern(20000); + auto recording_reader = std::make_shared(data); + DorisSniiFileReader reader(recording_reader); + + std::vector> outs; + std::vector<::doris::snii::io::Range> ranges {{100, 8}}; + auto status = reader.read_batch(ranges, &outs); + ASSERT_TRUE(status.ok()) << status.to_string(); + + ASSERT_EQ(outs.size(), 1); + EXPECT_EQ(as_string(outs[0]), data.substr(100, 8)); + ASSERT_EQ(recording_reader->reads().size(), 1); + // The read wrote directly into outs[0]'s storage (no double buffer). + EXPECT_EQ(recording_reader->reads()[0].dst, outs[0].data()); +} + +// FB-03: a batch mixing one coalesced group (temp + scatter) and one single-range +// group (direct read). Both branches produce correct bytes. +TEST(DorisSniiFileReaderTest, ReadBatchMixedSingleAndCoalescedGroups) { + const std::string data = make_pattern(20000); + auto recording_reader = std::make_shared(data); + DorisSniiFileReader reader(recording_reader); + + std::vector> outs; + std::vector<::doris::snii::io::Range> ranges {{0, 4}, {4, 4}, {9000, 4}}; + auto status = reader.read_batch(ranges, &outs); + ASSERT_TRUE(status.ok()) << status.to_string(); + + ASSERT_EQ(outs.size(), 3); + EXPECT_EQ(as_string(outs[0]), data.substr(0, 4)); + EXPECT_EQ(as_string(outs[1]), data.substr(4, 4)); + EXPECT_EQ(as_string(outs[2]), data.substr(9000, 4)); + + // Two physical segments: the coalesced [0,8) group and the single [9000,9004). + ASSERT_EQ(recording_reader->reads().size(), 2); + // The single-range group [9000,9004) read directly into outs[2]. + const auto& reads = recording_reader->reads(); + bool single_in_place = false; + for (const auto& r : reads) { + if (r.offset == 9000) { + single_in_place = (r.dst == outs[2].data()); + } + } + EXPECT_TRUE(single_in_place); +} + +// FB-04: empty batch and zero-length ranges. No physical reads; zero-length slots +// stay empty; outs aligns 1:1 with the input. +TEST(DorisSniiFileReaderTest, ReadBatchHandlesEmptyAndZeroLengthRanges) { + auto recording_reader = std::make_shared(make_pattern(64)); + DorisSniiFileReader reader(recording_reader); + + std::vector> empty_outs; + auto empty_status = reader.read_batch({}, &empty_outs); + ASSERT_TRUE(empty_status.ok()) << empty_status.to_string(); + EXPECT_TRUE(empty_outs.empty()); + EXPECT_EQ(recording_reader->reads().size(), 0); + + std::vector> outs; + std::vector<::doris::snii::io::Range> ranges {{5, 0}}; + auto status = reader.read_batch(ranges, &outs); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(outs.size(), 1); + EXPECT_TRUE(outs[0].empty()); + EXPECT_EQ(recording_reader->reads().size(), 0); +} + +// FB-05: an out-of-range request surfaces a corruption error and does not crash. +TEST(DorisSniiFileReaderTest, ReadBatchReturnsErrorForOutOfRange) { + auto recording_reader = std::make_shared(make_pattern(64)); + DorisSniiFileReader reader(recording_reader); + + std::vector> outs; + std::vector<::doris::snii::io::Range> ranges {{63, 100}}; + auto status = reader.read_batch(ranges, &outs); + EXPECT_FALSE(status.ok()); +} + +// FB-06: unsorted input still produces outputs in the caller's original index +// order (the skip-sort guard only avoids the sort when already sorted). +TEST(DorisSniiFileReaderTest, ReadBatchPreservesOriginalOrderForUnsortedInput) { + const std::string data = make_pattern(20000); + auto recording_reader = std::make_shared(data); + DorisSniiFileReader reader(recording_reader); + + std::vector> outs; + std::vector<::doris::snii::io::Range> ranges {{16384, 2}, {0, 4}, {8192, 3}}; + auto status = reader.read_batch(ranges, &outs); + ASSERT_TRUE(status.ok()) << status.to_string(); + + ASSERT_EQ(outs.size(), 3); + EXPECT_EQ(as_string(outs[0]), data.substr(16384, 2)); + EXPECT_EQ(as_string(outs[1]), data.substr(0, 4)); + EXPECT_EQ(as_string(outs[2]), data.substr(8192, 3)); +} + +// IOContext passthrough: every per-segment read sees the caller's flags. Each +// segment is routed through a private FileCacheStatistics slot (so disjoint reads +// never race), which is then merged back into the caller's sink. +TEST(DorisSniiFileReaderTest, ReadBatchPropagatesIOContextFlagsPerSegment) { + const std::string data = make_pattern(20000); + auto recording_reader = std::make_shared(data); + DorisSniiFileReader reader(recording_reader); + + io::FileCacheStatistics stats; + io::IOContext io_ctx; + io_ctx.is_index_data = true; + io_ctx.read_file_cache = false; + io_ctx.is_disposable = true; + io_ctx.expiration_time = 123; + io_ctx.file_cache_stats = &stats; + + std::vector> outs; + { + DorisSniiFileReader::ScopedIOContext scope(&io_ctx); + std::vector<::doris::snii::io::Range> ranges {{0, 4}, {8192, 4}, {16384, 4}}; + auto status = reader.read_batch(ranges, &outs); + ASSERT_TRUE(status.ok()) << status.to_string(); + } + + const auto& reads = recording_reader->reads(); + ASSERT_EQ(reads.size(), 3); + for (const auto& r : reads) { + EXPECT_TRUE(r.io_ctx.has_ctx); + EXPECT_TRUE(r.io_ctx.is_inverted_index); + EXPECT_TRUE(r.io_ctx.is_index_data); + EXPECT_FALSE(r.io_ctx.read_file_cache); + EXPECT_TRUE(r.io_ctx.is_disposable); + EXPECT_EQ(r.io_ctx.expiration_time, 123); + // Per-segment private stats slot, not the caller's sink directly. + EXPECT_NE(r.io_ctx.file_cache_stats, nullptr); + EXPECT_NE(r.io_ctx.file_cache_stats, &stats); + } + // Aggregate counters still land on the caller's sink after the merge. + EXPECT_EQ(stats.inverted_index_range_read_count, 3); + EXPECT_EQ(stats.inverted_index_serial_read_rounds, 1); +} + +// FB-08: with a real injected pool, eight disjoint segments are read in parallel. +// All bytes are correct, every segment is a distinct physical read, and the batch +// is one concurrent round. Run under TSAN to prove thread-safety. +TEST(DorisSniiFileReaderConcurrencyTest, ParallelSegmentReadsAreThreadSafe) { + const std::string data = make_pattern(60000); + auto recording_reader = std::make_shared(data); + DorisSniiFileReader reader(recording_reader); + + std::unique_ptr pool; + auto pool_st = + ThreadPoolBuilder("snii_batch_test").set_min_threads(2).set_max_threads(4).build(&pool); + ASSERT_TRUE(pool_st.ok()) << pool_st.to_string(); + IoPoolSeamGuard seam(pool.get()); + + io::FileCacheStatistics stats; + io::IOContext io_ctx; + io_ctx.file_cache_stats = &stats; + + std::vector<::doris::snii::io::Range> ranges; + for (size_t i = 0; i < 8; ++i) { + ranges.push_back({static_cast(i) * 8192, 4}); + } + + std::vector> outs; + { + DorisSniiFileReader::ScopedIOContext scope(&io_ctx); + auto status = reader.read_batch(ranges, &outs); + ASSERT_TRUE(status.ok()) << status.to_string(); + } + + ASSERT_EQ(outs.size(), 8); + for (size_t i = 0; i < 8; ++i) { + EXPECT_EQ(as_string(outs[i]), data.substr(i * 8192, 4)); + } + EXPECT_EQ(recording_reader->reads().size(), 8); + EXPECT_EQ(stats.inverted_index_range_read_count, 8); + EXPECT_EQ(stats.inverted_index_serial_read_rounds, 1); +} + +// FB-07: the injected-pool (parallel) path and the seam-nullptr path both produce +// the exact same outputs (and match ground truth) -- parallelism is invisible. +TEST(DorisSniiFileReaderConcurrencyTest, ParallelPathMatchesSerialPath) { + const std::string data = make_pattern(60000); + auto recording_reader = std::make_shared(data); + DorisSniiFileReader reader(recording_reader); + + std::vector<::doris::snii::io::Range> ranges; + std::vector> expected; + for (size_t i = 0; i < 8; ++i) { + const uint64_t off = static_cast(i) * 8192; + ranges.push_back({off, 4}); + const std::string chunk = data.substr(off, 4); + expected.emplace_back(chunk.begin(), chunk.end()); + } + + std::vector> serial_outs; + { + IoPoolSeamGuard seam(nullptr); // no injected pool + auto status = reader.read_batch(ranges, &serial_outs); + ASSERT_TRUE(status.ok()) << status.to_string(); + } + + std::unique_ptr pool; + auto pool_st = + ThreadPoolBuilder("snii_batch_test").set_min_threads(2).set_max_threads(4).build(&pool); + ASSERT_TRUE(pool_st.ok()) << pool_st.to_string(); + + std::vector> parallel_outs; + { + IoPoolSeamGuard seam(pool.get()); + auto status = reader.read_batch(ranges, ¶llel_outs); + ASSERT_TRUE(status.ok()) << status.to_string(); + } + + EXPECT_EQ(serial_outs, expected); + EXPECT_EQ(parallel_outs, expected); + EXPECT_EQ(serial_outs, parallel_outs); +} + +} // namespace doris::segment_v2::snii_doris diff --git a/be/test/storage/index/snii_frq_prelude_test.cpp b/be/test/storage/index/snii_frq_prelude_test.cpp new file mode 100644 index 00000000000000..b45802f6646a6a --- /dev/null +++ b/be/test/storage/index/snii_frq_prelude_test.cpp @@ -0,0 +1,520 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// T18 (FP-01..FP-09): FrqPrelude window-row trim. The trimmed on-disk row drops +// the three running-sum offsets (dd_off/freq_off/prx_off) and the two raw-region +// uncomp_len fields; the reader DERIVES them. These tests pin round-trip +// equivalence of the derived WindowMeta, the exact/shrunken byte size, the +// cross-super-block accumulation, the conditional zstd uncomp_len, corruption on +// truncation / sum-overflow, and end-to-end query equivalence with a smaller +// prelude fetch. + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/varint.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/frq_prelude.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii_query_test_util.h" + +using doris::Status; +using doris::snii::ByteSink; +using doris::snii::Slice; +using doris::snii::varint_len; +using doris::snii::format::build_frq_prelude; +using doris::snii::format::DictEntry; +using doris::snii::format::FrqPreludeColumns; +using doris::snii::format::FrqPreludeReader; +using doris::snii::format::WindowMeta; +using doris::snii::query::term_query; +using doris::snii::reader::LogicalIndexReader; +using doris::snii::reader::SniiSegmentReader; +using doris::snii::snii_test::assert_ok; +using doris::snii::snii_test::build_reader; +using doris::snii::snii_test::MemoryFile; + +namespace { + +// Writer constant (logical_index_writer.cpp kPreludeGroupSize): windows per +// super-block. Used both by the make_windows fixture and to reconstruct the E2E +// term's columns for the size model. +constexpr uint32_t kPreludeGroupSize = 64; + +// --------------------------------------------------------------------------- +// Exact byte-size model of the prelude. row_bytes/prelude_size mirror +// encode_window_row / build_frq_prelude field-for-field so prelude_size(kNew) +// reproduces the real build output exactly (cross-checked by EXPECT_EQ against +// sink.size() in every round-trip test), while prelude_size(kOld) models the +// pre-T18 layout (three extra offset varints + unconditional uncomp_len). +// --------------------------------------------------------------------------- +enum class Layout { kNew, kOld }; + +size_t row_bytes(const WindowMeta& m, uint64_t last_docid_delta, bool has_freq, bool has_prx, + Layout layout) { + const bool old_layout = layout == Layout::kOld; + size_t n = 0; + n += varint_len(last_docid_delta); + n += varint_len(m.doc_count); + n += 1; // win_mode (u8) + if (old_layout) { + n += varint_len(m.dd_off); + } + n += varint_len(m.dd_disk_len); + if (old_layout || m.dd_zstd) { + n += varint_len(m.dd_uncomp_len); + } + n += sizeof(uint32_t); // crc_dd + if (has_freq) { + if (old_layout) { + n += varint_len(m.freq_off); + } + n += varint_len(m.freq_disk_len); + if (old_layout || m.freq_zstd) { + n += varint_len(m.freq_uncomp_len); + } + n += sizeof(uint32_t); // crc_freq + } + if (has_prx) { + if (old_layout) { + n += varint_len(m.prx_off); + } + n += varint_len(m.prx_len); + } + n += varint_len(m.max_freq); + n += 1; // max_norm (u8) + return n; +} + +size_t prelude_size(const FrqPreludeColumns& cols, Layout layout) { + const uint64_t g = cols.group_size; + const size_t n = cols.windows.size(); + const size_t n_super = (g == 0) ? 0 : (n + static_cast(g) - 1) / static_cast(g); + + // Per-super-block window-block byte length + absolute last docid. + std::vector block_len(n_super, 0); + std::vector block_last(n_super, 0); + uint64_t prev_last = 0; + size_t s = 0; + for (size_t start = 0; start < n; start += static_cast(g), ++s) { + const size_t end = std::min(n, start + static_cast(g)); + size_t bl = 0; + for (size_t w = start; w < end; ++w) { + const uint64_t delta = cols.windows[w].last_docid - prev_last; + bl += row_bytes(cols.windows[w], delta, cols.has_freq, cols.has_prx, layout); + prev_last = cols.windows[w].last_docid; + } + block_len[s] = bl; + block_last[s] = prev_last; + } + + // super_block_dir: VInt last_docid_delta + VInt block_off + VInt block_len. + size_t sbdir_len = 0; + uint64_t dir_prev_last = 0; + uint64_t block_off = 0; + for (size_t i = 0; i < n_super; ++i) { + sbdir_len += varint_len(block_last[i] - dir_prev_last); + sbdir_len += varint_len(block_off); + sbdir_len += varint_len(block_len[i]); + dir_prev_last = block_last[i]; + block_off += block_len[i]; + } + + size_t window_region = 0; + for (size_t i = 0; i < n_super; ++i) { + window_region += block_len[i]; + } + + // covered = u8 flags + VInt N + VInt G + VInt n_super + VInt sbdir_len + sbdir. + const size_t covered = 1 + varint_len(n) + varint_len(g) + varint_len(n_super) + + varint_len(sbdir_len) + sbdir_len; + return covered + sizeof(uint32_t) /*crc*/ + window_region; +} + +// Builds n contiguous stride-100 windows with deterministic, all-raw metadata. +// dd/freq/prx offsets are the CONTIGUOUS running prefix sums the reader derives; +// raw uncomp_len == disk_len. Region disk lengths are constant so the NEW row is a +// fixed 16 B (docs+positions) / 10 B (docs-only), while the OLD row's extra offset +// varints grow past one byte for most windows (~24 B / ~13 B). +FrqPreludeColumns make_windows(uint32_t n, uint32_t group_size, bool has_freq, bool has_prx) { + FrqPreludeColumns cols; + cols.has_freq = has_freq; + cols.has_prx = has_prx; + cols.group_size = group_size; + uint64_t dd_run = 0; + uint64_t freq_run = 0; + uint64_t prx_run = 0; + for (uint32_t w = 0; w < n; ++w) { + WindowMeta m; + m.last_docid = (w + 1) * 100 - 1; // window w covers docids [w*100, w*100+99] + m.doc_count = 10; + m.dd_zstd = false; + m.dd_off = dd_run; + m.dd_disk_len = 64; + m.dd_uncomp_len = 64; // raw => == disk_len + m.crc_dd = 0xDD000000U + w; + dd_run += m.dd_disk_len; + if (has_freq) { + m.freq_zstd = false; + m.freq_off = freq_run; + m.freq_disk_len = 48; + m.freq_uncomp_len = 48; + m.crc_freq = 0xEE000000U + w; + freq_run += m.freq_disk_len; + } + if (has_prx) { + m.prx_off = prx_run; + m.prx_len = 96; + prx_run += m.prx_len; + } + m.max_freq = 5; + m.max_norm = 42; + cols.windows.push_back(m); + } + return cols; +} + +// Round-trips columns through build + open, asserting every decoded WindowMeta +// field (incl. the DERIVED dd_off/freq_off/prx_off/uncomp_len) matches the input, +// and that the real build size equals the exact NEW model. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +void expect_round_trip(const FrqPreludeColumns& cols) { + ByteSink sink; + ASSERT_TRUE(build_frq_prelude(cols, &sink).ok()); + + FrqPreludeReader reader; + ASSERT_TRUE(FrqPreludeReader::open(sink.view(), &reader).ok()); + ASSERT_EQ(reader.n_windows(), cols.windows.size()); + EXPECT_EQ(reader.has_freq(), cols.has_freq); + EXPECT_EQ(reader.has_prx(), cols.has_prx); + + uint64_t expect_win_base = 0; + uint64_t dd_run = 0; + uint64_t freq_run = 0; + uint64_t prx_run = 0; + for (uint32_t w = 0; w < reader.n_windows(); ++w) { + WindowMeta got; + ASSERT_TRUE(reader.window(w, &got).ok()) << "w=" << w; + const WindowMeta& exp = cols.windows[w]; + EXPECT_EQ(got.last_docid, exp.last_docid) << "w=" << w; + EXPECT_EQ(got.win_base, expect_win_base) << "w=" << w; + EXPECT_EQ(got.doc_count, exp.doc_count) << "w=" << w; + EXPECT_EQ(got.dd_zstd, exp.dd_zstd) << "w=" << w; + EXPECT_EQ(got.dd_off, exp.dd_off) << "w=" << w; // derived == explicit sum + EXPECT_EQ(got.dd_off, dd_run) << "w=" << w; // ...and == our running sum + EXPECT_EQ(got.dd_disk_len, exp.dd_disk_len) << "w=" << w; + EXPECT_EQ(got.dd_uncomp_len, exp.dd_uncomp_len) << "w=" << w; + EXPECT_EQ(got.crc_dd, exp.crc_dd) << "w=" << w; + dd_run += got.dd_disk_len; + if (cols.has_freq) { + EXPECT_EQ(got.freq_zstd, exp.freq_zstd) << "w=" << w; + EXPECT_EQ(got.freq_off, exp.freq_off) << "w=" << w; + EXPECT_EQ(got.freq_off, freq_run) << "w=" << w; + EXPECT_EQ(got.freq_disk_len, exp.freq_disk_len) << "w=" << w; + EXPECT_EQ(got.freq_uncomp_len, exp.freq_uncomp_len) << "w=" << w; + EXPECT_EQ(got.crc_freq, exp.crc_freq) << "w=" << w; + freq_run += got.freq_disk_len; + } + if (cols.has_prx) { + EXPECT_EQ(got.prx_off, exp.prx_off) << "w=" << w; + EXPECT_EQ(got.prx_off, prx_run) << "w=" << w; + EXPECT_EQ(got.prx_len, exp.prx_len) << "w=" << w; + prx_run += got.prx_len; + } + EXPECT_EQ(got.max_freq, exp.max_freq) << "w=" << w; + EXPECT_EQ(got.max_norm, exp.max_norm) << "w=" << w; + expect_win_base = exp.last_docid; + } + EXPECT_EQ(reader.dd_block_len(), dd_run); + if (cols.has_freq) { + EXPECT_EQ(reader.freq_block_len(), freq_run); + } + EXPECT_EQ(sink.size(), prelude_size(cols, Layout::kNew)); // model tracks the real encoder +} + +} // namespace + +// FP-01: N=200 windows (has_freq+has_prx, all raw) round-trip; the derived +// offsets/uncomp_lens equal the explicit running sums the columns carried. +TEST(SniiFrqPreludeTest, FP01DerivedOffsetsMatchExplicit) { + expect_round_trip(make_windows(/*n=*/200, kPreludeGroupSize, /*has_freq=*/true, + /*has_prx=*/true)); +} + +// FP-02: the trimmed prelude hits an EXACT byte size and is strictly smaller than +// the pre-T18 layout, for both docs+positions and docs-only rows. +TEST(SniiFrqPreludeTest, FP02RowTrimShrinksPreludeBytes) { + // docs+positions: drops 5 varints/row (dd_off, dd_uncomp_len, freq_off, + // freq_uncomp_len, prx_off). + const FrqPreludeColumns dp = make_windows(200, kPreludeGroupSize, /*has_freq=*/true, + /*has_prx=*/true); + ByteSink dp_sink; + ASSERT_TRUE(build_frq_prelude(dp, &dp_sink).ok()); + const size_t dp_new = prelude_size(dp, Layout::kNew); + const size_t dp_old = prelude_size(dp, Layout::kOld); + EXPECT_EQ(dp_sink.size(), dp_new); // exact new size + EXPECT_LT(dp_new, dp_old); // strictly < pre-T18 + EXPECT_GE(dp_old - dp_new, 5U * dp.windows.size()); // >= 5 bytes/row removed + EXPECT_GT((dp_old - dp_new) * 100, dp_old * 25); // ~1/3 saved (30-40% band) + + // docs-only: no freq / no positions -> drops only dd_off + dd_uncomp_len. + const FrqPreludeColumns doc = make_windows(200, kPreludeGroupSize, /*has_freq=*/false, + /*has_prx=*/false); + ByteSink doc_sink; + ASSERT_TRUE(build_frq_prelude(doc, &doc_sink).ok()); + const size_t doc_new = prelude_size(doc, Layout::kNew); + const size_t doc_old = prelude_size(doc, Layout::kOld); + EXPECT_EQ(doc_sink.size(), doc_new); + EXPECT_LT(doc_new, doc_old); + EXPECT_GE(doc_old - doc_new, 2U * doc.windows.size()); // >= 2 bytes/row removed +} + +// FP-03: degenerate empty prelude (N=0) builds + opens with zero windows. +TEST(SniiFrqPreludeTest, FP03EmptyWindows) { + FrqPreludeColumns cols; + cols.has_freq = true; + cols.has_prx = true; + cols.group_size = kPreludeGroupSize; + ByteSink sink; + ASSERT_TRUE(build_frq_prelude(cols, &sink).ok()); + FrqPreludeReader reader; + ASSERT_TRUE(FrqPreludeReader::open(sink.view(), &reader).ok()); + EXPECT_EQ(reader.n_windows(), 0U); + EXPECT_EQ(reader.n_super_blocks(), 0U); + EXPECT_EQ(reader.dd_block_len(), 0U); + EXPECT_EQ(reader.freq_block_len(), 0U); + EXPECT_EQ(sink.size(), prelude_size(cols, Layout::kNew)); +} + +// FP-04: single-window round-trip (win_base=0, one super-block). +TEST(SniiFrqPreludeTest, FP04SingleWindow) { + expect_round_trip( + make_windows(/*n=*/1, kPreludeGroupSize, /*has_freq=*/true, /*has_prx=*/true)); +} + +// FP-05: a df~20000 term (adaptive unit=1024 -> ~20 windows) split into multiple +// super-blocks; the derived dd/freq/prx offsets must accumulate ACROSS block +// boundaries (not reset per block). +TEST(SniiFrqPreludeTest, FP05CrossSuperBlockAccumulation) { + constexpr uint32_t kN = 20; // ceil(20000 / kAdaptiveWindowDocs=1024) + constexpr uint32_t kG = 8; // force 3 super-blocks (8, 8, 4) + const FrqPreludeColumns cols = make_windows(kN, kG, /*has_freq=*/true, /*has_prx=*/true); + ByteSink sink; + ASSERT_TRUE(build_frq_prelude(cols, &sink).ok()); + FrqPreludeReader reader; + ASSERT_TRUE(FrqPreludeReader::open(sink.view(), &reader).ok()); + ASSERT_EQ(reader.n_windows(), kN); + ASSERT_GT(reader.n_super_blocks(), 1U); // genuinely multi-block + + uint64_t dd_run = 0; + uint64_t freq_run = 0; + uint64_t prx_run = 0; + for (uint32_t w = 0; w < kN; ++w) { + WindowMeta got; + ASSERT_TRUE(reader.window(w, &got).ok()) << "w=" << w; + EXPECT_EQ(got.dd_off, dd_run) << "w=" << w; // continuous over all blocks + EXPECT_EQ(got.freq_off, freq_run) << "w=" << w; + EXPECT_EQ(got.prx_off, prx_run) << "w=" << w; + dd_run += got.dd_disk_len; + freq_run += got.freq_disk_len; + prx_run += got.prx_len; + } + EXPECT_EQ(reader.dd_block_len(), dd_run); + EXPECT_EQ(reader.freq_block_len(), freq_run); +} + +// FP-06: a zstd window stores its uncomp_len (!= disk_len) and reads it back, while +// a raw window in the same prelude derives uncomp_len == disk_len (no stored field). +TEST(SniiFrqPreludeTest, FP06ZstdWindowStoresConditionalUncompLen) { + FrqPreludeColumns cols; + cols.has_freq = true; + cols.has_prx = false; + cols.group_size = kPreludeGroupSize; + + WindowMeta z; // window 0: zstd dd + zstd freq, uncomp != disk + z.last_docid = 99; + z.doc_count = 10; + z.dd_zstd = true; + z.dd_off = 0; + z.dd_disk_len = 40; + z.dd_uncomp_len = 137; // meaningful only because dd_zstd + z.crc_dd = 0x0000ABCDU; + z.freq_zstd = true; + z.freq_off = 0; + z.freq_disk_len = 20; + z.freq_uncomp_len = 71; + z.crc_freq = 0x00001234U; + z.max_freq = 3; + z.max_norm = 7; + + WindowMeta r; // window 1: raw dd + raw freq, uncomp == disk + r.last_docid = 199; + r.doc_count = 10; + r.dd_zstd = false; + r.dd_off = z.dd_disk_len; + r.dd_disk_len = 55; + r.dd_uncomp_len = 55; + r.crc_dd = 0x0000BEEFU; + r.freq_zstd = false; + r.freq_off = z.freq_disk_len; + r.freq_disk_len = 25; + r.freq_uncomp_len = 25; + r.crc_freq = 0x00005678U; + r.max_freq = 4; + r.max_norm = 9; + cols.windows = {z, r}; + + ByteSink sink; + ASSERT_TRUE(build_frq_prelude(cols, &sink).ok()); + FrqPreludeReader reader; + ASSERT_TRUE(FrqPreludeReader::open(sink.view(), &reader).ok()); + + WindowMeta g0; + WindowMeta g1; + ASSERT_TRUE(reader.window(0, &g0).ok()); + ASSERT_TRUE(reader.window(1, &g1).ok()); + EXPECT_TRUE(g0.dd_zstd); + EXPECT_EQ(g0.dd_disk_len, 40U); + EXPECT_EQ(g0.dd_uncomp_len, 137U); // stored + read back + EXPECT_NE(g0.dd_uncomp_len, g0.dd_disk_len); + EXPECT_TRUE(g0.freq_zstd); + EXPECT_EQ(g0.freq_uncomp_len, 71U); + EXPECT_NE(g0.freq_uncomp_len, g0.freq_disk_len); + EXPECT_FALSE(g1.dd_zstd); + EXPECT_EQ(g1.dd_uncomp_len, g1.dd_disk_len); // raw: derived, not stored + EXPECT_EQ(g1.freq_uncomp_len, g1.freq_disk_len); + EXPECT_EQ(sink.size(), prelude_size(cols, Layout::kNew)); +} + +// FP-07: a prelude truncated into its window blocks fails to open. +TEST(SniiFrqPreludeTest, FP07TruncatedPreludeIsCorruption) { + const FrqPreludeColumns cols = make_windows(8, 4, /*has_freq=*/true, /*has_prx=*/true); + ByteSink sink; + ASSERT_TRUE(build_frq_prelude(cols, &sink).ok()); + std::vector bytes = sink.buffer(); + ASSERT_GT(bytes.size(), 6U); + bytes.resize(bytes.size() - 6); // chop into the last window block + FrqPreludeReader reader; + const Status s = FrqPreludeReader::open(Slice(bytes), &reader); + EXPECT_TRUE(s.is()) << s.to_string(); +} + +// FP-08: two raw windows whose dd_disk_len values overflow the derived dd-block +// offset accumulator (checked_add_u64) are rejected on open. +TEST(SniiFrqPreludeTest, FP08DdDiskLenSumOverflowIsCorruption) { + FrqPreludeColumns cols; + cols.has_freq = false; + cols.has_prx = false; + cols.group_size = kPreludeGroupSize; + + WindowMeta a; + a.last_docid = 10; + a.doc_count = 1; + a.dd_disk_len = std::numeric_limits::max() / 2 + 100; + a.dd_uncomp_len = a.dd_disk_len; // raw + a.crc_dd = 1; + a.max_freq = 1; + a.max_norm = 0; + WindowMeta b = a; + b.last_docid = 20; // second window's derivation overflows: sum > UINT64_MAX + b.crc_dd = 2; + cols.windows = {a, b}; + + ByteSink sink; + ASSERT_TRUE(build_frq_prelude(cols, &sink).ok()); // builder does not sum-check + FrqPreludeReader reader; + const Status s = FrqPreludeReader::open(sink.view(), &reader); + EXPECT_TRUE(s.is()) << s.to_string(); +} + +// FP-09 (E2E): a high-df windowed term ("failed", 9000 docs) queried through the +// real reader returns the full docid set (new path == old semantics), the docid +// path fetches exactly the (now-smaller) prelude range once, and the on-disk +// prelude_len matches the NEW model and is strictly smaller than the pre-T18 one. +TEST(SniiFrqPreludeTest, FP09WindowedEntryPreludeShrinksAndQueryEquivalent) { + MemoryFile file; + SniiSegmentReader segment_reader; + LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + bool found = false; + DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + assert_ok(index_reader.lookup("failed", &found, &entry, &frq_base, &prx_base)); + ASSERT_TRUE(found); + ASSERT_GT(entry.prelude_len, 0U) << "expected a windowed entry with a prelude"; + + const uint64_t prelude_abs = + index_reader.section_refs().posting_region.offset + frq_base + entry.frq_off_delta; + + // Reconstruct the term's windows from the (new) prelude and model both layouts: + // the on-disk length must equal the NEW model and be strictly below the OLD one. + std::vector prelude_bytes; + assert_ok(file.read_at(prelude_abs, entry.prelude_len, &prelude_bytes)); + FrqPreludeReader prelude; + assert_ok(FrqPreludeReader::open(Slice(prelude_bytes), &prelude)); + ASSERT_GT(prelude.n_windows(), 1U); // genuinely windowed + + FrqPreludeColumns cols; + cols.has_freq = prelude.has_freq(); + cols.has_prx = prelude.has_prx(); + cols.group_size = kPreludeGroupSize; + for (uint32_t w = 0; w < prelude.n_windows(); ++w) { + WindowMeta m; + assert_ok(prelude.window(w, &m)); + cols.windows.push_back(m); + } + EXPECT_EQ(entry.prelude_len, prelude_size(cols, Layout::kNew)); // on-disk == new model + EXPECT_LT(entry.prelude_len, prelude_size(cols, Layout::kOld)); // shrinks vs pre-T18 + + // Functional equivalence + deterministic IO on the docid path. + file.clear_reads(); + std::vector docids; + assert_ok(term_query(index_reader, "failed", &docids)); + + std::vector expected(9000); + std::iota(expected.begin(), expected.end(), 0U); + EXPECT_EQ(docids, expected); + + // The docid path issues ONE coalesced read starting at the prelude and spanning the + // prelude + dd-block (docid-only: the freq-block is not fetched), i.e. a single + // round-trip whose length lies within [prelude_len, frq_len]. The prelude — now the + // shrunk new-model size (prelude_len == new < old, asserted above) — is the head of + // that read, so the end-to-end docid fetch is strictly smaller than pre-T18. + const auto frq_region_reads = + std::ranges::count_if(file.reads(), [&](const MemoryFile::Read& r) { + return r.offset == prelude_abs && r.len >= entry.prelude_len && + r.len <= entry.frq_len; + }); + EXPECT_EQ(frq_region_reads, 1) + << "docid path must fetch the frq region in a single round-trip at the prelude; " + << "prelude_abs=" << prelude_abs << " frq_len=" << entry.frq_len + << " prelude_len=" << entry.prelude_len; +} diff --git a/be/test/storage/index/snii_query_test.cpp b/be/test/storage/index/snii_query_test.cpp new file mode 100644 index 00000000000000..6e4086bf242599 --- /dev/null +++ b/be/test/storage/index/snii_query_test.cpp @@ -0,0 +1,1899 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/pfor.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/format/prx_pod.h" +#include "storage/index/snii/format/tail_pointer.h" +#include "storage/index/snii/io/file_reader.h" +#include "storage/index/snii/io/file_writer.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/query/internal/regex_prefix.h" +#include "storage/index/snii/query/internal/term_expansion.h" +#include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/prefix_query.h" +#include "storage/index/snii/query/regexp_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/query/wildcard_query.h" +// T24 query op-count seam. Define the gate before the include so QueryTestCounters +// is visible in this TU; it is also auto-enabled library-wide by BE_TEST, so the +// phrase_query.cpp increments and the reads below share the same singleton. +#define SNII_QUERY_TEST_COUNTERS +#include "storage/index/snii/query/internal/query_test_counters.h" +#include "storage/index/snii/reader/dict_block_cache.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" +#include "storage/index/snii_query_test_util.h" + +namespace doris::snii::query { +using doris::Status; // RETURN_IF_ERROR expands to bare Status +namespace { + +// Shared reader/writer fixtures live in snii_query_test_util.h so other SNII test +// files can reuse them; pull them into this suite's scope unqualified. +using snii_test::assert_ok; +using snii_test::build_reader; +using snii_test::make_term; +using snii_test::MemoryFile; +using snii_test::PostingDoc; +using snii_test::ScopedEnv; + +class RecordingDocIdSink final : public DocIdSink { +public: + Status append_sorted(std::span docids) override { + out.insert(out.end(), docids.begin(), docids.end()); + return Status::OK(); + } + + Status append_range(uint32_t first, uint64_t last_exclusive) override { + ++range_calls; + for (uint64_t docid = first; docid < last_exclusive; ++docid) { + out.push_back(static_cast(docid)); + } + return Status::OK(); + } + + std::vector out; + size_t range_calls = 0; +}; + +struct PrxRange { + uint64_t offset = 0; + uint64_t len = 0; +}; + +void assert_selective_prx_matches_constant_positions(Slice window, + const std::vector& selected_docs, + uint32_t expected_position) { + std::vector selected_positions; + std::vector selected_offsets; + ByteSource selected_source(window); + assert_ok(format::read_prx_window_csr_selective(&selected_source, selected_docs, + &selected_positions, &selected_offsets)); + + ASSERT_EQ(selected_offsets.size(), selected_docs.size() + 1); + ASSERT_EQ(selected_positions.size(), selected_docs.size()); + for (size_t i = 0; i < selected_docs.size(); ++i) { + EXPECT_EQ(selected_offsets[i], i); + EXPECT_EQ(selected_positions[i], expected_position); + } + EXPECT_EQ(selected_offsets.back(), selected_positions.size()); +} + +// A FileReader decorator that counts read_batch() invocations. Each BatchRangeFetcher +// fetch() barrier issues exactly one read_batch (== one batched/remote serial round), +// so this isolates the number of I/O rounds a query plan emits. Single read_at() calls +// (dict-block / BSBF loads) delegate straight through and are intentionally not counted. +class BatchRoundCountingReader final : public doris::snii::io::FileReader { +public: + explicit BatchRoundCountingReader(doris::snii::io::FileReader* inner) : inner_(inner) {} + + Status read_at(uint64_t offset, size_t len, std::vector* out) override { + return inner_->read_at(offset, len, out); + } + Status read_batch(const std::vector& ranges, + std::vector>* outs) override { + ++batch_rounds_; + return inner_->read_batch(ranges, outs); + } + uint64_t size() const override { return inner_->size(); } + + size_t batch_rounds() const { return batch_rounds_; } + void reset_rounds() { batch_rounds_ = 0; } + +private: + doris::snii::io::FileReader* inner_; + size_t batch_rounds_ = 0; +}; + +// 480 docids with irregular gaps (deterministic LCG): the slim docs region PFOR +// then exceeds the 256B inline threshold so each term becomes a pod_ref, while +// df < 512 keeps it slim (not windowed). This is the layout that forced one PRX +// fetch() per term before T02. +std::vector slim_pod_ref_docids() { + std::vector ids; + ids.reserve(480); + uint32_t cur = 0; + uint32_t state = 0x9e3779b9U; + for (int i = 0; i < 480; ++i) { + ids.push_back(cur); + state = state * 1664525U + 1013904223U; + cur += 1U + (state >> 23) % 250U; // gap in [1, 250] + } + return ids; +} + +// Builds an index with three overlapping slim pod_ref terms ("paa"/"pbb"/"pcc") +// sharing one docid set, each with a single position (5/6/7) so the phrase +// "paa pbb pcc" matches every shared doc. The index is opened through +// `read_through` (e.g. a counting decorator wrapping `file`). `shared_docids` +// returns the docid set, which equals the expected phrase result. +Status build_slim_pod_ref_phrase_reader(MemoryFile* file, doris::snii::io::FileReader* read_through, + reader::SniiSegmentReader* segment_reader, + reader::LogicalIndexReader* index_reader, + std::vector* shared_docids) { + const std::vector docids = slim_pod_ref_docids(); + *shared_docids = docids; + auto make_pos_term = [&](std::string term, uint32_t position) { + std::vector docs; + docs.reserve(docids.size()); + for (uint32_t docid : docids) { + docs.push_back({docid, {position}}); + } + return make_term(std::move(term), std::move(docs)); + }; + + writer::SniiIndexInput input; + input.index_id = 11; + input.index_suffix = "Body"; + input.config = format::IndexConfig::kDocsPositions; + input.doc_count = docids.back() + 1; + input.terms = {make_pos_term("paa", 5), make_pos_term("pbb", 6), make_pos_term("pcc", 7)}; + std::ranges::sort(input.terms, + [](const writer::TermPostings& lhs, const writer::TermPostings& rhs) { + return lhs.term < rhs.term; + }); + + writer::SniiCompoundWriter writer(file); + RETURN_IF_ERROR(writer.add_logical_index(input)); + RETURN_IF_ERROR(writer.finish()); + EXPECT_TRUE(file->finalized()); + + RETURN_IF_ERROR(reader::SniiSegmentReader::open(read_through, segment_reader)); + return segment_reader->open_index(input.index_id, input.index_suffix, index_reader); +} + +writer::SniiIndexInput make_many_term_input(uint64_t index_id, std::string suffix, + uint32_t n_terms) { + writer::SniiIndexInput input; + input.index_id = index_id; + input.index_suffix = std::move(suffix); + input.config = format::IndexConfig::kDocsOnly; + input.doc_count = n_terms + 1; + input.target_dict_block_bytes = 128; + input.terms.reserve(n_terms); + for (uint32_t i = 0; i < n_terms; ++i) { + input.terms.push_back( + make_term("term_" + std::to_string(1000000 + i), {{.docid = i, .positions = {0}}})); + } + return input; +} + +format::TailPointer read_tail_pointer(MemoryFile* file) { + std::vector bytes; + assert_ok(file->read_at(file->size() - format::tail_pointer_size(), format::tail_pointer_size(), + &bytes)); + format::TailPointer tp; + assert_ok(format::decode_tail_pointer(Slice(bytes), &tp)); + return tp; +} + +TEST(SniiSegmentReaderTest, OpenDoesNotReadWholeTailMetaRegion) { + MemoryFile file; + writer::SniiCompoundWriter writer(&file); + assert_ok(writer.add_logical_index(make_many_term_input(7, "Body", 4096))); + assert_ok(writer.finish()); + + const format::TailPointer tp = read_tail_pointer(&file); + file.clear_reads(); + + reader::SniiSegmentReader segment_reader; + assert_ok(reader::SniiSegmentReader::open(&file, &segment_reader)); + + EXPECT_LT(file.read_bytes(), tp.meta_region_length); + for (const auto& read : file.reads()) { + EXPECT_FALSE(read.offset == tp.meta_region_offset && read.len == tp.meta_region_length); + } +} + +TEST(SniiSegmentReaderTest, IndexExistsUsesCachedTailDirectory) { + MemoryFile file; + writer::SniiCompoundWriter writer(&file); + writer::SniiIndexInput input = make_many_term_input(7, "Body", 4096); + assert_ok(writer.add_logical_index(input)); + assert_ok(writer.finish()); + + reader::SniiSegmentReader segment_reader; + assert_ok(reader::SniiSegmentReader::open(&file, &segment_reader)); + + file.clear_reads(); + bool exists = false; + assert_ok(segment_reader.index_exists(input.index_id, input.index_suffix, &exists)); + EXPECT_TRUE(exists); + EXPECT_EQ(file.read_bytes(), 0); + + assert_ok(segment_reader.index_exists(input.index_id + 1, input.index_suffix, &exists)); + EXPECT_FALSE(exists); + EXPECT_EQ(file.read_bytes(), 0); +} + +// F35: memory_usage() feeds the InvertedIndexSearcherCache charge and must now +// account for the resident sampled term index + DICT block directory (previously +// omitted -> under-charge -> over-commit). Exact per-field equality would need +// the reader's private members; the observable public properties asserted here +// are a charge floor (>= sizeof) and monotonic growth with the vocabulary (more +// DICT blocks -> larger sti_/dbd_ heap). The exact heap_bytes() formula the fix +// sums is pinned deterministically by the SampledTermIndex / DictBlockDirectory / +// DictBlock unit tests in snii_writer_test.cpp. +TEST(SniiSegmentReaderTest, MemoryUsageGrowsWithVocabulary) { + auto open_many_term_reader = [](uint32_t n_terms, MemoryFile* file, + reader::SniiSegmentReader* segment_reader, + reader::LogicalIndexReader* index_reader) { + writer::SniiCompoundWriter writer(file); + // target_dict_block_bytes == 128 (make_many_term_input) -> many small DICT + // blocks, so a larger vocabulary yields more sampled terms + block refs. + assert_ok(writer.add_logical_index(make_many_term_input(7, "Body", n_terms))); + assert_ok(writer.finish()); + assert_ok(reader::SniiSegmentReader::open(file, segment_reader)); + assert_ok(segment_reader->open_index(7, "Body", index_reader)); + }; + + MemoryFile small_file; + reader::SniiSegmentReader small_segment; + reader::LogicalIndexReader small_reader; + open_many_term_reader(64, &small_file, &small_segment, &small_reader); + + MemoryFile large_file; + reader::SniiSegmentReader large_segment; + reader::LogicalIndexReader large_reader; + open_many_term_reader(4096, &large_file, &large_segment, &large_reader); + + EXPECT_GE(small_reader.memory_usage(), sizeof(reader::LogicalIndexReader)); + EXPECT_GT(large_reader.memory_usage(), small_reader.memory_usage()); +} + +// G13 end-to-end: a many-term segment's per-index META BLOB (the first serial +// fetch of a cold open, dominated by the sti + dbd tables) must SHRINK on disk +// once those sections are zstd-compressed, the open must therefore fetch fewer +// bytes (MemoryFile::reads()), and every lookup / prefix enumeration must stay +// equal to an uncompressed control built from the identical input. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(SniiSegmentReaderTest, MetaCompressionShrinksOpenFetchAndKeepsLookupsEqual) { + constexpr uint32_t kTerms = 4096; + auto build = [](MemoryFile* file) { + writer::SniiCompoundWriter writer(file); + assert_ok(writer.add_logical_index(make_many_term_input(7, "Body", kTerms))); + assert_ok(writer.finish()); + }; + + MemoryFile compressed_file; + build(&compressed_file); // default: G13 compression active + MemoryFile control_file; + { + ScopedEnv off("SNII_META_COMPRESS_MIN", "18446744073709551615"); + build(&control_file); // pre-G13 layout from the identical input + } + + // The tail meta region (which embeds the per-index meta blob) shrinks. + const format::TailPointer tp_compressed = read_tail_pointer(&compressed_file); + const format::TailPointer tp_control = read_tail_pointer(&control_file); + ASSERT_GT(tp_control.meta_region_length, 0U); + EXPECT_LT(tp_compressed.meta_region_length, tp_control.meta_region_length); + + // So does the single per-index meta blob read_index_meta() fetches. + reader::SniiSegmentReader compressed_segment; + assert_ok(reader::SniiSegmentReader::open(&compressed_file, &compressed_segment)); + reader::SniiSegmentReader control_segment; + assert_ok(reader::SniiSegmentReader::open(&control_file, &control_segment)); + std::vector compressed_meta; + assert_ok(compressed_segment.read_index_meta(7, "Body", &compressed_meta)); + std::vector control_meta; + assert_ok(control_segment.read_index_meta(7, "Body", &control_meta)); + EXPECT_LT(compressed_meta.size(), control_meta.size()); + + // Cold index open fetches fewer bytes end to end (same reads elsewhere: the + // dict blocks / bsbf bytes are identical; only the meta blob differs). + compressed_file.clear_reads(); + reader::LogicalIndexReader compressed_reader; + assert_ok(compressed_segment.open_index(7, "Body", &compressed_reader)); + const size_t compressed_open_bytes = compressed_file.read_bytes(); + + control_file.clear_reads(); + reader::LogicalIndexReader control_reader; + assert_ok(control_segment.open_index(7, "Body", &control_reader)); + const size_t control_open_bytes = control_file.read_bytes(); + EXPECT_LT(compressed_open_bytes, control_open_bytes); + std::cout << "[G13] meta blob: raw=" << control_meta.size() + << "B zstd=" << compressed_meta.size() << "B; open fetch: raw=" << control_open_bytes + << "B zstd=" << compressed_open_bytes << "B\n"; + + // Lookups stay equal across the two layouts: present terms resolve to the + // same entry essentials, absent terms miss on both. + for (uint32_t i = 0; i < kTerms; i += 97) { + const std::string term = "term_" + std::to_string(1000000 + i); + bool found_a = false; + bool found_b = false; + format::DictEntry entry_a; + format::DictEntry entry_b; + uint64_t frq_a = 0; + uint64_t prx_a = 0; + uint64_t frq_b = 0; + uint64_t prx_b = 0; + assert_ok(compressed_reader.lookup(term, &found_a, &entry_a, &frq_a, &prx_a)); + assert_ok(control_reader.lookup(term, &found_b, &entry_b, &frq_b, &prx_b)); + ASSERT_TRUE(found_a) << term; + ASSERT_TRUE(found_b) << term; + EXPECT_EQ(entry_a.term, entry_b.term); + EXPECT_EQ(entry_a.kind, entry_b.kind); + EXPECT_EQ(entry_a.df, entry_b.df); + EXPECT_EQ(frq_a, frq_b); + EXPECT_EQ(prx_a, prx_b); + } + bool found_absent = true; + format::DictEntry absent_entry; + uint64_t frq = 0; + uint64_t prx = 0; + assert_ok( + compressed_reader.lookup("zzzz_not_indexed", &found_absent, &absent_entry, &frq, &prx)); + EXPECT_FALSE(found_absent); + + // Ordered prefix enumeration walks sti + dbd + dict blocks on both layouts + // and must produce the identical term sequence. + std::vector hits_a; + std::vector hits_b; + assert_ok(compressed_reader.prefix_terms("term_10001", &hits_a)); + assert_ok(control_reader.prefix_terms("term_10001", &hits_b)); + ASSERT_EQ(hits_a.size(), hits_b.size()); + ASSERT_EQ(hits_a.size(), 100U); // term_1000100 .. term_1000199 + for (size_t i = 0; i < hits_a.size(); ++i) { + EXPECT_EQ(hits_a[i].term, hits_b[i].term); + EXPECT_EQ(hits_a[i].entry.df, hits_b[i].entry.df); + } +} + +// P1 cold-read fix: a NON-resident bloom is skipped entirely. open() must not +// read the 28B header (which the old L1 path cached) and lookup() must not issue a +// 32B body probe; absent / present terms still resolve correctly via sti -> dict. +TEST(SniiSegmentReaderTest, NonResidentBsbfIsSkippedNotProbed) { + ScopedEnv disable_resident_bsbf("SNII_BSBF_RESIDENT_MAX", "0"); + + MemoryFile file; + writer::SniiCompoundWriter writer(&file); + writer::SniiIndexInput input = make_many_term_input(7, "Body", 1024); + assert_ok(writer.add_logical_index(input)); + assert_ok(writer.finish()); + + reader::SniiSegmentReader segment_reader; + assert_ok(reader::SniiSegmentReader::open(&file, &segment_reader)); + format::SectionRefs refs; + assert_ok(segment_reader.section_refs_for_index(input.index_id, input.index_suffix, &refs)); + ASSERT_GT(refs.bsbf.length, format::kBsbfHeaderSize); // a real filter exists on disk + const uint64_t bsbf_end = refs.bsbf.offset + refs.bsbf.length; + auto touches_bsbf = [&](uint64_t offset, size_t len) { + return offset < bsbf_end && refs.bsbf.offset < offset + len; + }; + + // open() must NOT touch the bsbf section at all on the non-resident path. + file.clear_reads(); + reader::LogicalIndexReader index_reader; + assert_ok(segment_reader.open_index(input.index_id, input.index_suffix, &index_reader)); + for (const auto& read : file.reads()) { + EXPECT_FALSE(touches_bsbf(read.offset, read.len)) + << "non-resident open must skip the bsbf header"; + } + + // An absent term still resolves to empty via sti -> dict, and the lookup must + // NOT probe the bsbf section. + file.clear_reads(); + std::vector docids; + assert_ok(term_query(index_reader, "absent_term", &docids)); + EXPECT_TRUE(docids.empty()); + for (const auto& read : file.reads()) { + EXPECT_FALSE(touches_bsbf(read.offset, read.len)) + << "non-resident lookup must not probe the bsbf section"; + } + + // A present term is still found via sti -> dict (correctness with no bloom). + std::vector present; + assert_ok(term_query(index_reader, "term_1000000", &present)); + EXPECT_FALSE(present.empty()); +} + +TEST(SniiSegmentReaderTest, LogicalIndexOpenCachesResidentMetadataAndSmallHeaders) { + ScopedEnv resident_bsbf("SNII_BSBF_RESIDENT_MAX", "1048576"); + ScopedEnv resident_dict("SNII_DICT_RESIDENT_MAX", "1048576"); + + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + file.clear_reads(); + bool found = false; + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + assert_ok(index_reader.lookup("failed", &found, &entry, &frq_base, &prx_base)); + EXPECT_TRUE(found); + EXPECT_EQ(file.read_bytes(), 0); + + std::vector hits; + assert_ok(index_reader.prefix_terms("ord", &hits, 10)); + ASSERT_EQ(hits.size(), 2); + EXPECT_EQ(hits[0].term, "order"); + EXPECT_EQ(hits[1].term, "ordinal"); + EXPECT_EQ(file.read_bytes(), 0); +} + +TEST(SniiPhraseQueryTest, WindowedPhraseQueryKeepsCorrectCandidateOrdinals) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + std::vector docids; + assert_ok(phrase_query(index_reader, {"failed", "order"}, &docids)); + + const std::vector expected {5000, 7000, 8000}; + EXPECT_EQ(docids, expected); +} + +TEST(SniiPhraseQueryTest, WindowedPhrasePrefixQueryKeepsCorrectCandidateOrdinals) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + std::vector docids; + assert_ok(phrase_prefix_query(index_reader, {"failed", "ord"}, &docids, 10)); + + const std::vector expected {5000, 6000, 7000, 8000}; + EXPECT_EQ(docids, expected); +} + +TEST(SniiPhraseQueryTest, SingleTailPhrasePrefixUsesStreamingPhrasePath) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + std::vector docids; + assert_ok(phrase_prefix_query(index_reader, {"failed", "orde"}, &docids, 10)); + + const std::vector expected {5000, 7000, 8000}; + EXPECT_EQ(docids, expected); +} + +// PV1 (deterministic perf): a 3-term slim pod_ref phrase issues exactly one batched +// PRX round. read_batch count = 1 (round1 docs) + 1 (shared PRX fetch) = 2; the +// docid conjunction decodes slim docs from round1 (no extra round) and dict/BSBF +// loads use uncounted read_at. Before T02 this was 4 (round1 + one PRX fetch per +// term). +TEST(SniiPhraseQueryTest, PhraseQueryIssuesSinglePrxBatchRound) { + MemoryFile file; + BatchRoundCountingReader counting(&file); + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + std::vector shared_docids; + assert_ok(build_slim_pod_ref_phrase_reader(&file, &counting, &segment_reader, &index_reader, + &shared_docids)); + + // Guard: every phrase term must be a slim pod_ref (not inline, not windowed), + // otherwise the per-term PRX fetch this test isolates would not occur. + for (std::string_view term : {"paa", "pbb", "pcc"}) { + bool found = false; + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + assert_ok(index_reader.lookup(term, &found, &entry, &frq_base, &prx_base)); + ASSERT_TRUE(found) << term; + EXPECT_EQ(entry.kind, format::DictEntryKind::kPodRef) << term; + EXPECT_EQ(entry.enc, format::DictEntryEnc::kSlim) << term; + } + + counting.reset_rounds(); + std::vector docids; + assert_ok(phrase_query(index_reader, {"paa", "pbb", "pcc"}, &docids)); + + EXPECT_EQ(docids, shared_docids); + EXPECT_EQ(counting.batch_rounds(), 2U); +} + +// FV3 (functional equivalence): the shared single-batch PRX path returns the result +// dictated by the position layout across three overlapping slim pod_ref terms. A +// backfill mistake (wrong plan/chunk/handle mapping) would misread a term's +// positions and drop docs. +TEST(SniiPhraseQueryTest, ThreeTermPhraseMatchesAcrossSharedPrxFetch) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + std::vector shared_docids; + assert_ok(build_slim_pod_ref_phrase_reader(&file, &file, &segment_reader, &index_reader, + &shared_docids)); + + std::vector docids; + assert_ok(phrase_query(index_reader, {"paa", "pbb", "pcc"}, &docids)); + EXPECT_EQ(docids, shared_docids); + + // Same terms, non-consecutive order (positions 7,6,5): the candidate set is + // identical but no doc satisfies the phrase, so the shared PRX fetch must yield + // an empty result. + std::vector reversed; + assert_ok(phrase_query(index_reader, {"pcc", "pbb", "paa"}, &reversed)); + EXPECT_TRUE(reversed.empty()); +} + +TEST(SniiPhraseQueryTest, TwoTermPhraseUsesHiddenBigramPosting) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader, + /*include_phrase_bigrams=*/true)); + + const auto original_prx_span = [&](std::string_view term) { + bool found = false; + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + assert_ok(index_reader.lookup(term, &found, &entry, &frq_base, &prx_base)); + EXPECT_TRUE(found); + return PrxRange {.offset = index_reader.section_refs().posting_region.offset + prx_base + + entry.prx_off_delta, + .len = entry.prx_len}; + }; + const std::vector original_prx { + original_prx_span("failed"), + original_prx_span("order"), + }; + + file.clear_reads(); + std::vector docids; + assert_ok(phrase_query(index_reader, {"failed", "order"}, &docids)); + + const std::vector expected {5000, 7000, 8000}; + EXPECT_EQ(docids, expected); + for (const PrxRange& prx : original_prx) { + const bool original_prx_read = std::ranges::any_of(file.reads(), [&](const auto& read) { + return read.offset < prx.offset + prx.len && prx.offset < read.offset + read.len; + }); + EXPECT_FALSE(original_prx_read); + } +} + +TEST(SniiPhraseQueryTest, TwoTermPhraseWithNonIndexableTermFallsBackToPositions) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader, + /*include_phrase_bigrams=*/true)); + + std::vector docids; + assert_ok(phrase_query(index_reader, {"trace", "123"}, &docids)); + + const std::vector expected {42}; + EXPECT_EQ(docids, expected); +} + +TEST(SniiPhraseQueryTest, TwoTermPhrasePrefixWorksWithHiddenBigramFormat) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader, + /*include_phrase_bigrams=*/true)); + + std::vector docids; + assert_ok(phrase_prefix_query(index_reader, {"failed", "ord"}, &docids, 10)); + + const std::vector expected {5000, 6000, 7000, 8000}; + EXPECT_EQ(docids, expected); +} + +TEST(SniiPhraseQueryTest, PrefixQueryDoesNotExposeHiddenBigramTerms) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader, + /*include_phrase_bigrams=*/true)); + + std::vector docids; + assert_ok(prefix_query(index_reader, std::string(format::kPhraseBigramTermMarker), &docids)); + EXPECT_TRUE(docids.empty()); +} + +TEST(SniiPhraseQueryTest, WildcardQueryDoesNotExposeHiddenBigramTerms) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader, + /*include_phrase_bigrams=*/true)); + + std::vector docids; + assert_ok(wildcard_query(index_reader, "*failed*order*", &docids)); + EXPECT_TRUE(docids.empty()); +} + +TEST(SniiPhraseQueryTest, RegexpQueryDoesNotExposeHiddenBigramTerms) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader, + /*include_phrase_bigrams=*/true)); + + std::vector docids; + assert_ok(regexp_query(index_reader, ".*failed.*order.*", &docids)); + EXPECT_TRUE(docids.empty()); +} + +std::vector all_docids_0_to(uint32_t end_exclusive) { + std::vector docids(end_exclusive); + std::iota(docids.begin(), docids.end(), 0U); + return docids; +} + +// RQ-01: a plain literal pattern is anchored at both ends (RE2::FullMatch), so it +// matches exactly the "order" term and returns its full docid range. +TEST(SniiRegexpQueryTest, MatchesAnchoredLiteralTerm) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + std::vector docids; + assert_ok(regexp_query(index_reader, "order", &docids)); + EXPECT_EQ(docids, all_docids_0_to(9000)); +} + +// RQ-03: alternation under a shared prefix expands to "order" and "ordinal"; both +// span the full docid range, so the union must dedup back to [0..8999]. +TEST(SniiRegexpQueryTest, CharClassMatchesMultipleTermsDeduped) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + std::vector docids; + assert_ok(regexp_query(index_reader, "ord(er|inal)", &docids)); + EXPECT_EQ(docids, all_docids_0_to(9000)); +} + +// Alternation across non-adjacent terms with disjoint docids yields a sorted, +// deduplicated union. +TEST(SniiRegexpQueryTest, AlternationUnionsDistinctTerms) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + std::vector docids; + assert_ok(regexp_query(index_reader, "(123|needle)", &docids)); + const std::vector expected {42, 100, 101, 102, 6000}; + EXPECT_EQ(docids, expected); +} + +// RQ-04: a non-existent prefix enumerates nothing and returns an empty result. +TEST(SniiRegexpQueryTest, NoTermMatchesEmpty) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + std::vector docids; + assert_ok(regexp_query(index_reader, "zzz.*", &docids)); + EXPECT_TRUE(docids.empty()); +} + +// RQ-05: a character-class pattern matches only the numeric "123" term. +TEST(SniiRegexpQueryTest, NumericTermMatch) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + std::vector docids; + assert_ok(regexp_query(index_reader, "[0-9]+", &docids)); + const std::vector expected {42}; + EXPECT_EQ(docids, expected); +} + +// RQ-06: backreferences are unsupported by RE2 (but accepted by std::regex); +// re.ok() is false so the call returns InvalidArgument without throwing. +TEST(SniiRegexpQueryTest, InvalidPatternReturnsInvalidArgument) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + std::vector docids; + const Status status = regexp_query(index_reader, "(a)\\1", &docids); + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(status.is()) << status.to_string(); +} + +// RQ-07: a syntactically invalid pattern also returns InvalidArgument. +TEST(SniiRegexpQueryTest, UnbalancedParenReturnsInvalidArgument) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + std::vector docids; + const Status status = regexp_query(index_reader, "(order", &docids); + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(status.is()) << status.to_string(); +} + +// RQ-08: null output / null sink return InvalidArgument (no crash, no throw). +TEST(SniiRegexpQueryTest, NullOutputReturnsInvalidArgument) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + std::vector* const null_docids = nullptr; + EXPECT_TRUE(regexp_query(index_reader, "order", null_docids) + .is()); + + DocIdSink* const null_sink = nullptr; + EXPECT_TRUE(regexp_query(index_reader, "order", null_sink) + .is()); +} + +// RQ-09: max_expansions caps the number of expanded terms. Terms enumerate in +// sorted order, so the first non-bigram term "123" is the only expansion. +TEST(SniiRegexpQueryTest, MaxExpansionsCapsExpansion) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + std::vector docids; + assert_ok(regexp_query(index_reader, ".*", &docids, /*max_expansions=*/1)); + const std::vector expected {42}; + EXPECT_EQ(docids, expected); +} + +// RQ-02/RQ-10: hidden phrase-bigram terms must never leak. ".*failed.*order.*" +// would FullMatch the bigram term text "failedorder" if it were visible, +// but the bigram filter hides it, so the result is empty. +TEST(SniiRegexpQueryTest, DoesNotExposeHiddenBigramTerms) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader, + /*include_phrase_bigrams=*/true)); + + std::vector bigram_leak; + assert_ok(regexp_query(index_reader, ".*failed.*order.*", &bigram_leak)); + EXPECT_TRUE(bigram_leak.empty()); + + // A match-all pattern returns only real-term docids (bigram terms filtered). + std::vector match_all; + assert_ok(regexp_query(index_reader, ".*", &match_all)); + EXPECT_EQ(match_all, all_docids_0_to(9000)); +} + +// RQ-11: the RE2 path reproduces the std::regex_match golden result set exactly. +TEST(SniiRegexpQueryTest, EquivalenceMatchesStdRegexGolden) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + struct GoldenCase { + std::string_view pattern; + std::vector expected; + }; + const std::vector cases { + {.pattern = "order", .expected = all_docids_0_to(9000)}, + {.pattern = "ord(er|inal)", .expected = all_docids_0_to(9000)}, + {.pattern = "zzz.*", .expected = {}}, + {.pattern = "[0-9]+", .expected = {42}}, + {.pattern = "(123|needle)", .expected = {42, 100, 101, 102, 6000}}, + }; + for (const GoldenCase& c : cases) { + std::vector docids; + assert_ok(regexp_query(index_reader, c.pattern, &docids)); + EXPECT_EQ(docids, c.expected) << "pattern=" << c.pattern; + } +} + +// Deterministic perf (golden prefix): RE2::PossibleMatchRange tightens the +// enumeration prefix for left-anchored patterns whose literal scan stops early. +TEST(SniiRegexpQueryTest, AnchoredPrefixIsTightened) { + auto prefix_of = [](std::string_view pattern) -> std::string { + re2::RE2::Options options; + options.set_log_errors(false); + const re2::RE2 re(re2::StringPiece(pattern.data(), pattern.size()), options); + EXPECT_TRUE(re.ok()) << pattern; + return internal::regex_enum_prefix(pattern, re); + }; + + // Tightened beyond the naive literal scan (which would yield ""). + EXPECT_EQ(prefix_of("^(order)"), "order"); + EXPECT_EQ(prefix_of("^(order|ordinal)"), "ord"); + EXPECT_EQ(prefix_of("^ord[ei]"), "ord"); + // Anchored literal already maximal. + EXPECT_EQ(prefix_of("^order"), "order"); + // Non-anchored patterns fall back to the conservative literal scan. + EXPECT_EQ(prefix_of("ord.*"), "ord"); + EXPECT_EQ(prefix_of(".*failed.*order.*"), ""); + EXPECT_EQ(prefix_of("[0-9]+"), ""); +} + +// Deterministic perf (op-count): the tightened "^(order)" prefix reaches a single +// dictionary term, so the matcher is invoked once instead of once per term. +TEST(SniiRegexpQueryTest, AnchoredPrefixEnumeratesSingleTerm) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + constexpr std::string_view kPattern = "^(order)"; + re2::RE2::Options options; + options.set_log_errors(false); + const re2::RE2 re(re2::StringPiece(kPattern.data(), kPattern.size()), options); + ASSERT_TRUE(re.ok()); + + const std::string enum_prefix = internal::regex_enum_prefix(kPattern, re); + EXPECT_EQ(enum_prefix, "order"); + + auto count_matcher = [&](std::string_view prefix) { + int calls = 0; + std::vector docids; + VectorDocIdSink sink(docids); + assert_ok(internal::emit_expanded_docid_union( + index_reader, prefix, + [&](std::string_view term) { + ++calls; + return re2::RE2::FullMatch(re2::StringPiece(term.data(), term.size()), re); + }, + &sink)); + return std::pair> {calls, std::move(docids)}; + }; + + // Tightened prefix enumerates only "order" -> exactly one matcher call. + auto [tight_calls, tight_docids] = count_matcher(enum_prefix); + EXPECT_EQ(tight_calls, 1); + + // The baseline empty prefix (old behavior) scans all 11 dictionary terms. + auto [full_calls, full_docids] = count_matcher(""); + EXPECT_EQ(full_calls, 11); + + // Narrowing is a pure optimization: identical result either way. + EXPECT_EQ(tight_docids, all_docids_0_to(9000)); + EXPECT_EQ(full_docids, all_docids_0_to(9000)); +} + +TEST(SniiPhraseQueryTest, MultiTailPhrasePrefixFiltersTailPrxByExpectedDocs) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + std::vector tail_hits; + assert_ok(index_reader.prefix_terms("ord", &tail_hits, 10)); + ASSERT_EQ(tail_hits.size(), 2); + + struct PrxRange { + uint64_t offset = 0; + uint64_t len = 0; + }; + std::vector full_tail_prx_ranges; + for (const auto& hit : tail_hits) { + full_tail_prx_ranges.push_back({index_reader.section_refs().posting_region.offset + + hit.prx_base + hit.entry.prx_off_delta, + hit.entry.prx_len}); + } + + file.clear_reads(); + std::vector docids; + assert_ok(phrase_prefix_query(index_reader, {"needle", "ord"}, &docids, 10)); + + const std::vector expected {6000}; + EXPECT_EQ(docids, expected); + for (const PrxRange& prx : full_tail_prx_ranges) { + const bool full_tail_prx_read = std::ranges::any_of(file.reads(), [&](const auto& read) { + return read.offset == prx.offset && read.len == prx.len; + }); + EXPECT_FALSE(full_tail_prx_read); + } +} + +// --------------------------------------------------------------------------- +// T24: phrase-prefix micro-opt (sparsest anchor + expected_docids hoist). +// +// build_reader is shared across many SNII suites, so instead of mutating it we +// build small, self-contained kDocsPositions indexes here. Each scenario uses a +// distinct tail prefix so prefix_terms() expansion stays isolated per test, and +// the deterministic op-counters in query_test_counters.h are read directly. +// --------------------------------------------------------------------------- +Status build_positions_reader(MemoryFile* file, reader::SniiSegmentReader* segment_reader, + reader::LogicalIndexReader* index_reader, + std::vector terms, uint32_t doc_count) { + writer::SniiIndexInput input; + input.index_id = 21; + input.index_suffix = "Body"; + input.config = format::IndexConfig::kDocsPositions; + input.doc_count = doc_count; + input.terms = std::move(terms); + std::ranges::sort(input.terms, + [](const writer::TermPostings& lhs, const writer::TermPostings& rhs) { + return lhs.term < rhs.term; + }); + + writer::SniiCompoundWriter writer(file); + RETURN_IF_ERROR(writer.add_logical_index(input)); + RETURN_IF_ERROR(writer.finish()); + EXPECT_TRUE(file->finalized()); + + RETURN_IF_ERROR(reader::SniiSegmentReader::open(file, segment_reader)); + return segment_reader->open_index(input.index_id, input.index_suffix, index_reader); +} + +// "lead mid axt*": the leading exact term is high-frequency (5 positions/doc) while +// the middle exact term is a single position/doc -> "mid" is the sparsest exact +// term and becomes the anchor. doc400 is a valid "lead mid" candidate (expected +// tail@8) with NO tail term, so it must be filtered out of the final result. +// NOLINTBEGIN(modernize-use-designated-initializers): positional aggregate init of test posting data +std::vector anchor_scenario_terms() { + return {make_term("lead", {{100, {0, 3, 6, 9, 12}}, + {200, {0, 3, 6, 9, 12}}, + {300, {0, 3, 6, 9, 12}}, + {400, {0, 3, 6, 9, 12}}}), + make_term("mid", {{100, {1}}, {200, {7}}, {300, {7}}, {400, {7}}}), + make_term("axta", {{100, {2}}}), // doc100: lead@0, mid@1 -> tail@2 + make_term("axtb", {{200, {8}}}), // doc200: lead@6, mid@7 -> tail@8 + make_term("axtc", {{300, {8}}})}; // doc300: lead@6, mid@7 -> tail@8 +} + +// "dlead dmid dxt*": dlead is a single position/doc (sparsest), dmid is +// high-frequency. The anchor therefore stays at phrase position 0 (dlead), which +// is exactly the old span[0] behavior -- the degenerate no-change case. +std::vector leading_sparse_scenario_terms() { + return {make_term("dlead", {{500, {3}}, {600, {3}}}), + make_term("dmid", {{500, {0, 2, 4, 6, 8}}, {600, {0, 2, 4, 6, 8}}}), + make_term("dxta", {{500, {5}}}), // dlead@3, dmid@4 -> tail@5 + make_term("dxtb", {{600, {5}}})}; +} + +// "ulead umid uxt*": umid (single position) is the anchor. In doc800 umid sits at +// position 0, which is < its phrase offset (1), so a general anchor would underflow +// `start`; the underflow guard skips it and doc800 is correctly excluded. +std::vector anchor_underflow_scenario_terms() { + return {make_term("ulead", {{800, {5, 9}}, {810, {5, 9}}, {820, {5, 9}}}), + make_term("umid", {{800, {0}}, {810, {6}}, {820, {6}}}), + make_term("uxta", {{810, {7}}}), // ulead@5, umid@6 -> tail@7 + make_term("uxtb", {{820, {7}}})}; +} +// NOLINTEND(modernize-use-designated-initializers) + +// FUNC-1: the new sparsest-anchor path produces the correct result set. The anchor +// (mid) is not phrase-position 0, exercising the general anchor formula + underflow +// guard; doc400 (candidate, no tail term) is filtered out. +TEST(SniiPhraseQueryTest, MultiTermPhrasePrefixAnchorsOnSparsestTerm) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_positions_reader(&file, &segment_reader, &index_reader, anchor_scenario_terms(), + /*doc_count=*/500)); + + std::vector docids; + assert_ok(phrase_prefix_query(index_reader, {"lead", "mid", "axt"}, &docids, 10)); + + const std::vector expected {100, 200, 300}; + EXPECT_EQ(docids, expected); +} + +// Perf (deterministic): the outer anchor enumeration size == docs x min_span. With +// mid as the anchor (1 position/doc) over 4 candidate docs the count is 4, strictly +// below the old span[0] baseline of Sum|lead| == 4 x 5 == 20. +TEST(SniiPhraseQueryTest, MultiTermPhrasePrefixAnchorIterationsMinimal) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_positions_reader(&file, &segment_reader, &index_reader, anchor_scenario_terms(), + /*doc_count=*/500)); + + internal::query_test_counters() = internal::QueryTestCounters {}; + std::vector docids; + assert_ok(phrase_prefix_query(index_reader, {"lead", "mid", "axt"}, &docids, 10)); + + constexpr uint64_t kCandidateDocs = 4; // {100,200,300,400} + constexpr uint64_t kMinSpan = 1; // mid has one position per doc + constexpr uint64_t kOldLeadSpanTotal = kCandidateDocs * 5; // Sum|span[0]| (lead@5pos) + EXPECT_EQ(internal::query_test_counters().anchor_iterations, kCandidateDocs * kMinSpan); + EXPECT_LT(internal::query_test_counters().anchor_iterations, kOldLeadSpanTotal); +} + +// FUNC-3: when the leading exact term is already the sparsest, the anchor stays at +// phrase-position 0, so anchor_iterations == Sum|span[0]| exactly (no regression / +// no change vs. the old hardcoded anchor). Result is still correct. +TEST(SniiPhraseQueryTest, MultiTermPhrasePrefixLeadingTermSparsestIsDegenerate) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_positions_reader(&file, &segment_reader, &index_reader, + leading_sparse_scenario_terms(), /*doc_count=*/700)); + + internal::query_test_counters() = internal::QueryTestCounters {}; + std::vector docids; + assert_ok(phrase_prefix_query(index_reader, {"dlead", "dmid", "dxt"}, &docids, 10)); + + const std::vector expected {500, 600}; + EXPECT_EQ(docids, expected); + // dlead has one position/doc over 2 docs -> Sum|span[0]| == 2, unchanged. + EXPECT_EQ(internal::query_test_counters().anchor_iterations, 2U); +} + +// FUNC-4: a doc whose anchor position is smaller than the anchor's phrase offset +// (umid@0, offset 1) is skipped by the underflow guard and never false-matches; the +// two well-formed docs still match via distinct tails. +TEST(SniiPhraseQueryTest, MultiTermPhrasePrefixSkipsAnchorUnderflowDoc) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_positions_reader(&file, &segment_reader, &index_reader, + anchor_underflow_scenario_terms(), /*doc_count=*/900)); + + internal::query_test_counters() = internal::QueryTestCounters {}; + std::vector docids; + assert_ok(phrase_prefix_query(index_reader, {"ulead", "umid", "uxt"}, &docids, 10)); + + const std::vector expected {810, 820}; // doc800 excluded (underflow) + EXPECT_EQ(docids, expected); + // 3 candidates {800,810,820}, umid is the single-position anchor -> 3 x 1. + EXPECT_EQ(internal::query_test_counters().anchor_iterations, 3U); +} + +// Perf (deterministic): the multi-tail branch materializes expected_docids exactly +// once per query (hoisted out of the per-tail loop). Here prefix "axt" expands to 3 +// tails; the old per-tail rebuild would have counted 3. +TEST(SniiPhraseQueryTest, MultiTailPhrasePrefixBuildsExpectedDocidsOnce) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_positions_reader(&file, &segment_reader, &index_reader, anchor_scenario_terms(), + /*doc_count=*/500)); + + std::vector tail_hits; + assert_ok(index_reader.prefix_terms("axt", &tail_hits, 10)); + ASSERT_EQ(tail_hits.size(), 3); // axta, axtb, axtc -> multi-tail branch + + internal::query_test_counters() = internal::QueryTestCounters {}; + std::vector docids; + assert_ok(phrase_prefix_query(index_reader, {"lead", "mid", "axt"}, &docids, 10)); + + const std::vector expected {100, 200, 300}; + EXPECT_EQ(docids, expected); + EXPECT_EQ(internal::query_test_counters().expected_docids_build, 1U); +} + +// FUNC-6: a single tail expansion takes the streaming ExecuteResolvedPhraseTerms +// path, never the multi-tail branch, so expected_docids_build stays 0. Result is +// unchanged from SingleTailPhrasePrefixUsesStreamingPhrasePath. +TEST(SniiPhraseQueryTest, SingleTailPhrasePrefixDoesNotBuildExpectedDocids) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + internal::query_test_counters() = internal::QueryTestCounters {}; + std::vector docids; + assert_ok(phrase_prefix_query(index_reader, {"failed", "orde"}, &docids, 10)); + + const std::vector expected {5000, 7000, 8000}; + EXPECT_EQ(docids, expected); + EXPECT_EQ(internal::query_test_counters().expected_docids_build, 0U); +} + +// FUNC-5: an empty tail expansion returns OK with an empty result before the +// multi-tail branch, so no expected_docids vector is built. +TEST(SniiPhraseQueryTest, MultiTermPhrasePrefixEmptyTailExpansionReturnsEmpty) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_positions_reader(&file, &segment_reader, &index_reader, anchor_scenario_terms(), + /*doc_count=*/500)); + + internal::query_test_counters() = internal::QueryTestCounters {}; + std::vector docids; + assert_ok(phrase_prefix_query(index_reader, {"lead", "mid", "zzz"}, &docids, 10)); + + EXPECT_TRUE(docids.empty()); + EXPECT_EQ(internal::query_test_counters().expected_docids_build, 0U); +} + +// FUNC-7: a null output pointer returns InvalidArgument (no crash, no throw). +TEST(SniiPhraseQueryTest, PhrasePrefixQueryNullOutReturnsInvalidArgument) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_positions_reader(&file, &segment_reader, &index_reader, anchor_scenario_terms(), + /*doc_count=*/500)); + + std::vector* const null_docids = nullptr; + EXPECT_TRUE(phrase_prefix_query(index_reader, {"lead", "mid", "axt"}, null_docids, 10) + .is()); +} + +// FUNC-8: hidden phrase-bigram terms never leak through the multi-tail prefix path; +// result matches the non-bigram layout. +TEST(SniiPhraseQueryTest, MultiTailPhrasePrefixWithHiddenBigramsDoesNotLeak) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader, /*include_phrase_bigrams=*/true)); + + std::vector docids; + assert_ok(phrase_prefix_query(index_reader, {"failed", "ord"}, &docids, 10)); + + const std::vector expected {5000, 6000, 7000, 8000}; + EXPECT_EQ(docids, expected); +} + +// FUNC-2: byte-for-byte result equivalence on the existing fixture across the three +// canonical phrase-prefix shapes (multi-tail single-exact, single-tail, multi-tail +// single-doc). Locks the sparsest-anchor + hoist changes as pure optimizations. +TEST(SniiPhraseQueryTest, MultiTailPhrasePrefixEquivalenceRegression) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + std::vector failed_ord; + assert_ok(phrase_prefix_query(index_reader, {"failed", "ord"}, &failed_ord, 10)); + EXPECT_EQ(failed_ord, (std::vector {5000, 6000, 7000, 8000})); + + std::vector failed_orde; + assert_ok(phrase_prefix_query(index_reader, {"failed", "orde"}, &failed_orde, 10)); + EXPECT_EQ(failed_orde, (std::vector {5000, 7000, 8000})); + + std::vector needle_ord; + assert_ok(phrase_prefix_query(index_reader, {"needle", "ord"}, &needle_ord, 10)); + EXPECT_EQ(needle_ord, (std::vector {6000})); +} + +// Perf (deterministic, corroborating): the CPU-only anchor reorder + expected_docids +// hoist do not change which bytes are fetched (spans/candidates are materialized +// before anchor selection). Two independently-built identical readers therefore +// issue an identical, non-zero number of physical reads for the same query. +TEST(SniiPhraseQueryTest, MultiTermPhrasePrefixDoesNotRegressReadCount) { + MemoryFile file_a; + reader::SniiSegmentReader segment_a; + reader::LogicalIndexReader index_a; + assert_ok(build_positions_reader(&file_a, &segment_a, &index_a, anchor_scenario_terms(), + /*doc_count=*/500)); + file_a.clear_reads(); + std::vector docids_a; + assert_ok(phrase_prefix_query(index_a, {"lead", "mid", "axt"}, &docids_a, 10)); + const size_t reads_a = file_a.reads().size(); + + MemoryFile file_b; + reader::SniiSegmentReader segment_b; + reader::LogicalIndexReader index_b; + assert_ok(build_positions_reader(&file_b, &segment_b, &index_b, anchor_scenario_terms(), + /*doc_count=*/500)); + file_b.clear_reads(); + std::vector docids_b; + assert_ok(phrase_prefix_query(index_b, {"lead", "mid", "axt"}, &docids_b, 10)); + const size_t reads_b = file_b.reads().size(); + + EXPECT_EQ(docids_a, docids_b); + // T24 is a CPU-only change (sparsest-term anchor + expected_docids hoist), so the + // query's physical IO must be unchanged. The small anchor-scenario index is fully + // resident, so the read count is a deterministic constant across identical readers. + EXPECT_EQ(reads_a, reads_b); +} + +TEST(SniiPhraseQueryTest, MultiTermPhraseUsesPairPrefilter) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + std::vector docids; + assert_ok(phrase_query(index_reader, {"failed", "order", "ordinal"}, &docids)); + + const std::vector expected {5000, 7000}; + EXPECT_EQ(docids, expected); +} + +TEST(SniiPhraseQueryTest, RepeatedTermPhraseUsesCachedPostingSpan) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + std::vector docids; + assert_ok(phrase_query(index_reader, {"repeat", "repeat", "repeat"}, &docids)); + + std::vector expected(9000); + std::iota(expected.begin(), expected.end(), 0); + EXPECT_EQ(docids, expected); +} + +TEST(SniiPhraseQueryTest, DenseTermWithMissingDocKeepsCandidateOrdinals) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + std::vector driver_docids; + assert_ok(term_query(index_reader, "driver", &driver_docids)); + EXPECT_EQ(driver_docids.size(), 8000); + + std::vector almost_docids; + assert_ok(term_query(index_reader, "almost", &almost_docids)); + EXPECT_EQ(almost_docids.size(), 8999); + ASSERT_GT(almost_docids.size(), 6144); + EXPECT_EQ(almost_docids[3999], 3999); + EXPECT_EQ(almost_docids[4000], 4001); + EXPECT_EQ(almost_docids[6143], 6144); + EXPECT_EQ(almost_docids[6144], 6145); + + std::vector docids; + assert_ok(phrase_query(index_reader, {"driver", "almost"}, &docids)); + + std::vector expected; + expected.reserve(7999); + for (uint32_t docid = 0; docid < 8000; ++docid) { + if (docid != 4000) { + expected.push_back(docid); + } + } + EXPECT_EQ(docids, expected); +} + +TEST(SniiPhraseQueryTest, SparseWindowBitsetKeepsCandidateOrdinals) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + std::vector docids; + assert_ok(phrase_query(index_reader, {"sparse_left", "sparse_right"}, &docids)); + + std::vector expected; + for (uint32_t docid = 0; docid < 9000; ++docid) { + if (docid % 3 == 0 && docid % 4 != 1) { + expected.push_back(docid); + } + } + EXPECT_EQ(docids, expected); +} + +TEST(SniiTermQueryTest, WindowedDenseTermEmitsRangesToSink) { + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + RecordingDocIdSink sink; + assert_ok(term_query(index_reader, "failed", &sink)); + + std::vector expected(9000); + std::iota(expected.begin(), expected.end(), 0); + EXPECT_EQ(sink.out, expected); + EXPECT_GT(sink.range_calls, 0); +} + +TEST(SniiPrxPodTest, SelectivePforCsrMatchesFullCsrAcrossRuns) { + std::vector freqs; + std::vector positions; + freqs.reserve(320); + for (uint32_t doc = 0; doc < 320; ++doc) { + const uint32_t freq = (doc % 5 == 0) ? 2 : 1; + freqs.push_back(freq); + positions.push_back(doc * 3); + if (freq == 2) { + positions.push_back(doc * 3 + 2); + } + } + + ByteSink sink; + assert_ok(format::build_prx_window_flat(positions, freqs, -1, &sink)); + + std::vector full_positions; + std::vector full_offsets; + ByteSource full_source(sink.view()); + assert_ok(format::read_prx_window_csr(&full_source, &full_positions, &full_offsets)); + + auto assert_selected_matches_full = [&](const std::vector& selected_docs) { + std::vector selected_positions; + std::vector selected_offsets; + ByteSource selected_source(sink.view()); + assert_ok(format::read_prx_window_csr_selective(&selected_source, selected_docs, + &selected_positions, &selected_offsets)); + + ASSERT_EQ(selected_offsets.size(), selected_docs.size() + 1); + for (size_t i = 0; i < selected_docs.size(); ++i) { + const uint32_t doc = selected_docs[i]; + const std::vector expected(full_positions.begin() + full_offsets[doc], + full_positions.begin() + full_offsets[doc + 1]); + const std::vector actual( + selected_positions.begin() + selected_offsets[i], + selected_positions.begin() + selected_offsets[i + 1]); + EXPECT_EQ(actual, expected); + } + }; + + assert_selected_matches_full({0, 1, 2}); + assert_selected_matches_full({0, 1, 127, 128, 129, 255, 256, 319}); +} + +TEST(SniiPrxPodTest, SelectivePforCsrHandlesDocsSpanningPforRuns) { + const std::vector freqs {300, 1, 260, 2, 1}; + std::vector positions; + positions.reserve(564); + auto append_doc = [&](uint32_t count, uint32_t base, uint32_t seed) { + uint32_t pos = base; + uint32_t state = seed; + for (uint32_t i = 0; i < count; ++i) { + state = state * 1664525 + 1013904223; + pos += 1 + (state & 0xFFFF); + positions.push_back(pos); + } + }; + append_doc(freqs[0], 3, 11); + append_doc(freqs[1], 11, 13); + append_doc(freqs[2], 19, 17); + append_doc(freqs[3], 29, 19); + append_doc(freqs[4], 37, 23); + + ByteSink sink; + assert_ok(format::build_prx_window_flat(positions, freqs, -1, &sink)); + ASSERT_FALSE(sink.buffer().empty()); + ASSERT_EQ(sink.buffer().front(), static_cast(format::PrxCodec::kPfor)); + + std::vector full_positions; + std::vector full_offsets; + ByteSource full_source(sink.view()); + assert_ok(format::read_prx_window_csr(&full_source, &full_positions, &full_offsets)); + + const std::vector selected_docs {0, 2, 3}; + std::vector selected_positions; + std::vector selected_offsets; + ByteSource selected_source(sink.view()); + assert_ok(format::read_prx_window_csr_selective(&selected_source, selected_docs, + &selected_positions, &selected_offsets)); + + ASSERT_EQ(selected_offsets.size(), selected_docs.size() + 1); + for (size_t i = 0; i < selected_docs.size(); ++i) { + const uint32_t doc = selected_docs[i]; + const std::vector expected(full_positions.begin() + full_offsets[doc], + full_positions.begin() + full_offsets[doc + 1]); + const std::vector actual(selected_positions.begin() + selected_offsets[i], + selected_positions.begin() + selected_offsets[i + 1]); + EXPECT_EQ(actual, expected); + } +} + +TEST(SniiPrxPodTest, AutoCodecUsesZstdWhenItIsSmaller) { + std::vector freqs(1024, 1); + std::vector positions(1024, 7); + + ByteSink sink; + assert_ok(format::build_prx_window_flat(positions, freqs, -1, &sink)); + + ASSERT_FALSE(sink.buffer().empty()); + EXPECT_EQ(sink.buffer().front(), static_cast(format::PrxCodec::kZstd)); + + std::vector decoded_positions; + std::vector decoded_offsets; + ByteSource source(sink.view()); + assert_ok(format::read_prx_window_csr(&source, &decoded_positions, &decoded_offsets)); + + ASSERT_EQ(decoded_offsets.size(), freqs.size() + 1); + ASSERT_EQ(decoded_positions.size(), positions.size()); + for (size_t i = 0; i < freqs.size(); ++i) { + EXPECT_EQ(decoded_offsets[i], i); + EXPECT_EQ(decoded_positions[i], 7); + } + EXPECT_EQ(decoded_offsets.back(), positions.size()); + + const std::vector selected_docs {0, 17, 511, 1023}; + assert_selective_prx_matches_constant_positions(sink.view(), selected_docs, 7); +} + +TEST(SniiPrxPodTest, AutoCodecKeepsPforForTinyWindows) { + const std::vector freqs {1, 2}; + const std::vector positions {3, 5, 8}; + + ByteSink sink; + assert_ok(format::build_prx_window_flat(positions, freqs, -1, &sink)); + + ASSERT_FALSE(sink.buffer().empty()); + EXPECT_EQ(sink.buffer().front(), static_cast(format::PrxCodec::kPfor)); +} + +TEST(SniiPforTest, LowBitWidthFastPathsRoundTrip) { + auto assert_round_trip = [](const std::vector& values, uint8_t expected_width) { + ByteSink sink; + doris::snii::pfor_encode(values.data(), values.size(), &sink); + ASSERT_FALSE(sink.buffer().empty()); + EXPECT_EQ(sink.buffer().front(), expected_width); + + std::vector decoded(values.size(), 0xFFFFFFFF); + ByteSource source(sink.view()); + assert_ok(doris::snii::pfor_decode(&source, values.size(), decoded.data())); + EXPECT_TRUE(source.eof()); + EXPECT_EQ(decoded, values); + }; + + std::vector one_bit(128); + for (size_t i = 0; i < one_bit.size(); ++i) { + one_bit[i] = static_cast(i & 1); + } + assert_round_trip(one_bit, 1); + + one_bit[17] = 1000; + assert_round_trip(one_bit, 1); + + std::vector two_bit(128); + for (size_t i = 0; i < two_bit.size(); ++i) { + two_bit[i] = static_cast(i & 3); + } + assert_round_trip(two_bit, 2); + + std::vector three_bit(131); + for (size_t i = 0; i < three_bit.size(); ++i) { + three_bit[i] = static_cast(i & 7); + } + assert_round_trip(three_bit, 3); + + std::vector four_bit(128); + for (size_t i = 0; i < four_bit.size(); ++i) { + four_bit[i] = static_cast(i & 15); + } + assert_round_trip(four_bit, 4); + + std::vector five_bit(129); + for (size_t i = 0; i < five_bit.size(); ++i) { + five_bit[i] = static_cast(i & 31); + } + assert_round_trip(five_bit, 5); + + std::vector six_bit(130); + for (size_t i = 0; i < six_bit.size(); ++i) { + six_bit[i] = static_cast(i & 63); + } + assert_round_trip(six_bit, 6); + + std::vector seven_bit(131); + for (size_t i = 0; i < seven_bit.size(); ++i) { + seven_bit[i] = static_cast(i & 127); + } + assert_round_trip(seven_bit, 7); + + std::vector eight_bit(256); + for (size_t i = 0; i < eight_bit.size(); ++i) { + eight_bit[i] = static_cast(i); + } + assert_round_trip(eight_bit, 8); +} + +// =========================================================================== +// T04 -- DICT block request-scoped cache (MRU) + resident single-range read. +// =========================================================================== + +namespace t04 { + +std::string many_term_key(uint32_t i) { + return "term_" + std::to_string(1000000 + i); +} + +// Builds an index of `n_terms` tiny docs-only terms with a small target block +// size so the dictionary spans several DICT blocks, opened through `read_through`. +Status build_multi_block_reader(MemoryFile* file, doris::snii::io::FileReader* read_through, + reader::SniiSegmentReader* segment_reader, + reader::LogicalIndexReader* index_reader, uint32_t n_terms) { + writer::SniiIndexInput input = make_many_term_input(21, "Body", n_terms); + std::ranges::sort(input.terms, + [](const writer::TermPostings& lhs, const writer::TermPostings& rhs) { + return lhs.term < rhs.term; + }); + writer::SniiCompoundWriter writer(file); + RETURN_IF_ERROR(writer.add_logical_index(input)); + RETURN_IF_ERROR(writer.finish()); + EXPECT_TRUE(file->finalized()); + RETURN_IF_ERROR(reader::SniiSegmentReader::open(read_through, segment_reader)); + return segment_reader->open_index(input.index_id, input.index_suffix, index_reader); +} + +// Serializes read_at calls so the recording MemoryFile can be shared by the +// concurrency test without racing on its own bookkeeping. Test infra only -- the +// production FileReader (Doris IO / S3) is itself concurrent-read safe; this lock +// is NOT part of the reader under test and never wraps a decode. +class LockedFileReader final : public doris::snii::io::FileReader { +public: + explicit LockedFileReader(doris::snii::io::FileReader* inner) : inner_(inner) {} + Status read_at(uint64_t offset, size_t len, std::vector* out) override { + std::lock_guard guard(mu_); + return inner_->read_at(offset, len, out); + } + uint64_t size() const override { return inner_->size(); } + +private: + doris::snii::io::FileReader* inner_; + std::mutex mu_; +}; + +// Captures everything lookup() returns so resident / on-demand / cached paths can +// be asserted byte-identical. +struct LookupResult { + bool found = false; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + std::string term; + uint64_t frq_off_delta = 0; + uint64_t frq_len = 0; + uint64_t df = 0; + bool operator==(const LookupResult&) const = default; +}; + +LookupResult do_lookup(const reader::LogicalIndexReader& idx, std::string_view term, + reader::DictBlockCache* cache) { + bool found = false; + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + const Status st = idx.lookup(term, &found, &entry, &frq_base, &prx_base, cache); + EXPECT_TRUE(st.ok()) << st.to_string(); + LookupResult r; + r.found = found; + if (found) { + r.frq_base = frq_base; + r.prx_base = prx_base; + r.term = entry.term; + r.frq_off_delta = entry.frq_off_delta; + r.frq_len = entry.frq_len; + r.df = static_cast(entry.df); + } + return r; +} + +size_t count_reads_in_region(const MemoryFile& file, const format::RegionRef& region, + bool* one_covers_region) { + size_t count = 0; + *one_covers_region = false; + for (const MemoryFile::Read& r : file.reads()) { + if (r.offset >= region.offset && r.offset < region.offset + region.length) { + ++count; + if (r.offset == region.offset && r.len == region.length) { + *one_covers_region = true; + } + } + } + return count; +} + +} // namespace t04 + +// F10: a resident dictionary that spans several blocks is loaded with ONE range +// read over dict_region (was one read_at per block -> up to ~4 serial S3 rounds). +TEST(SniiLogicalReaderTest, ResidentDictLoadIssuesSingleRangeRead) { + ScopedEnv resident_dict("SNII_DICT_RESIDENT_MAX", "1048576"); + ScopedEnv resident_bsbf("SNII_BSBF_RESIDENT_MAX", "1048576"); + + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(t04::build_multi_block_reader(&file, &file, &segment_reader, &index_reader, 64)); + + const format::RegionRef& dict_region = index_reader.section_refs().dict_region; + ASSERT_GT(dict_region.length, 0U); + bool one_covers_region = false; + const size_t region_reads = t04::count_reads_in_region(file, dict_region, &one_covers_region); + EXPECT_EQ(region_reads, 1U); + EXPECT_TRUE(one_covers_region); +} + +// F08/F20 + perf gate: forced on-demand, a block hit K times by repeated lookups +// AND a block shared by several distinct terms of one query each decode ONCE when +// a single request-scoped cache is threaded through -- dict_decode_counter() == +// unique_blocks (1 here; build_reader's small dictionary is a single block). +TEST(SniiLogicalReaderTest, OnDemandLookupDecompressesBlockOncePerUniqueBlock) { + ScopedEnv on_demand("SNII_DICT_RESIDENT_MAX", "0"); + + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + doris::snii::testing::reset_dict_decode_counter(); + reader::DictBlockCache cache; + for (int i = 0; i < 5; ++i) { + const t04::LookupResult r = t04::do_lookup(index_reader, "failed", &cache); + EXPECT_TRUE(r.found); + } + EXPECT_EQ(doris::snii::testing::dict_decode_counter(), 1U); +} + +// The headline gate: a multi-term query whose terms fall in the same DICT block +// decodes it ONCE with a shared cache, and once-per-term without one. +TEST(SniiLogicalReaderTest, SharedDictBlockDecodesOncePerQuery) { + ScopedEnv on_demand("SNII_DICT_RESIDENT_MAX", "0"); + + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + const std::vector terms = {"failed", "order", "driver"}; + + // With one request-scoped cache: the shared block decodes once. + doris::snii::testing::reset_dict_decode_counter(); + reader::DictBlockCache cache; + for (std::string_view t : terms) { + EXPECT_TRUE(t04::do_lookup(index_reader, t, &cache).found); + } + EXPECT_EQ(doris::snii::testing::dict_decode_counter(), 1U); // == unique_blocks + + // Baseline (no cache): each term re-decodes the same block. + doris::snii::testing::reset_dict_decode_counter(); + for (std::string_view t : terms) { + EXPECT_TRUE(t04::do_lookup(index_reader, t, nullptr).found); + } + EXPECT_EQ(doris::snii::testing::dict_decode_counter(), terms.size()); +} + +// New/old equivalence: the on-demand + cache path returns exactly what the +// resident path does -- lookups and full term_query docid sets are identical. +TEST(SniiLogicalReaderTest, OnDemandResultsMatchResidentBaseline) { + const std::vector present = {"failed", "order", "ordinal", + "driver", "needle", "almost"}; + const std::string_view absent = "definitely_absent_term"; + + std::vector resident_lookups; + std::vector> resident_docids; + { + ScopedEnv resident_dict("SNII_DICT_RESIDENT_MAX", "1048576"); + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + for (std::string_view t : present) { + resident_lookups.push_back(t04::do_lookup(index_reader, t, nullptr)); + std::vector docids; + assert_ok(term_query(index_reader, t, &docids)); + resident_docids.push_back(std::move(docids)); + } + EXPECT_FALSE(t04::do_lookup(index_reader, absent, nullptr).found); + } + + ScopedEnv on_demand("SNII_DICT_RESIDENT_MAX", "0"); + MemoryFile file; + reader::SniiSegmentReader segment_reader; + reader::LogicalIndexReader index_reader; + assert_ok(build_reader(&file, &segment_reader, &index_reader)); + + reader::DictBlockCache cache; + for (size_t i = 0; i < present.size(); ++i) { + // cached vs uncached on-demand both equal the resident baseline. + EXPECT_EQ(t04::do_lookup(index_reader, present[i], &cache), resident_lookups[i]); + EXPECT_EQ(t04::do_lookup(index_reader, present[i], nullptr), resident_lookups[i]); + std::vector docids; + assert_ok(term_query(index_reader, present[i], &docids)); + EXPECT_EQ(docids, resident_docids[i]); + } + EXPECT_FALSE(t04::do_lookup(index_reader, absent, &cache).found); +} + +// F-05: prefix enumeration across many on-demand blocks is correct and a second +// pass over the same cache adds no decodes (cross-call request-scoped reuse). +TEST(SniiLogicalReaderTest, PrefixEnumerationReusesCachedBlocks) { + constexpr uint32_t kTerms = 64; + + // Resident baseline ordering. + std::vector resident_terms; + { + ScopedEnv resident_dict("SNII_DICT_RESIDENT_MAX", "1048576"); + MemoryFile file; + reader::SniiSegmentReader seg; + reader::LogicalIndexReader idx; + assert_ok(t04::build_multi_block_reader(&file, &file, &seg, &idx, kTerms)); + std::vector hits; + assert_ok(idx.prefix_terms("term_", &hits, 0)); + for (auto& h : hits) { + resident_terms.push_back(h.term); + } + } + ASSERT_EQ(resident_terms.size(), kTerms); + + ScopedEnv on_demand("SNII_DICT_RESIDENT_MAX", "0"); + MemoryFile file; + reader::SniiSegmentReader seg; + reader::LogicalIndexReader idx; + assert_ok(t04::build_multi_block_reader(&file, &file, &seg, &idx, kTerms)); + + reader::DictBlockCache cache(/*max_entries=*/128); + doris::snii::testing::reset_dict_decode_counter(); + + std::vector hits1; + assert_ok(idx.prefix_terms("term_", &hits1, 0, &cache)); + const uint64_t blocks = doris::snii::testing::dict_decode_counter(); + EXPECT_GE(blocks, 2U); // genuinely multi-block (else the reuse gate is vacuous) + + std::vector hits2; + assert_ok(idx.prefix_terms("term_", &hits2, 0, &cache)); + EXPECT_EQ(doris::snii::testing::dict_decode_counter(), blocks); // second pass: no re-decode + + std::vector got1; + std::vector got2; + for (auto& h : hits1) { + got1.push_back(h.term); + } + for (auto& h : hits2) { + got2.push_back(h.term); + } + EXPECT_EQ(got1, resident_terms); + EXPECT_EQ(got2, resident_terms); +} + +// Request-scoped cache unit: MRU promotion, LRU eviction, a hard size bound, and +// pins that keep an evicted block alive -- no file IO involved. +TEST(SniiLogicalReaderTest, RequestCacheEvictsLruStaysBoundedAndPinsSurvive) { + reader::DictBlockCache cache(/*max_entries=*/2); + int loads = 0; + auto loader_for = [&](uint8_t tag) { + return [&loads, tag](std::shared_ptr* out) -> Status { + ++loads; + auto block = std::make_shared(); + block->bytes.assign(4, tag); + *out = block; + return Status::OK(); + }; + }; + + std::shared_ptr pin0; + assert_ok(cache.get_or_load(0, loader_for(0), &pin0)); + EXPECT_EQ(loads, 1); + std::shared_ptr tmp; + assert_ok(cache.get_or_load(1, loader_for(1), &tmp)); // {1,0} + EXPECT_EQ(loads, 2); + assert_ok(cache.get_or_load(0, loader_for(0), &tmp)); // hit -> {0,1} + EXPECT_EQ(loads, 2); + assert_ok(cache.get_or_load(2, loader_for(2), &tmp)); // miss -> evict LRU(1) -> {2,0} + EXPECT_EQ(loads, 3); + EXPECT_EQ(cache.size(), 2U); + + assert_ok(cache.get_or_load(0, loader_for(0), &tmp)); // 0 still resident -> hit + EXPECT_EQ(loads, 3); + assert_ok(cache.get_or_load(1, loader_for(1), &tmp)); // 1 was evicted -> reload + EXPECT_EQ(loads, 4); + EXPECT_LE(cache.size(), cache.max_entries()); + + // pin0 was taken before 0 ever cycled; even across evictions its bytes stay live. + ASSERT_TRUE(pin0); + ASSERT_EQ(pin0->bytes.size(), 4U); + EXPECT_EQ(pin0->bytes[0], 0); +} + +// F-06: with a 1-entry cache, alternating two different blocks forces reloads; +// a reloaded block still decodes correctly (Slice not corrupted by eviction). +TEST(SniiLogicalReaderTest, SmallCacheReloadsEvictedBlockCorrectly) { + ScopedEnv on_demand("SNII_DICT_RESIDENT_MAX", "0"); + constexpr uint32_t kTerms = 64; + + MemoryFile file; + reader::SniiSegmentReader seg; + reader::LogicalIndexReader idx; + assert_ok(t04::build_multi_block_reader(&file, &file, &seg, &idx, kTerms)); + + const std::string first = t04::many_term_key(0); // smallest -> block 0 + const std::string last = t04::many_term_key(kTerms - 1); // largest -> last block + + reader::DictBlockCache cache(/*max_entries=*/1); + doris::snii::testing::reset_dict_decode_counter(); + + const t04::LookupResult first_a = t04::do_lookup(idx, first, &cache); + const uint64_t after_first = doris::snii::testing::dict_decode_counter(); + EXPECT_TRUE(t04::do_lookup(idx, last, &cache).found); // evicts block 0 + const t04::LookupResult first_b = t04::do_lookup(idx, first, &cache); // reload block 0 + const uint64_t after_reload = doris::snii::testing::dict_decode_counter(); + + EXPECT_TRUE(first_a.found); + EXPECT_EQ(first_a, first_b); // reload produced the identical entry + EXPECT_GT(after_reload, after_first); // a reload actually happened (evicted) + EXPECT_LE(cache.size(), cache.max_entries()); +} + +// Concurrency: N queries share the const reader, each with its OWN request-scoped +// cache. No shared mutable state is added to the reader, so every thread decodes +// the on-demand block once -> dict_decode_counter() == thread count, results are +// correct, and the run is TSAN-clean (BUILD_TYPE_UT=TSAN). +TEST(SniiLogicalReaderConcurrencyTest, ConcurrentQueriesUseIndependentRequestCaches) { + ScopedEnv on_demand("SNII_DICT_RESIDENT_MAX", "0"); + + MemoryFile file; + reader::SniiSegmentReader seg0; + reader::LogicalIndexReader idx0; + assert_ok(build_reader(&file, &seg0, &idx0)); // populate `file` + + t04::LockedFileReader locked(&file); + reader::SniiSegmentReader seg; + assert_ok(reader::SniiSegmentReader::open(&locked, &seg)); + reader::LogicalIndexReader idx; + assert_ok(seg.open_index(7, "Body", &idx)); + + doris::snii::testing::reset_dict_decode_counter(); + constexpr int kThreads = 8; + constexpr int kIters = 16; + std::atomic failures {0}; + auto worker = [&]() { + reader::DictBlockCache cache; // request-scoped: one per query/thread + for (int i = 0; i < kIters; ++i) { + bool found = false; + format::DictEntry entry; + uint64_t frq_base = 0; + uint64_t prx_base = 0; + const Status st = idx.lookup("failed", &found, &entry, &frq_base, &prx_base, &cache); + if (!st.ok() || !found) { + failures.fetch_add(1, std::memory_order_relaxed); + } + } + }; + + std::vector threads; + threads.reserve(kThreads); + for (int t = 0; t < kThreads; ++t) { + threads.emplace_back(worker); + } + for (std::thread& t : threads) { + t.join(); + } + + EXPECT_EQ(failures.load(), 0); + // Each thread's cache decodes the shared block exactly once. + EXPECT_EQ(doris::snii::testing::dict_decode_counter(), static_cast(kThreads)); +} + +} // namespace +} // namespace doris::snii::query diff --git a/be/test/storage/index/snii_query_test_util.h b/be/test/storage/index/snii_query_test_util.h new file mode 100644 index 00000000000000..5dee5d498ef917 --- /dev/null +++ b/be/test/storage/index/snii_query_test_util.h @@ -0,0 +1,279 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +// Shared gtest fixtures for the SNII reader/writer test suites. Other test files +// reuse these by including this header and pulling the symbols in with +// `using namespace doris::snii::snii_test;` (see snii_query_test.cpp). + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/io/file_reader.h" +#include "storage/index/snii/io/file_writer.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +namespace doris::snii::snii_test { +using doris::Status; // RETURN_IF_ERROR / Status::OK() expand to a bare Status below. + +// An in-memory FileReader+FileWriter that records every read() (offset/len) so +// snii-layer round-trips and exact-range assertions can be made against it. +class MemoryFile final : public doris::snii::io::FileReader, public doris::snii::io::FileWriter { +public: + struct Read { + uint64_t offset = 0; + size_t len = 0; + }; + + Status append(Slice data) override { + data_.insert(data_.end(), data.data(), data.data() + data.size()); + return Status::OK(); + } + + Status finalize() override { + finalized_ = true; + return Status::OK(); + } + + uint64_t bytes_written() const override { return data_.size(); } + + // NOLINTBEGIN(readability-non-const-parameter): FileReader interface writes into out. + Status read_at(uint64_t offset, size_t len, std::vector* out) override { + if (offset > data_.size() || len > data_.size() - offset) { + return Status::Corruption("memory file read past eof"); + } + reads_.push_back({offset, len}); + read_bytes_ += len; + out->resize(len); + if (len != 0) { + std::memcpy(out->data(), data_.data() + offset, len); + } + return Status::OK(); + } + // NOLINTEND(readability-non-const-parameter) + + uint64_t size() const override { return data_.size(); } + bool finalized() const { return finalized_; } + const std::vector& reads() const { return reads_; } + size_t read_bytes() const { return read_bytes_; } + void clear_reads() { + reads_.clear(); + read_bytes_ = 0; + } + +private: + std::vector data_; + std::vector reads_; + size_t read_bytes_ = 0; + bool finalized_ = false; +}; + +// RAII helper that sets an environment variable for the duration of a test and +// restores (or clears) the previous value on destruction. +class ScopedEnv { +public: + ScopedEnv(const char* key, const char* value) : key_(key) { + if (const char* old = std::getenv(key); old != nullptr) { + old_value_ = old; + } + setenv(key, value, 1); + } + + ~ScopedEnv() { + if (old_value_.has_value()) { + setenv(key_, old_value_->c_str(), 1); + } else { + unsetenv(key_); + } + } + +private: + const char* key_; + std::optional old_value_; +}; + +// A single document's postings (docid + its in-doc positions) used to drive the +// writer fixtures below. +struct PostingDoc { + uint32_t docid = 0; + std::vector positions; +}; + +inline writer::TermPostings make_term(std::string term, std::vector docs) { + std::ranges::sort(docs, [](const PostingDoc& lhs, const PostingDoc& rhs) { + return lhs.docid < rhs.docid; + }); + + writer::TermPostings posting; + posting.term = std::move(term); + posting.docids.reserve(docs.size()); + posting.freqs.reserve(docs.size()); + for (const PostingDoc& doc : docs) { + posting.docids.push_back(doc.docid); + posting.freqs.push_back(static_cast(doc.positions.size())); + posting.positions_flat.insert(posting.positions_flat.end(), doc.positions.begin(), + doc.positions.end()); + } + return posting; +} + +inline std::vector docs_with_one_position(uint32_t begin, uint32_t end, + uint32_t position) { + std::vector docs; + docs.reserve(end - begin); + for (uint32_t docid = begin; docid < end; ++docid) { + docs.push_back({docid, {position}}); + } + return docs; +} + +inline void assert_ok(const Status& status) { + ASSERT_TRUE(status.ok()) << status.to_string(); +} + +// The standard reader-side fixture: a 9000-doc kDocsPositions index whose terms +// exercise dense/sparse/windowed/tail postings, optionally including the hidden +// phrase-bigram terms. Opens `segment_reader`/`index_reader` over `file`. +// `bigram_prune_min_df` != 0 builds a G01 df-PRUNED segment: bigram terms with +// df below it are dropped at materialization, survivors are written docs-only, +// and the per-index meta records the threshold (reader fallback gate). 0 (the +// default) keeps the legacy layout byte-identical to the pre-G01 fixture. +// `bigram_prune_max_df` != 0 additionally arms the G15 UPPER gate: bigram +// terms with df strictly above it are dropped, and the meta records the max +// (the reader fallback gate keys on either threshold being non-zero). +inline Status build_reader(MemoryFile* file, reader::SniiSegmentReader* segment_reader, + reader::LogicalIndexReader* index_reader, + bool include_phrase_bigrams = false, uint32_t bigram_prune_min_df = 0, + uint64_t bigram_prune_max_df = 0) { + constexpr uint32_t kDocCount = 9000; + auto failed_docs = docs_with_one_position(0, kDocCount, 0); + auto order_docs = docs_with_one_position(0, kDocCount, 2); + auto ordinal_docs = docs_with_one_position(0, kDocCount, 2); + auto driver_docs = docs_with_one_position(0, 8000, 0); + auto almost_docs = docs_with_one_position(0, kDocCount, 1); + std::vector needle_docs {{.docid = 100, .positions = {0}}, + {.docid = 101, .positions = {0}}, + {.docid = 102, .positions = {0}}, + {.docid = 6000, .positions = {0}}}; + std::vector numeric_tail_docs {{.docid = 42, .positions = {1}}}; + std::vector sparse_left_docs; + std::vector sparse_right_docs; + std::vector repeat_docs; + std::vector trace_docs {{.docid = 42, .positions = {0}}}; + sparse_left_docs.reserve(kDocCount / 3 + 1); + sparse_right_docs.reserve(kDocCount); + repeat_docs.reserve(kDocCount); + for (uint32_t docid = 0; docid < kDocCount; ++docid) { + if (docid % 3 == 0) { + sparse_left_docs.push_back({docid, {0}}); + } + if (docid % 4 != 1) { + sparse_right_docs.push_back({docid, {1}}); + } + repeat_docs.push_back({docid, {0, 1, 2}}); + } + almost_docs.erase(almost_docs.begin() + 4000); + failed_docs[8000].positions = {0, 4}; + for (PostingDoc& doc : order_docs) { + if (doc.docid == 5000 || doc.docid == 7000) { + doc.positions = {1}; + } else if (doc.docid == 8000) { + doc.positions = {5}; + } + } + for (PostingDoc& doc : ordinal_docs) { + if (doc.docid == 6000) { + doc.positions = {1}; + } + } + + writer::SniiIndexInput input; + input.index_id = 7; + input.index_suffix = "Body"; + input.config = format::IndexConfig::kDocsPositions; + input.doc_count = kDocCount; + input.bigram_prune_min_df = bigram_prune_min_df; + input.bigram_prune_max_df = bigram_prune_max_df; + input.terms = {make_term("almost", std::move(almost_docs)), + make_term("123", std::move(numeric_tail_docs)), + make_term("driver", std::move(driver_docs)), + make_term("failed", std::move(failed_docs)), + make_term("needle", std::move(needle_docs)), + make_term("order", std::move(order_docs)), + make_term("ordinal", std::move(ordinal_docs)), + make_term("repeat", std::move(repeat_docs)), + make_term("sparse_left", std::move(sparse_left_docs)), + make_term("sparse_right", std::move(sparse_right_docs)), + make_term("trace", std::move(trace_docs))}; + if (include_phrase_bigrams) { + input.terms.push_back(make_term(format::make_phrase_bigram_sentinel_term(), + {{.docid = 0, .positions = {0}}})); + input.terms.push_back(make_term(format::make_phrase_bigram_term("failed", "order"), + {{.docid = 5000, .positions = {0}}, + {.docid = 7000, .positions = {0}}, + {.docid = 8000, .positions = {4}}})); + input.terms.push_back(make_term(format::make_phrase_bigram_term("failed", "ordinal"), + {{.docid = 6000, .positions = {0}}})); + // bigram(order, ordinal): only docs 5000/7000 have "order"@1 immediately + // followed by "ordinal"@2 (elsewhere both sit at position 2, not + // adjacent). Drives the n>=3 {"failed","order","ordinal"} bigram path. + input.terms.push_back( + make_term(format::make_phrase_bigram_term("order", "ordinal"), + {{.docid = 5000, .positions = {1}}, {.docid = 7000, .positions = {1}}})); + // bigram(repeat, repeat): every doc has "repeat"@{0,1,2}, so the adjacent + // pair occurs in all docs. Exercises the ordered-pair construction for + // repeated-term phrases (the same bigram appears for both adjacent pairs). + std::vector repeat_repeat_docs; + repeat_repeat_docs.reserve(kDocCount); + for (uint32_t docid = 0; docid < kDocCount; ++docid) { + repeat_repeat_docs.push_back({docid, {0}}); + } + input.terms.push_back(make_term(format::make_phrase_bigram_term("repeat", "repeat"), + std::move(repeat_repeat_docs))); + } + std::ranges::sort(input.terms, + [](const writer::TermPostings& lhs, const writer::TermPostings& rhs) { + return lhs.term < rhs.term; + }); + + writer::SniiCompoundWriter writer(file); + RETURN_IF_ERROR(writer.add_logical_index(input)); + RETURN_IF_ERROR(writer.finish()); + EXPECT_TRUE(file->finalized()); + + RETURN_IF_ERROR(reader::SniiSegmentReader::open(file, segment_reader)); + return segment_reader->open_index(input.index_id, input.index_suffix, index_reader); +} + +} // namespace doris::snii::snii_test diff --git a/be/test/storage/index/snii_spill_io_test.cpp b/be/test/storage/index/snii_spill_io_test.cpp new file mode 100644 index 00000000000000..e8ec2239a3abe3 --- /dev/null +++ b/be/test/storage/index/snii_spill_io_test.cpp @@ -0,0 +1,226 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// R10-io-local-s3 spill backend tests. +// +// SpillableByteBuffer now spills to disk via Doris IO (io::global_local_filesystem() +// create_file / appendv / close + read_at on read-back) instead of the standalone +// doris::snii::io::Local{File}Writer/Reader. These tests pin the externally observable contract: +// whatever lands on the spill scratch file, streamed back in append order, is byte-for-byte +// identical to the concatenation of every append -- across the 0-byte branch, the +// cap-crossing spill trigger, post-spill appends, and a >256 KiB chunk that also spans the +// stream_into() copy window. The scratch file is a process-private intermediate (NOT the +// published on-disk format v2), so this is an equivalence/round-trip guarantee, not a golden +// byte test. Everything is deterministic: fixed cap_bytes and a fixed byte pattern. + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/io/file_writer.h" +#include "storage/index/snii/writer/spillable_byte_buffer.h" + +namespace doris::snii::writer { +using doris::Status; +namespace { + +// Deterministic, position- and seed-dependent byte pattern so a reorder, truncation, or +// in-place corruption of any chunk is caught by a full-buffer comparison. +std::vector make_bytes(uint32_t seed, size_t n) { + std::vector v(n); + for (size_t i = 0; i < n; ++i) { + v[i] = static_cast((seed * 1103515245U + static_cast(i) * 12345U + 7U) & + 0xFFU); + } + return v; +} + +// In-memory doris::snii::io::FileWriter that records the exact bytes (and order) streamed into it. +class CapturingSniiFileWriter final : public doris::snii::io::FileWriter { +public: + doris::Status append(doris::snii::Slice data) override { + bytes_.insert(bytes_.end(), data.data(), data.data() + data.size()); + return doris::Status::OK(); + } + doris::Status finalize() override { return doris::Status::OK(); } + uint64_t bytes_written() const override { return bytes_.size(); } + + const std::vector& bytes() const { return bytes_; } + +private: + std::vector bytes_; +}; + +void expect_bytes_eq(const std::vector& actual, const std::vector& expected) { + // Compare sizes first, then memcmp, so a mismatch on a multi-MiB buffer does not dump the + // whole vector into the test log. + ASSERT_EQ(actual.size(), expected.size()); + if (!expected.empty()) { + EXPECT_EQ(std::memcmp(actual.data(), expected.data(), expected.size()), 0); + } +} + +// Points SNII_TEMP_DIR (resolve_temp_dir()'s first choice) at a throwaway directory so the +// spill scratch files are hermetic to the test and cleaned up afterwards. +class SniiSpillIoTest : public ::testing::Test { +protected: + void SetUp() override { + std::error_code ec; + std::filesystem::path base = std::filesystem::temp_directory_path(ec); + if (ec) { + base = "/tmp"; + } + dir_ = base / ("snii_spill_io_test_" + std::to_string(::getpid())); + std::filesystem::create_directories(dir_, ec); + ::setenv("SNII_TEMP_DIR", dir_.c_str(), 1); + } + + void TearDown() override { + ::unsetenv("SNII_TEMP_DIR"); + std::error_code ec; + std::filesystem::remove_all(dir_, ec); + } + + std::filesystem::path dir_; +}; + +// Append a mix of pre-spill, cap-crossing, and post-spill chunks (including a 0-byte append on +// each side of the spill), then assert the streamed-back bytes equal the concatenation in order. +TEST_F(SniiSpillIoTest, SpillRoundTripPreservesBytesAndOrder) { + SpillableByteBuffer buf(/*cap_bytes=*/16, "roundtrip"); + + const std::vector> chunks = { + make_bytes(1, 5), // pre-spill (ram=5) + make_bytes(2, 0), // 0-byte append, pre-spill (must be a no-op, not a chunk) + make_bytes(3, 9), // pre-spill (ram=14, still under cap) + make_bytes(4, 7), // crosses cap (ram=21 >= 16) -> spill flushes [c0,c2,c3] + make_bytes(5, 0), // 0-byte append, post-spill (len==0 appendv no-op) + make_bytes(6, 4), // post-spill, routed straight to the scratch file + make_bytes(7, 33) // post-spill + }; + + std::vector expected; + for (const auto& c : chunks) { + ASSERT_TRUE(buf.append(doris::snii::Slice(c)).ok()); + expected.insert(expected.end(), c.begin(), c.end()); + } + + EXPECT_TRUE(buf.spilled()); + EXPECT_EQ(buf.size(), static_cast(expected.size())); + + ASSERT_TRUE(buf.seal().ok()); + + CapturingSniiFileWriter out; + ASSERT_TRUE(buf.stream_into(&out).ok()); + expect_bytes_eq(out.bytes(), expected); +} + +// Exercises the 0-byte branch and a >256 KiB chunk appended after the spill (the old standalone +// writer's "direct write" path); the total also exceeds stream_into()'s 1 MiB copy window so the +// read-back loop runs more than once. +TEST_F(SniiSpillIoTest, EmptyAndLargeChunk) { + SpillableByteBuffer buf(/*cap_bytes=*/16, "largechunk"); + + constexpr size_t kLarge = 1300U * 1024U; // > 256 KiB and > the 1 MiB stream window + + std::vector> chunks; + chunks.push_back(make_bytes(10, 20)); // 20 >= 16 -> spill immediately, flushes [c0] + chunks.push_back(make_bytes(11, 0)); // post-spill 0-byte append (len==0 branch) + chunks.push_back(make_bytes(12, kLarge)); // post-spill large append straight to disk + chunks.push_back(make_bytes(13, 5)); // post-spill tail + + std::vector expected; + for (const auto& c : chunks) { + ASSERT_TRUE(buf.append(doris::snii::Slice(c)).ok()); + expected.insert(expected.end(), c.begin(), c.end()); + } + + EXPECT_TRUE(buf.spilled()); + EXPECT_EQ(buf.size(), static_cast(expected.size())); + + ASSERT_TRUE(buf.seal().ok()); + + CapturingSniiFileWriter out; + ASSERT_TRUE(buf.stream_into(&out).ok()); + expect_bytes_eq(out.bytes(), expected); +} + +// A buffer that never crosses the cap stays RAM-resident; stream_into() must still reproduce the +// exact bytes from the in-memory chunk chain (no scratch file is ever created). +TEST_F(SniiSpillIoTest, RamOnlyRoundTripNoSpill) { + SpillableByteBuffer buf(/*cap_bytes=*/UINT64_MAX, "ramonly"); // spilling disabled + + const std::vector> chunks = {make_bytes(30, 7), make_bytes(31, 0), + make_bytes(32, 13), make_bytes(33, 50)}; + + std::vector expected; + for (const auto& c : chunks) { + ASSERT_TRUE(buf.append(doris::snii::Slice(c)).ok()); + expected.insert(expected.end(), c.begin(), c.end()); + } + + EXPECT_FALSE(buf.spilled()); + EXPECT_EQ(buf.size(), static_cast(expected.size())); + + ASSERT_TRUE(buf.seal().ok()); // no-op for a RAM-resident buffer + + CapturingSniiFileWriter out; + ASSERT_TRUE(buf.stream_into(&out).ok()); + expect_bytes_eq(out.bytes(), expected); +} + +// The move-append path must spill and round-trip identically to the copy-append path. +TEST_F(SniiSpillIoTest, AppendMoveSpillRoundTrip) { + SpillableByteBuffer buf(/*cap_bytes=*/16, "movespill"); + + const std::vector> spec = { + {40, 10}, // pre-spill (ram=10) + {41, 0}, // empty, pre-spill + {42, 12}, // crosses cap (ram=22) -> spill flushes [c0,c2] + {43, 0}, // empty, post-spill + {44, 7} // post-spill + }; + + std::vector expected; + for (const auto& [seed, n] : spec) { + std::vector c = make_bytes(seed, n); + expected.insert(expected.end(), c.begin(), c.end()); + ASSERT_TRUE(buf.append_move(std::move(c)).ok()); + } + + EXPECT_TRUE(buf.spilled()); + EXPECT_EQ(buf.size(), static_cast(expected.size())); + + ASSERT_TRUE(buf.seal().ok()); + + CapturingSniiFileWriter out; + ASSERT_TRUE(buf.stream_into(&out).ok()); + expect_bytes_eq(out.bytes(), expected); +} + +} // namespace +} // namespace doris::snii::writer diff --git a/be/test/storage/index/snii_spimi_intern_test.cpp b/be/test/storage/index/snii_spimi_intern_test.cpp new file mode 100644 index 00000000000000..76b77fb978d81a --- /dev/null +++ b/be/test/storage/index/snii_spimi_intern_test.cpp @@ -0,0 +1,325 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// T05: SPIMI vocab transparent-hash interning + single-string storage. +// +// These tests pin the writer-side SpimiTermBuffer owned-mode interning to its new +// shape: each distinct vocab string is materialized into owned_vocab_ EXACTLY ONCE +// (no double-store, no per-token temporary probe std::string), and the term-id +// assignment / finalize output is byte-identical to the prior behavior. Writer-only, +// no reader fixture (build_reader) needed -- the buffer is driven directly via +// add_token(string_view) and drained via finalize_sorted(). + +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" + +using doris::snii::format::make_phrase_bigram_sentinel_term; +using doris::snii::format::make_phrase_bigram_term; +using doris::snii::writer::SpimiTermBuffer; +using doris::snii::writer::TermPostings; +// Alias the writer's TEST-ONLY counter namespace so it never collides with gtest's +// own ::testing namespace. +namespace stb_testing = doris::snii::writer::testing; + +namespace { + +// A >SSO phrase-bigram term (32 bytes: 20-byte marker + varint + "quick" + "brown"), +// the F21 case where the OLD path heap-allocated a temporary std::string per token. +std::string MixedBigram() { + return make_phrase_bigram_term("quick", "brown"); +} + +// A >SSO multibyte (CJK) token (a direct UTF-8 string literal): 18 bytes (6 chars x +// 3 bytes), well past libstdc++'s 15-byte SSO, exercising a long non-ASCII vocab key. +std::string MixedCjk() { + return "中文长词条目"; +} + +// Feeds a fixed, implementation-independent token script mixing a hidden phrase +// bigram, a plain ASCII term, and a >SSO CJK token across several docids (some +// re-touched). Two buffers fed this script must finalize byte-identically. +void FeedMixedScript(SpimiTermBuffer& b) { + const std::string bigram = MixedBigram(); + const std::string cjk = MixedCjk(); + b.add_token(std::string_view("alpha"), 0, 0); + b.add_token(std::string_view(bigram), 0, 1); + b.add_token(std::string_view(cjk), 0, 2); + b.add_token(std::string_view("alpha"), 1, 0); + b.add_token(std::string_view(bigram), 1, 1); + b.add_token(std::string_view("alpha"), 5, 3); + b.add_token(std::string_view(cjk), 5, 4); + b.add_token(std::string_view("alpha"), 9, 0); +} + +void ExpectPostingsEqual(const std::vector& a, const std::vector& b) { + ASSERT_EQ(a.size(), b.size()); + for (size_t i = 0; i < a.size(); ++i) { + EXPECT_EQ(a[i].term, b[i].term); + EXPECT_EQ(a[i].docids, b[i].docids); + EXPECT_EQ(a[i].freqs, b[i].freqs); + EXPECT_EQ(a[i].positions_flat, b[i].positions_flat); + } +} + +} // namespace + +// --------------------------------------------------------------------------------- +// Functional verification (FV1-FV9) +// --------------------------------------------------------------------------------- + +// FV1: ids are assigned in first-seen order (b=0,a=1,c=2) but the emitted order is +// lexicographic (a,b,c); docids/freqs are recovered correctly. +TEST(SniiSpimiTermBufferTest, VocabAssignsIdsInFirstSeenOrder) { + SpimiTermBuffer buf(/*has_positions=*/false); + buf.add_token(std::string_view("b"), 0, 0); + buf.add_token(std::string_view("a"), 1, 0); + buf.add_token(std::string_view("b"), 2, 0); + buf.add_token(std::string_view("c"), 3, 0); + buf.add_token(std::string_view("a"), 4, 0); + + EXPECT_EQ(buf.unique_terms(), 3U); + EXPECT_EQ(buf.total_tokens(), 5U); + EXPECT_TRUE(buf.status().ok()); + + std::vector terms = buf.finalize_sorted(); + ASSERT_EQ(terms.size(), 3U); + EXPECT_EQ(terms[0].term, "a"); + EXPECT_EQ(terms[1].term, "b"); + EXPECT_EQ(terms[2].term, "c"); + EXPECT_EQ(terms[0].docids, (std::vector {1U, 4U})); + EXPECT_EQ(terms[1].docids, (std::vector {0U, 2U})); + EXPECT_EQ(terms[2].docids, (std::vector {3U})); + EXPECT_EQ(terms[0].freqs, (std::vector {1U, 1U})); + EXPECT_EQ(terms[1].freqs, (std::vector {1U, 1U})); + EXPECT_EQ(terms[2].freqs, (std::vector {1U})); +} + +// FV2: the same >SSO bigram term fed 1000 times reuses ONE id (heterogeneous hit +// path), yielding a single term with 1000 ascending docids and freq 1 each. +TEST(SniiSpimiTermBufferTest, RepeatedTermReusesSingleId) { + SpimiTermBuffer buf(/*has_positions=*/false); + const std::string term = MixedBigram(); + ASSERT_GT(term.size(), 15U); // exceeds libstdc++ SSO: the OLD probe heap-allocated + + constexpr uint32_t kRepeats = 1000; + for (uint32_t d = 0; d < kRepeats; ++d) { + buf.add_token(std::string_view(term), d, 0); + } + EXPECT_EQ(buf.unique_terms(), 1U); + EXPECT_EQ(buf.total_tokens(), kRepeats); + + std::vector terms = buf.finalize_sorted(); + ASSERT_EQ(terms.size(), 1U); + EXPECT_EQ(terms[0].term, term); + ASSERT_EQ(terms[0].docids.size(), kRepeats); + for (uint32_t d = 0; d < kRepeats; ++d) { + EXPECT_EQ(terms[0].docids[d], d); + EXPECT_EQ(terms[0].freqs[d], 1U); + } +} + +// FV3 (also the byte-identity perf gate): two independent buffers fed the same mixed +// script (bigram + plain + CJK long token) finalize to element-identical postings, +// and the hidden terms survive interning with exact bytes. +TEST(SniiSpimiTermBufferTest, FinalizeIsByteIdenticalAcrossRuns) { + SpimiTermBuffer a(/*has_positions=*/true); + SpimiTermBuffer b(/*has_positions=*/true); + FeedMixedScript(a); + FeedMixedScript(b); + + std::vector ra = a.finalize_sorted(); + std::vector rb = b.finalize_sorted(); + ExpectPostingsEqual(ra, rb); + EXPECT_TRUE(a.status().ok()); + EXPECT_TRUE(b.status().ok()); + + bool saw_bigram = false; + bool saw_cjk = false; + for (const auto& tp : ra) { + if (tp.term == MixedBigram()) { + saw_bigram = true; + } + if (tp.term == MixedCjk()) { + saw_cjk = true; + } + } + EXPECT_TRUE(saw_bigram); + EXPECT_TRUE(saw_cjk); +} + +// FV4: no tokens -> empty result, status stays OK. +TEST(SniiSpimiTermBufferTest, EmptyVocabProducesNoTerms) { + SpimiTermBuffer buf(/*has_positions=*/false); + EXPECT_EQ(buf.unique_terms(), 0U); + std::vector terms = buf.finalize_sorted(); + EXPECT_TRUE(terms.empty()); + EXPECT_TRUE(buf.status().ok()); +} + +// FV5: a single token yields a single term, single docid, freq 1. +TEST(SniiSpimiTermBufferTest, SingleTokenProducesSingleTerm) { + SpimiTermBuffer buf(/*has_positions=*/true); + buf.add_token(std::string_view("solo"), 7, 3); + EXPECT_EQ(buf.unique_terms(), 1U); + + std::vector terms = buf.finalize_sorted(); + ASSERT_EQ(terms.size(), 1U); + EXPECT_EQ(terms[0].term, "solo"); + EXPECT_EQ(terms[0].docids, (std::vector {7U})); + EXPECT_EQ(terms[0].freqs, (std::vector {1U})); + EXPECT_EQ(terms[0].positions_flat, (std::vector {3U})); +} + +// FV6: the empty string is a valid distinct term; the heterogeneous equality functor +// matches "" against a stored "" so a repeat empty token reuses the same id. +TEST(SniiSpimiTermBufferTest, EmptyStringIsAValidDistinctTerm) { + SpimiTermBuffer buf(/*has_positions=*/false); + buf.add_token(std::string_view(""), 0, 0); // empty term, first occurrence + buf.add_token(std::string_view("x"), 1, 0); + buf.add_token(std::string_view(""), 2, 0); // empty term reused via transparent eq + + EXPECT_EQ(buf.unique_terms(), 2U); + std::vector terms = buf.finalize_sorted(); + ASSERT_EQ(terms.size(), 2U); + EXPECT_EQ(terms[0].term, ""); // "" sorts before "x" + EXPECT_EQ(terms[0].docids, (std::vector {0U, 2U})); + EXPECT_EQ(terms[1].term, "x"); + EXPECT_EQ(terms[1].docids, (std::vector {1U})); + EXPECT_TRUE(buf.status().ok()); +} + +// FV7: add_token(string_view) on a BORROWED-vocab buffer is rejected (latches +// InvalidArgument, token ignored). The interning functors hold &owned_vocab_ but are +// never dereferenced on this path (reject happens before the find), so empty +// owned_vocab_ is never indexed out of bounds. +TEST(SniiSpimiTermBufferTest, BorrowedModeRejectsStringView) { + const std::vector vocab = {"a", "b"}; + SpimiTermBuffer buf(&vocab, /*has_positions=*/false); + buf.add_token(0U, 0, 0); // valid id-path token + buf.add_token(std::string_view("a"), 1, 0); // illegal on a borrowed-vocab buffer + + EXPECT_FALSE(buf.status().ok()); + EXPECT_EQ(buf.total_tokens(), 1U); // the string-view token was ignored + EXPECT_EQ(buf.unique_terms(), 1U); +} + +// FV8: a hidden phrase-bigram sentinel and bigram term coexist with a plain term that +// shares a substring; all three are distinct ids and round-trip with exact bytes (the +// hidden terms must not leak into / be confused with the plain term). +TEST(SniiSpimiTermBufferTest, PhraseBigramHiddenTermRoundTrips) { + SpimiTermBuffer buf(/*has_positions=*/true); + const std::string sentinel = make_phrase_bigram_sentinel_term(); + const std::string bigram = make_phrase_bigram_term("failed", "order"); + buf.add_token(std::string_view(sentinel), 0, 0); + buf.add_token(std::string_view(bigram), 0, 1); + buf.add_token(std::string_view("order"), 0, 2); + + EXPECT_EQ(buf.unique_terms(), 3U); + std::vector terms = buf.finalize_sorted(); + ASSERT_EQ(terms.size(), 3U); + + bool saw_sentinel = false; + bool saw_bigram = false; + bool saw_plain = false; + for (const auto& tp : terms) { + if (tp.term == sentinel) { + saw_sentinel = true; + } else if (tp.term == bigram) { + saw_bigram = true; + } else if (tp.term == "order") { + saw_plain = true; + } + } + EXPECT_TRUE(saw_sentinel); + EXPECT_TRUE(saw_bigram); + EXPECT_TRUE(saw_plain); +} + +// FV9: out-of-order / revisited docids for one term coalesce into one strictly +// ascending entry per docid (orthogonal to the intern change; guards no regression). +TEST(SniiSpimiTermBufferTest, OutOfOrderDocidCoalesces) { + SpimiTermBuffer buf(/*has_positions=*/true); + buf.add_token(std::string_view("t"), 5, 50); + buf.add_token(std::string_view("t"), 1, 10); + buf.add_token(std::string_view("t"), 5, 52); // revisit doc 5 + + std::vector terms = buf.finalize_sorted(); + ASSERT_EQ(terms.size(), 1U); + EXPECT_EQ(terms[0].docids, (std::vector {1U, 5U})); + EXPECT_EQ(terms[0].freqs, (std::vector {1U, 2U})); + EXPECT_EQ(terms[0].positions_flat, (std::vector {10U, 50U, 52U})); + EXPECT_TRUE(buf.status().ok()); +} + +// --------------------------------------------------------------------------------- +// Deterministic performance verification (allocation seam counts) +// --------------------------------------------------------------------------------- + +// The same >SSO term fed M times materializes its string EXACTLY ONCE: no per-token +// temporary probe std::string (F21) and no second owned-string map key (F03). The +// OLD instrumented baseline would have been M temporaries + 1 emplace = M+1. +TEST(SniiSpimiTermBufferTest, VocabInterningMaterializesEachStringOnce) { + stb_testing::reset_vocab_string_materialization_count(); + SpimiTermBuffer buf(/*has_positions=*/false); + const std::string term = MixedBigram(); + ASSERT_GT(term.size(), 15U); + + constexpr uint32_t kRepeats = 1000; + for (uint32_t d = 0; d < kRepeats; ++d) { + buf.add_token(std::string_view(term), d, 0); + } + EXPECT_EQ(stb_testing::vocab_string_materialization_count(), 1U); + EXPECT_EQ(buf.unique_terms(), 1U); +} + +// N distinct >SSO terms, each fed twice, materialize exactly N strings: the count +// tracks DISTINCT terms (one owned_vocab_.emplace_back each), not total tokens -- the +// repeat of an already-seen term allocates nothing on the heterogeneous hit path. +TEST(SniiSpimiTermBufferTest, VocabMaterializesOncePerDistinctTerm) { + stb_testing::reset_vocab_string_materialization_count(); + SpimiTermBuffer buf(/*has_positions=*/false); + + constexpr uint32_t kDistinct = 500; + for (uint32_t i = 0; i < kDistinct; ++i) { + const std::string term = make_phrase_bigram_term("term", "number" + std::to_string(i)); + buf.add_token(std::string_view(term), i, 0); + buf.add_token(std::string_view(term), i + kDistinct, 0); // repeat: zero materialization + } + EXPECT_EQ(stb_testing::vocab_string_materialization_count(), static_cast(kDistinct)); + EXPECT_EQ(buf.unique_terms(), static_cast(kDistinct)); +} + +// The seam resets cleanly between measurements (guards the reset_/count_ contract the +// two perf tests above rely on for determinism in a shared process). +TEST(SniiSpimiTermBufferTest, MaterializationCounterResets) { + stb_testing::reset_vocab_string_materialization_count(); + EXPECT_EQ(stb_testing::vocab_string_materialization_count(), 0U); + SpimiTermBuffer buf(/*has_positions=*/false); + buf.add_token(std::string_view("one"), 0, 0); + buf.add_token(std::string_view("two"), 0, 0); + EXPECT_EQ(stb_testing::vocab_string_materialization_count(), 2U); + stb_testing::reset_vocab_string_materialization_count(); + EXPECT_EQ(stb_testing::vocab_string_materialization_count(), 0U); +} diff --git a/be/test/storage/index/snii_writer_test.cpp b/be/test/storage/index/snii_writer_test.cpp new file mode 100644 index 00000000000000..d2f6c5feac5e73 --- /dev/null +++ b/be/test/storage/index/snii_writer_test.cpp @@ -0,0 +1,487 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// T25 unit tests: +// A (F28) -- emit_adjacent_phrase_bigrams: the writer's phrase-bigram pair +// emission extracted as a dependency-free template. Asserts the refactored +// set equals the pre-refactor "sort (position, term) then window" baseline, +// the is_sorted guard short-circuits on analyzer-ordered input (did_sort == +// false) yet still corrects a shuffled input, and the reused positioned-term +// vector keeps its capacity across batches (no per-row realloc). +// B (F35) -- heap_bytes() accessors on the resident format readers +// (SampledTermIndexReader / DictBlockDirectoryReader / DictBlockReader) that +// LogicalIndexReader::memory_usage() sums so the searcher-cache charge stops +// under-counting. Exact hand-computed equality for SSO terms; the string-heap +// accumulation is exercised with an over-15-byte term. + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/common/slice.h" +#include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/format/dict_block.h" +#include "storage/index/snii/format/dict_block_directory.h" +#include "storage/index/snii/format/dict_entry.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/phrase_bigram.h" +#include "storage/index/snii/format/sampled_term_index.h" +#include "storage/index/snii/query/docid_sink.h" +#include "storage/index/snii/snii_index_writer.h" +#include "storage/index/snii/snii_phrase_bigram_build.h" + +namespace { + +using doris::segment_v2::emit_adjacent_phrase_bigrams; +using doris::segment_v2::PhrasePositionedTerm; +using doris::snii::ByteSink; +using doris::snii::Slice; +using namespace doris::snii::format; // NOLINT(google-build-using-namespace) + +void assert_ok(const doris::Status& status) { + ASSERT_TRUE(status.ok()) << status.to_string(); +} + +// ============================ sub-task A ============================ + +// One emitted phrase-bigram pair, owning its term bytes so it survives past the +// backing string_view storage. Ordered/compared as a multiset element. +struct BigramTriple { + std::string left; + std::string right; + uint32_t position = 0; + + bool operator==(const BigramTriple&) const = default; + bool operator<(const BigramTriple& o) const { + return std::tie(position, left, right) < std::tie(o.position, o.left, o.right); + } +}; + +// Builds position-tagged views over `storage` (which must outlive the returned +// vector -- callers keep it as a stable local). +std::vector make_positioned(const std::vector& storage, + const std::vector& positions) { + EXPECT_EQ(storage.size(), positions.size()); + std::vector out; + out.reserve(storage.size()); + for (size_t i = 0; i < storage.size(); ++i) { + out.push_back({std::string_view(storage[i]), positions[i]}); + } + return out; +} + +// The pre-refactor implementation: sort by (position, term) -- primary AND the +// now-dropped secondary key -- then run the identical position-group window. The +// result is returned as a sorted multiset so a comparison proves dropping the +// secondary key never changes the emitted set. +std::vector reference_old_bigrams(std::vector terms) { + std::ranges::sort(terms, [](const auto& a, const auto& b) { + if (a.position != b.position) { + return a.position < b.position; + } + return a.term < b.term; + }); + std::vector out; + size_t left_begin = 0; + while (left_begin < terms.size()) { + size_t left_end = left_begin + 1; + while (left_end < terms.size() && terms[left_end].position == terms[left_begin].position) { + ++left_end; + } + size_t right_begin = left_end; + while (right_begin < terms.size() && + terms[right_begin].position <= terms[left_begin].position) { + ++right_begin; + } + if (right_begin == terms.size() || + terms[right_begin].position != terms[left_begin].position + 1) { + left_begin = left_end; + continue; + } + size_t right_end = right_begin + 1; + while (right_end < terms.size() && + terms[right_end].position == terms[right_begin].position) { + ++right_end; + } + for (size_t l = left_begin; l < left_end; ++l) { + for (size_t r = right_begin; r < right_end; ++r) { + out.push_back({std::string(terms[l].term), std::string(terms[r].term), + terms[l].position}); + } + } + left_begin = left_end; + } + // BigramTriple has operator< but is not std::totally_ordered, so std::ranges::sort's + // sortable concept is not satisfied. + std::sort(out.begin(), out.end()); // NOLINT(modernize-use-ranges) + return out; +} + +// Runs the refactored emitter, collecting the pairs as a sorted multiset and (if +// requested) reporting the guard's did_sort. +std::vector collect_new_bigrams(std::vector terms, + bool* did_sort = nullptr) { + std::vector out; + const bool ds = emit_adjacent_phrase_bigrams( + terms, [&](std::string_view left, std::string_view right, uint32_t position) { + out.push_back({std::string(left), std::string(right), position}); + }); + if (did_sort != nullptr) { + *did_sort = ds; + } + // BigramTriple has operator< but is not std::totally_ordered, so std::ranges::sort's + // sortable concept is not satisfied. + std::sort(out.begin(), out.end()); // NOLINT(modernize-use-ranges) + return out; +} + +// A1: monotone positions {0,1,2} -> two adjacent pairs, guard short-circuits. +TEST(SniiPhraseBigramBuildTest, EmitsSamePairsAsSortedBaseline) { + const std::vector storage = {"alpha", "beta", "gamma"}; + auto terms = make_positioned(storage, {0, 1, 2}); + + bool did_sort = true; + const auto got = collect_new_bigrams(terms, &did_sort); + + const std::vector expected = {{.left = "alpha", .right = "beta", .position = 0}, + {.left = "beta", .right = "gamma", .position = 1}}; + EXPECT_EQ(got, expected); + EXPECT_EQ(got, reference_old_bigrams(terms)); + EXPECT_FALSE(did_sort); // analyzer-ordered input never trips the sort guard +} + +// A2: two tokens share position 0 -> full left x right product at the boundary. +TEST(SniiPhraseBigramBuildTest, SamePositionEmitsFullProduct) { + const std::vector storage = {"aa", "bb", "cc"}; + auto terms = make_positioned(storage, {0, 0, 1}); + + bool did_sort = true; + const auto got = collect_new_bigrams(terms, &did_sort); + + const std::vector expected = {{.left = "aa", .right = "cc", .position = 0}, + {.left = "bb", .right = "cc", .position = 0}}; + EXPECT_EQ(got, expected); + EXPECT_EQ(got, reference_old_bigrams(terms)); + EXPECT_FALSE(did_sort); +} + +// A3: a position gap {0,2} has no adjacent (+1) pair -> empty, guard clean. +TEST(SniiPhraseBigramBuildTest, PositionGapEmitsNothing) { + const std::vector storage = {"aa", "bb"}; + auto terms = make_positioned(storage, {0, 2}); + + bool did_sort = true; + const auto got = collect_new_bigrams(terms, &did_sort); + + EXPECT_TRUE(got.empty()); + EXPECT_EQ(got, reference_old_bigrams(terms)); + EXPECT_FALSE(did_sort); +} + +// A4 (degenerate): 0 and 1 element inputs emit nothing and do not crash. +TEST(SniiPhraseBigramBuildTest, EmptyAndSingleEmitNothing) { + std::vector empty; + bool did_sort = true; + EXPECT_TRUE(collect_new_bigrams(empty, &did_sort).empty()); + EXPECT_FALSE(did_sort); + + const std::vector storage = {"solo"}; + auto single = make_positioned(storage, {0}); + EXPECT_TRUE(collect_new_bigrams(single, &did_sort).empty()); + EXPECT_FALSE(did_sort); +} + +// A5 (equivalence): a shuffled input trips the guard (did_sort == true) yet the +// emitted set still equals the sorted baseline -- new path == old path. +TEST(SniiPhraseBigramBuildTest, UnsortedInputSortsAndMatches) { + const std::vector storage = {"gamma", "alpha", "beta"}; + auto terms = make_positioned(storage, {2, 0, 1}); + + bool did_sort = false; + const auto got = collect_new_bigrams(terms, &did_sort); + + const std::vector expected = {{.left = "alpha", .right = "beta", .position = 0}, + {.left = "beta", .right = "gamma", .position = 1}}; + EXPECT_EQ(got, expected); + EXPECT_EQ(got, reference_old_bigrams(terms)); + EXPECT_TRUE(did_sort); // shuffled input forces the cold-path sort +} + +// A6 (hidden term): the writer filters tokens through +// is_phrase_bigram_indexable_term before emitting, so a non-indexable token (too +// long / non-ASCII-alpha) never leaks into a bigram pair. Mirror that filter and +// assert the excluded token appears in no emitted pair. +TEST(SniiPhraseBigramBuildTest, NonIndexableTermIsFilteredOut) { + const std::string too_long(40, 'z'); // > 32 bytes: not indexable + const std::string non_alpha = "ab12"; // digits: not indexable + const std::vector storage = {"left", too_long, non_alpha, "right"}; + const std::vector positions = {0, 1, 2, 3}; + + // Filter exactly as SniiIndexColumnWriter::_add_phrase_bigram_tokens does. + std::vector filtered; + for (size_t i = 0; i < storage.size(); ++i) { + if (is_phrase_bigram_indexable_term(storage[i])) { + filtered.push_back({std::string_view(storage[i]), positions[i]}); + } + } + // Only "left"@0 and "right"@3 survive; they are not adjacent (+1), so no pair + // is emitted and neither hidden token is ever referenced. + const auto got = collect_new_bigrams(filtered); + EXPECT_TRUE(got.empty()); + for (const auto& triple : got) { + EXPECT_NE(triple.left, too_long); + EXPECT_NE(triple.right, too_long); + EXPECT_NE(triple.left, non_alpha); + EXPECT_NE(triple.right, non_alpha); + } +} + +// Deterministic perf seam: the writer reuses one positioned-term vector via +// clear() (capacity retained). Emulate two consecutive rows and assert the second +// batch triggers no reallocation. +TEST(SniiPhraseBigramBuildTest, ReusedVectorKeepsCapacityAcrossBatches) { + const std::vector batch1 = {"aa", "bb", "cc", "dd", "ee", "ff"}; + const std::vector batch2 = {"gg", "hh", "ii"}; + + std::vector reused; + auto fill = [&](const std::vector& storage) { + reused.clear(); // retains capacity, exactly like _bigram_positioned + for (uint32_t i = 0; i < storage.size(); ++i) { + reused.push_back({std::string_view(storage[i]), i}); + } + }; + + fill(batch1); + static_cast(emit_adjacent_phrase_bigrams( + reused, [](std::string_view, std::string_view, uint32_t) {})); + const size_t cap_after_batch1 = reused.capacity(); + ASSERT_GE(cap_after_batch1, batch1.size()); + + fill(batch2); // batch2.size() <= cap_after_batch1 -> no realloc + static_cast(emit_adjacent_phrase_bigrams( + reused, [](std::string_view, std::string_view, uint32_t) {})); + EXPECT_EQ(reused.capacity(), cap_after_batch1); +} + +// ============================ sub-task B ============================ + +// SampledTermIndexReader::heap_bytes(): all-SSO sample terms have no per-string +// heap, so the charge is exactly n_blocks * sizeof(std::string) (reserve-exact +// backing buffer). +TEST(SniiSegmentReaderTest, SampledTermIndexHeapBytesMatchesFormula) { + const std::vector terms = {"s000", "s001", "s002", "s003", "s004", "s005"}; + SampledTermIndexBuilder builder; + for (const auto& term : terms) { + builder.add_block_first_term(term); // strictly ascending, SSO + } + ByteSink sink; + builder.finish(&sink); + + SampledTermIndexReader reader; + assert_ok(SampledTermIndexReader::open(sink.view(), &reader)); + ASSERT_EQ(reader.n_blocks(), terms.size()); + EXPECT_EQ(reader.heap_bytes(), terms.size() * sizeof(std::string)); +} + +// The std_string_heap_bytes accumulation: an over-15-byte sample term adds its +// heap buffer on top of the vector buffer. +TEST(SniiSegmentReaderTest, SampledTermIndexHeapBytesCountsLongTerms) { + const std::string long_term = "b_this_is_a_long_sample_term_well_over_15_bytes"; + ASSERT_GT(long_term.size(), 15U); + const std::vector terms = {"a_short", long_term, "c_short"}; + SampledTermIndexBuilder builder; + for (const auto& term : terms) { + builder.add_block_first_term(term); + } + ByteSink sink; + builder.finish(&sink); + + SampledTermIndexReader reader; + assert_ok(SampledTermIndexReader::open(sink.view(), &reader)); + const size_t vector_only = terms.size() * sizeof(std::string); + EXPECT_GT(reader.heap_bytes(), vector_only); + // capacity() >= size(), so the long term contributes >= size()+1 heap bytes. + EXPECT_GE(reader.heap_bytes(), vector_only + long_term.size() + 1); + // Cross-check the shared helper on an SSO vs non-SSO string. + EXPECT_EQ(std_string_heap_bytes(std::string("short")), 0U); + EXPECT_GT(std_string_heap_bytes(long_term), 0U); +} + +// DictBlockDirectoryReader::heap_bytes(): BlockRef is trivially copyable, so the +// charge is exactly n_blocks * sizeof(BlockRef). +TEST(SniiSegmentReaderTest, DictBlockDirectoryHeapBytesMatchesFormula) { + DictBlockDirectoryBuilder builder; + constexpr uint32_t kBlocks = 5; + for (uint32_t i = 0; i < kBlocks; ++i) { + BlockRef ref; + ref.offset = 100000ULL * (i + 1); // multi-byte varints -> each ref > 8 bytes + ref.length = 640; + ref.n_entries = 3; + ref.flags = 0; + ref.checksum = 0xDEAD0000U + i; + builder.add(ref); + } + ByteSink sink; + builder.finish(&sink); + + DictBlockDirectoryReader reader; + assert_ok(DictBlockDirectoryReader::open(sink.view(), &reader)); + ASSERT_EQ(reader.n_blocks(), kBlocks); + EXPECT_EQ(reader.heap_bytes(), static_cast(kBlocks) * sizeof(BlockRef)); +} + +// A minimal slim pod_ref entry that round-trips at tier T1 (extra tier>=T2 fields +// are ignored on encode). Terms are supplied by the caller in ascending order. +DictEntry make_pod_ref(std::string term) { + DictEntry e; + e.term = std::move(term); + e.kind = DictEntryKind::kPodRef; + e.enc = DictEntryEnc::kSlim; + e.df = 3; + e.ttf_delta = 6; + e.max_freq = 9; + e.frq_off_delta = 0; + e.frq_len = 128; + e.frq_docs_len = 64; // dd region on-disk length (<= frq_len) + e.dd_meta.uncomp_len = 70; + e.dd_meta.crc = 0xABCD1234U; + e.freq_meta.uncomp_len = 40; + e.freq_meta.crc = 0x55AA00FFU; + e.prx_off_delta = 0; + e.prx_len = 64; + return e; +} + +std::vector build_dict_block(const std::vector& terms, + uint32_t anchor_interval) { + DictBlockBuilder builder(IndexTier::kT1, /*has_positions=*/false, /*frq_base=*/0, + /*prx_base=*/0, anchor_interval); + for (const auto& term : terms) { + builder.add_entry(make_pod_ref(term)); + } + ByteSink sink; + builder.finish(&sink); + return sink.buffer(); +} + +// DictBlockReader::heap_bytes(): with anchor_interval 16 and 20 SSO entries there +// are two anchors (indices 0 and 16), each with an SSO anchor term, so the charge +// is exactly n_anchors * (sizeof(uint32_t) + sizeof(std::string)). +TEST(SniiSegmentReaderTest, DictBlockAnchorHeapBytesMatchesFormula) { + constexpr uint32_t kEntries = 20; + constexpr uint32_t kAnchorInterval = 16; + std::vector terms; + terms.reserve(kEntries); + for (uint32_t i = 0; i < kEntries; ++i) { + // "dt_00".."dt_19": strictly ascending, 5 bytes (SSO). + terms.push_back("dt_" + std::string(1, static_cast('0' + i / 10)) + + std::string(1, static_cast('0' + i % 10))); + } + const std::vector bytes = build_dict_block(terms, kAnchorInterval); + + DictBlockReader reader; + assert_ok( + DictBlockReader::open(Slice(bytes), IndexTier::kT1, /*has_positions=*/false, &reader)); + ASSERT_EQ(reader.n_entries(), kEntries); + + const size_t n_anchors = (kEntries + kAnchorInterval - 1) / kAnchorInterval; // == 2 + EXPECT_EQ(reader.heap_bytes(), n_anchors * (sizeof(uint32_t) + sizeof(std::string))); +} + +// A long (> 15 byte) anchor term adds its heap buffer beyond the anchor vectors. +TEST(SniiSegmentReaderTest, DictBlockAnchorHeapBytesCountsLongAnchor) { + // One entry -> one anchor (entry 0), whose term is > 15 bytes. + const std::string long_term = "a_dict_anchor_term_well_over_15_bytes"; + ASSERT_GT(long_term.size(), 15U); + const std::vector bytes = build_dict_block({long_term}, /*anchor_interval=*/16); + + DictBlockReader reader; + assert_ok( + DictBlockReader::open(Slice(bytes), IndexTier::kT1, /*has_positions=*/false, &reader)); + ASSERT_EQ(reader.n_entries(), 1U); + const size_t vector_only = sizeof(uint32_t) + sizeof(std::string); // one anchor + EXPECT_GT(reader.heap_bytes(), vector_only); + EXPECT_GE(reader.heap_bytes(), vector_only + long_term.size() + 1); +} + +// ==================== null-docids growth-policy regression pins ==================== +// +// append_nullable feeds add_nulls once per NULL RUN -- millions of calls on a +// large interleaved-null compaction segment. An exact reserve(size()+count) +// inside add_nulls capped capacity at "just enough", so EVERY subsequent call +// reallocated + memcpy'd the whole array: O(runs x N) total memcpy (the +// agentlogs full-compaction pathology: ~TBs of memcpy per tablet, 8x+ slower +// than V3). These pins count capacity changes across many small appends: with +// geometric growth that is O(log n); with the exact-reserve bug it was one per +// call. add_nulls touches only _null_docids/_rid, so a scaffold-free writer +// (null collaborators, no init()) exercises the real production code path. + +TEST(SniiWriterNullDocids, AddNullsGrowsGeometricallyNotQuadratically) { + doris::segment_v2::SniiIndexColumnWriter writer(nullptr, nullptr, /*single_field=*/true); + constexpr uint32_t kRuns = 4096; + size_t capacity_changes = 0; + size_t last_cap = writer.null_docids_for_test().capacity(); + for (uint32_t i = 0; i < kRuns; ++i) { + assert_ok(writer.add_nulls(1)); + const size_t cap = writer.null_docids_for_test().capacity(); + if (cap != last_cap) { + ++capacity_changes; + last_cap = cap; + } + } + // Geometric growth reallocates O(log n) times (libstdc++ doubling: ~13 for + // 4096); the exact-reserve bug reallocated on every call (4096). The bound + // leaves generous headroom for any sane growth policy while still failing + // a per-call realloc by two orders of magnitude. + EXPECT_LE(capacity_changes, 64U) << "add_nulls reallocates per call again"; + // Content unchanged by the policy fix: docids 0..kRuns-1 in order. + const auto& nulls = writer.null_docids_for_test(); + ASSERT_EQ(nulls.size(), kRuns); + EXPECT_EQ(nulls.front(), 0U); + EXPECT_EQ(nulls.back(), kRuns - 1); + EXPECT_TRUE(std::ranges::is_sorted(nulls)); +} + +TEST(SniiDocIdSinkGrowth, AppendRangeGrowsGeometrically) { + std::vector docids; + doris::snii::query::VectorDocIdSink sink(docids); + constexpr uint32_t kRuns = 4096; + size_t capacity_changes = 0; + size_t last_cap = docids.capacity(); + for (uint32_t i = 0; i < kRuns; ++i) { + assert_ok(sink.append_range(i, static_cast(i) + 1)); // one docid per run + const size_t cap = docids.capacity(); + if (cap != last_cap) { + ++capacity_changes; + last_cap = cap; + } + } + EXPECT_LE(capacity_changes, 64U) << "append_range reallocates per call again"; + ASSERT_EQ(docids.size(), kRuns); + EXPECT_EQ(docids.front(), 0U); + EXPECT_EQ(docids.back(), kRuns - 1); + EXPECT_TRUE(std::ranges::is_sorted(docids)); +} + +} // namespace diff --git a/be/test/storage/segment/count_on_index_fastpath_test.cpp b/be/test/storage/segment/count_on_index_fastpath_test.cpp new file mode 100644 index 00000000000000..e529aa7c5a6bdd --- /dev/null +++ b/be/test/storage/segment/count_on_index_fastpath_test.cpp @@ -0,0 +1,313 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/segment/count_on_index_fastpath.h" + +#include + +// Truth table for the G02 count-only fast-path caller guard. SegmentIterator +// fills CountOnIndexFastpathFacts from its state right before the index apply; +// this test pins that the ONLY admitted configuration is: COUNT_ON_INDEX agg + +// exactly one pushed-down MATCH expr + zero other filters + zero deletes + +// full row bitmap + zero row-id consumers + the no-read-data contract active. +// Each single deviation must veto the fast path (fall through to the +// row-accurate bitmap), because the fabricated [0, df) bitmap is only +// count-equivalent, never row-equivalent. +namespace doris::segment_v2 { + +namespace { + +// The one configuration that admits the fast path. +CountOnIndexFastpathFacts safe_facts() { + CountOnIndexFastpathFacts f; + f.is_count_on_index_agg = true; + f.has_column_predicates = false; + f.common_expr_count = 1; + f.single_expr_is_match_pred = true; + f.has_virtual_column_exprs = false; + f.has_delete_predicates = false; + f.segment_delete_bitmap_empty = true; + f.has_col_id_predicates = false; + f.has_topn_filters = false; + f.has_external_row_ranges = false; + f.row_bitmap_is_full = true; + f.record_rowids = false; + f.has_ann_topn = false; + f.has_score_runtime = false; + f.no_need_read_data_opt_enabled = true; + f.keys_type_supported = true; + return f; +} + +} // namespace + +TEST(CountOnIndexFastpath, AllGuardsSatisfiedAdmits) { + EXPECT_TRUE(count_on_index_fastpath_safe(safe_facts())); +} + +// Non-count context: no COUNT_ON_INDEX pushdown (the "context flag absent" +// case) -> the reader must take the normal bitmap path. +TEST(CountOnIndexFastpath, NotCountOnIndexVetoes) { + auto f = safe_facts(); + f.is_count_on_index_agg = false; + EXPECT_FALSE(count_on_index_fastpath_safe(f)); +} + +// Extra predicates beside the single MATCH veto: the count of one predicate +// is not the count of a conjunction. +TEST(CountOnIndexFastpath, ExtraPredicatesVeto) { + { + auto f = safe_facts(); + f.has_column_predicates = true; + EXPECT_FALSE(count_on_index_fastpath_safe(f)); + } + { + auto f = safe_facts(); + f.common_expr_count = 2; + EXPECT_FALSE(count_on_index_fastpath_safe(f)); + } + { + auto f = safe_facts(); + f.common_expr_count = 0; + f.single_expr_is_match_pred = false; + EXPECT_FALSE(count_on_index_fastpath_safe(f)); + } + { + // One expr, but a compound / non-MATCH root: leaf-level fabricated + // bitmaps would be combined with sibling bitmaps. + auto f = safe_facts(); + f.single_expr_is_match_pred = false; + EXPECT_FALSE(count_on_index_fastpath_safe(f)); + } + { + auto f = safe_facts(); + f.has_col_id_predicates = true; + EXPECT_FALSE(count_on_index_fastpath_safe(f)); + } + { + auto f = safe_facts(); + f.has_topn_filters = true; + EXPECT_FALSE(count_on_index_fastpath_safe(f)); + } + { + auto f = safe_facts(); + f.has_virtual_column_exprs = true; + EXPECT_FALSE(count_on_index_fastpath_safe(f)); + } +} + +// Deletes veto: COUNT_ON_INDEX counts matching rows MINUS deleted rows, and a +// fabricated id range cannot participate in the delete-bitmap subtraction or +// in delete-predicate filtering (mirror of the V3 _lazy_init handling). +TEST(CountOnIndexFastpath, DeletesVeto) { + { + auto f = safe_facts(); + f.segment_delete_bitmap_empty = false; + EXPECT_FALSE(count_on_index_fastpath_safe(f)); + } + { + auto f = safe_facts(); + f.has_delete_predicates = true; + EXPECT_FALSE(count_on_index_fastpath_safe(f)); + } +} + +// A pre-pruned or later-restricted row space vetoes: [0, df) only counts +// correctly against the full [0, num_rows) bitmap. +TEST(CountOnIndexFastpath, PrunedRowSpaceVetoes) { + { + auto f = safe_facts(); + f.row_bitmap_is_full = false; // condition cache / key ranges pruned + EXPECT_FALSE(count_on_index_fastpath_safe(f)); + } + { + auto f = safe_facts(); + f.has_external_row_ranges = true; + EXPECT_FALSE(count_on_index_fastpath_safe(f)); + } +} + +// Consumers of REAL row ids veto. +TEST(CountOnIndexFastpath, RowIdConsumersVeto) { + { + auto f = safe_facts(); + f.record_rowids = true; + EXPECT_FALSE(count_on_index_fastpath_safe(f)); + } + { + auto f = safe_facts(); + f.has_ann_topn = true; + EXPECT_FALSE(count_on_index_fastpath_safe(f)); + } + { + auto f = safe_facts(); + f.has_score_runtime = true; + EXPECT_FALSE(count_on_index_fastpath_safe(f)); + } +} + +// The COUNT_ON_INDEX no-read-data contract must be active, otherwise column +// data would be materialized at fabricated row ids. +TEST(CountOnIndexFastpath, DataReadContractVetoes) { + { + auto f = safe_facts(); + f.no_need_read_data_opt_enabled = false; + EXPECT_FALSE(count_on_index_fastpath_safe(f)); + } + { + auto f = safe_facts(); + f.keys_type_supported = false; // e.g. AGG keys / MOR without MOW + EXPECT_FALSE(count_on_index_fastpath_safe(f)); + } +} + +// --- G03 count-emission shortcut guard truth table -------------------------- +// The shortcut replaces the per-rowid batch loop with a defaults-fill +// countdown; the ONLY admitted configuration is: reader answered from df + +// zero surviving evaluation stages + zero row-id/value consumers + pure +// countdown batch accounting + a schema-shaped block whose every column is +// defaults-fillable. Each single deviation must refuse (keep today's batch +// loop, which is always count-exact). + +namespace { + +CountEmitShortcutFacts safe_emit_facts() { + CountEmitShortcutFacts f; + f.count_fastpath_hit = true; + f.needs_vec_eval = false; + f.needs_short_eval = false; + f.needs_expr_eval = false; + f.has_remaining_col_predicates = false; + f.has_remaining_common_exprs = false; + f.has_delete_predicates = false; + f.lazy_materialization_read = false; + f.has_virtual_columns = false; + f.record_rowids = false; + f.has_read_limit = false; + f.read_orderby_key_reverse = false; + f.has_condition_cache_digest = false; + f.block_shape_matches_schema = true; + f.all_columns_emit_defaults = true; + return f; +} + +} // namespace + +TEST(CountEmitShortcut, AllGuardsSatisfiedAdmits) { + EXPECT_TRUE(count_emit_shortcut_safe(safe_emit_facts())); +} + +// The reader must have ANSWERED via the fabricated count bitmap. A mere G02 +// guard pass whose reader fell through to a row-accurate decode (multi-term, +// pruned bigram, CLucene index, query-cache hit) keeps today's emission. +TEST(CountEmitShortcut, NoReaderHitRefuses) { + auto f = safe_emit_facts(); + f.count_fastpath_hit = false; + EXPECT_FALSE(count_emit_shortcut_safe(f)); +} + +// Any surviving evaluation stage refuses: the shortcut emits unconditionally +// and cannot re-apply filters (e.g. an index-eval downgrade kept the expr). +TEST(CountEmitShortcut, SurvivingEvaluationRefuses) { + { + auto f = safe_emit_facts(); + f.needs_vec_eval = true; + EXPECT_FALSE(count_emit_shortcut_safe(f)); + } + { + auto f = safe_emit_facts(); + f.needs_short_eval = true; + EXPECT_FALSE(count_emit_shortcut_safe(f)); + } + { + auto f = safe_emit_facts(); + f.needs_expr_eval = true; + EXPECT_FALSE(count_emit_shortcut_safe(f)); + } + { + auto f = safe_emit_facts(); + f.has_remaining_col_predicates = true; + EXPECT_FALSE(count_emit_shortcut_safe(f)); + } + { + auto f = safe_emit_facts(); + f.has_remaining_common_exprs = true; + EXPECT_FALSE(count_emit_shortcut_safe(f)); + } + { + auto f = safe_emit_facts(); + f.has_delete_predicates = true; + EXPECT_FALSE(count_emit_shortcut_safe(f)); + } + { + auto f = safe_emit_facts(); + f.lazy_materialization_read = true; + EXPECT_FALSE(count_emit_shortcut_safe(f)); + } +} + +// Consumers of real row ids or per-row values refuse. +TEST(CountEmitShortcut, RowIdOrValueConsumersRefuse) { + { + auto f = safe_emit_facts(); + f.has_virtual_columns = true; + EXPECT_FALSE(count_emit_shortcut_safe(f)); + } + { + auto f = safe_emit_facts(); + f.record_rowids = true; + EXPECT_FALSE(count_emit_shortcut_safe(f)); + } +} + +// Batch accounting must be a pure countdown: limits, reverse key-ordered +// reads, and condition-cache writes all keep today's loop. +TEST(CountEmitShortcut, NonCountdownBatchAccountingRefuses) { + { + auto f = safe_emit_facts(); + f.has_read_limit = true; + EXPECT_FALSE(count_emit_shortcut_safe(f)); + } + { + auto f = safe_emit_facts(); + f.read_orderby_key_reverse = true; + EXPECT_FALSE(count_emit_shortcut_safe(f)); + } + { + auto f = safe_emit_facts(); + f.has_condition_cache_digest = true; + EXPECT_FALSE(count_emit_shortcut_safe(f)); + } +} + +// The emitted block must be exactly the read schema and every column must be +// one today's path fills with defaults (no real read, no storage->schema +// cast, no version/lsn/tso rewrite). +TEST(CountEmitShortcut, BlockShapeOrColumnFillMismatchRefuses) { + { + auto f = safe_emit_facts(); + f.block_shape_matches_schema = false; + EXPECT_FALSE(count_emit_shortcut_safe(f)); + } + { + auto f = safe_emit_facts(); + f.all_columns_emit_defaults = false; + EXPECT_FALSE(count_emit_shortcut_safe(f)); + } +} + +} // namespace doris::segment_v2 diff --git a/be/test/storage/segment/inverted_index_file_reader_test.cpp b/be/test/storage/segment/inverted_index_file_reader_test.cpp index ba14fbb8359622..8080a45073615d 100644 --- a/be/test/storage/segment/inverted_index_file_reader_test.cpp +++ b/be/test/storage/segment/inverted_index_file_reader_test.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include "io/fs/local_file_system.h" #include "runtime/exec_env.h" @@ -31,6 +32,9 @@ #include "storage/index/inverted/inverted_index_cache.h" #include "storage/index/inverted/inverted_index_desc.h" #include "storage/index/inverted/inverted_index_fs_directory.h" +#include "storage/index/snii/snii_doris_adapter.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii/writer/spimi_term_buffer.h" #include "storage/options.h" #include "storage/storage_engine.h" #include "storage/tablet/tablet_schema.h" @@ -214,6 +218,44 @@ class InvertedIndexFileReaderTest : public testing::Test { create_v2_file_with_null_bitmap(file_path, index_id, index_suffix, false); } + void create_snii_file_with_null_bitmap(const std::string& file_path, uint64_t index_id, + const std::string& index_suffix, bool has_null_bitmap) { + std::filesystem::path parent_path = std::filesystem::path(file_path).parent_path(); + if (!std::filesystem::exists(parent_path)) { + std::filesystem::create_directories(parent_path); + } + + io::FileWriterPtr file_writer; + io::FileWriterOptions opts; + Status st = io::global_local_filesystem()->create_file(file_path, &file_writer, &opts); + ASSERT_TRUE(st.ok()) << st; + + snii_doris::DorisSniiFileWriter snii_file_writer(file_writer.get()); + doris::snii::writer::SniiCompoundWriter writer(&snii_file_writer); + doris::snii::writer::TermPostings term; + term.term = "apple"; + term.docids = {0}; + term.freqs = {1}; + term.positions_flat = {0}; + + doris::snii::writer::SniiIndexInput input; + input.index_id = index_id; + input.index_suffix = index_suffix; + input.config = doris::snii::format::IndexConfig::kDocsPositions; + input.doc_count = 2; + input.terms = {std::move(term)}; + if (has_null_bitmap) { + input.null_docids = {1}; + } + + st = writer.add_logical_index(input); + ASSERT_TRUE(st.ok()) << st; + st = writer.finish(); + ASSERT_TRUE(st.ok()) << st; + st = file_writer->close(false); + ASSERT_TRUE(st.ok()) << st; + } + private: StorageEngine* _engine_ref = nullptr; std::unique_ptr _data_dir = nullptr; @@ -376,6 +418,42 @@ TEST_F(InvertedIndexFileReaderTest, TestHasNullV2WithSmallNullBitmap) { EXPECT_FALSE(res); // Small bitmap should return false } +TEST_F(InvertedIndexFileReaderTest, TestHasNullSniiWithNullBitmap) { + std::string index_path = kTestDir + "/has_null_snii"; + create_snii_file_with_null_bitmap(index_path + ".idx", 1, "test", true); + + InvertedIndexFileInfo file_info; + IndexFileReader reader(io::global_local_filesystem(), index_path, + InvertedIndexStorageFormatPB::SNII, file_info); + + Status init_status = reader.init(4096); + EXPECT_TRUE(init_status.ok()); + + MockTabletIndex tablet_index(1, "test"); + bool res = false; + Status status = reader.has_null(&tablet_index, &res); + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(res); +} + +TEST_F(InvertedIndexFileReaderTest, TestHasNullSniiWithoutNullBitmap) { + std::string index_path = kTestDir + "/has_null_snii_without_nulls"; + create_snii_file_with_null_bitmap(index_path + ".idx", 1, "test", false); + + InvertedIndexFileInfo file_info; + IndexFileReader reader(io::global_local_filesystem(), index_path, + InvertedIndexStorageFormatPB::SNII, file_info); + + Status init_status = reader.init(4096); + EXPECT_TRUE(init_status.ok()); + + MockTabletIndex tablet_index(1, "test"); + bool res = true; + Status status = reader.has_null(&tablet_index, &res); + EXPECT_TRUE(status.ok()); + EXPECT_FALSE(res); +} + // Test case for has_null method with V2 format stream nullptr TEST_F(InvertedIndexFileReaderTest, TestHasNullV2StreamNullptr) { std::string index_path = kTestDir + "/has_null_stream_null"; @@ -440,4 +518,4 @@ TEST_F(InvertedIndexFileReaderTest, TestGetAllDirectoriesError) { EXPECT_TRUE(result.value().empty()); // Should be empty since no init was called } -} // namespace doris::segment_v2 \ No newline at end of file +} // namespace doris::segment_v2 diff --git a/be/test/storage/segment/inverted_index_file_writer_test.cpp b/be/test/storage/segment/inverted_index_file_writer_test.cpp index f0a7680d25152b..d182603ac144d9 100644 --- a/be/test/storage/segment/inverted_index_file_writer_test.cpp +++ b/be/test/storage/segment/inverted_index_file_writer_test.cpp @@ -1315,8 +1315,7 @@ class MockRowsetWriter : public RowsetWriter { public: MockRowsetWriter(const io::FileSystemSPtr& fs, const std::string& segment_path_prefix, const RowsetId& rowset_id, TabletSchemaSPtr tablet_schema, - ReaderType compaction_type = ReaderType::READER_QUERY) - : RowsetWriter() { + ReaderType compaction_type = ReaderType::READER_QUERY) { _context.rowset_id = rowset_id; _context.tablet_schema = tablet_schema; _context.compaction_type = compaction_type; @@ -1344,8 +1343,8 @@ class MockRowsetWriter : public RowsetWriter { RowsetSharedPtr manual_build(const RowsetMetaSharedPtr& rowset_meta) override { return nullptr; } - PUniqueId load_id() override { return PUniqueId(); } - Version version() override { return Version(); } + PUniqueId load_id() override { return {}; } + Version version() override { return {}; } int64_t num_rows() const override { return 0; } int64_t num_rows_updated() const override { return 0; } int64_t num_rows_deleted() const override { return 0; } @@ -1725,8 +1724,12 @@ TEST_F(IndexFileWriterTest, GetIndexFileNamesTest) { bool found_1 = false; bool found_2 = false; for (const auto& name : file_names) { - if (name == expected_name_1) found_1 = true; - if (name == expected_name_2) found_2 = true; + if (name == expected_name_1) { + found_1 = true; + } + if (name == expected_name_2) { + found_2 = true; + } } EXPECT_TRUE(found_1); @@ -2181,6 +2184,29 @@ TEST_F(IndexFileWriterTest, EmptyIndexV2Test) { ASSERT_TRUE(close_status.ok()); } +TEST_F(IndexFileWriterTest, EmptySniiBeginCloseClosesUnderlyingWriter) { + io::FileWriterPtr file_writer; + std::string snii_index_path_prefix = _index_path_prefix + "_snii"; + std::string index_path = + InvertedIndexDescriptor::get_index_file_path_v2(snii_index_path_prefix); + io::FileWriterOptions opts; + Status st = _fs->create_file(index_path, &file_writer, &opts); + ASSERT_TRUE(st.ok()); + + io::FileWriter* raw_file_writer = file_writer.get(); + IndexFileWriter writer(_fs, snii_index_path_prefix, _rowset_id, _seg_id, + InvertedIndexStorageFormatPB::SNII, std::move(file_writer)); + ASSERT_EQ(raw_file_writer->state(), io::FileWriter::State::OPENED); + + Status close_status = writer.begin_close(); + ASSERT_TRUE(close_status.ok()); + EXPECT_EQ(raw_file_writer->state(), io::FileWriter::State::ASYNC_CLOSING); + + close_status = writer.finish_close(); + ASSERT_TRUE(close_status.ok()); + EXPECT_EQ(raw_file_writer->state(), io::FileWriter::State::CLOSED); +} + // Test for StreamSinkFileWriter path in close() TEST_F(IndexFileWriterTest, StreamSinkFileWriterCloseTest) { // Create a writer without providing a file_writer to trigger the StreamSinkFileWriter path diff --git a/be/test/storage/segment/inverted_index_fs_directory_test.cpp b/be/test/storage/segment/inverted_index_fs_directory_test.cpp index d42559a0e39975..99cd9d8b613cc7 100644 --- a/be/test/storage/segment/inverted_index_fs_directory_test.cpp +++ b/be/test/storage/segment/inverted_index_fs_directory_test.cpp @@ -287,6 +287,58 @@ TEST_F(DorisFSDirectoryTest, FSIndexInputReadInternalWithBytesReadError) { _CLDELETE(input); } +TEST_F(DorisFSDirectoryTest, FSIndexInputReadInternalRecordsIndexIOStatsAndContext) { + std::filesystem::path test_file = _tmp_dir / "test_file_with_stats"; + std::ofstream ofs(test_file); + ofs << "test content for stats"; + ofs.close(); + + lucene::store::IndexInput* input = nullptr; + CLuceneError error; + + bool result = + DorisFSDirectory::FSIndexInput::open(_fs, test_file.string().c_str(), input, error); + EXPECT_TRUE(result); + + io::FileCacheStatistics stats; + io::IOContext io_ctx; + io_ctx.is_disposable = true; + io_ctx.is_index_data = false; + io_ctx.read_file_cache = false; + io_ctx.file_cache_stats = &stats; + + input->setIoContext(&io_ctx); + input->setIndexFile(true); + + uint8_t buffer[6]; + input->readBytes(buffer, 6, false); + EXPECT_EQ(std::string(reinterpret_cast(buffer), 6), "test c"); + + const auto* captured = static_cast(input->getIoContext()); + EXPECT_TRUE(captured->is_inverted_index); + EXPECT_TRUE(captured->is_index_data); + EXPECT_FALSE(captured->read_file_cache); + EXPECT_TRUE(captured->is_disposable); + EXPECT_EQ(captured->file_cache_stats, &stats); + + EXPECT_EQ(stats.inverted_index_request_bytes, 6); + EXPECT_EQ(stats.inverted_index_read_bytes, 6); + EXPECT_EQ(stats.inverted_index_range_read_count, 1); + EXPECT_EQ(stats.inverted_index_serial_read_rounds, 1); + + input->setIoContext(nullptr); + captured = static_cast(input->getIoContext()); + EXPECT_TRUE(captured->is_inverted_index); + EXPECT_TRUE(captured->is_index_data); + EXPECT_EQ(captured->file_cache_stats, nullptr); + + input->setIndexFile(false); + captured = static_cast(input->getIoContext()); + EXPECT_FALSE(captured->is_index_data); + + _CLDELETE(input); +} + // Test 19: FSIndexOutput init error TEST_F(DorisFSDirectoryTest, FSIndexOutputInitError) { DebugPoints::instance()->add( @@ -841,4 +893,4 @@ TEST_F(DorisFSDirectoryTest, PrivGetFN) { } } -} // namespace doris::segment_v2 \ No newline at end of file +} // namespace doris::segment_v2 diff --git a/be/test/storage/segment/segment_iterator_count_emit_shortcut_test.cpp b/be/test/storage/segment/segment_iterator_count_emit_shortcut_test.cpp new file mode 100644 index 00000000000000..d141637c48b6d0 --- /dev/null +++ b/be/test/storage/segment/segment_iterator_count_emit_shortcut_test.cpp @@ -0,0 +1,419 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// G03 count-emission shortcut white-box tests. Given a post-apply _row_bitmap +// of cardinality N (the exact count G02 fabricated), the shortcut must emit +// EXACTLY N rows shaped as NOT-NULL defaults across VStatisticsIterator-sized +// batches, then EOF -- byte-for-byte what today's per-rowid batch loop emits +// for a count-fastpath scan, minus the per-rowid work. The engage decision +// must admit only the provably emission-only configuration and refuse on any +// deviation (falling through to today's loop). Uses the established +// `#define private public` convention of segment_iterator_limit_opt_test.cpp +// over a bare SegmentIterator (no real segment needed: the shortcut never +// touches segment data). +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/block/column_with_type_and_name.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "storage/index/index_query_context.h" +#include "storage/olap_common.h" +#include "storage/segment/count_on_index_fastpath.h" +#include "storage/tablet/tablet_schema.h" + +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wkeyword-macro" +#endif +#define private public +#define protected public +#include "storage/segment/segment_iterator.h" +#undef private +#undef protected +#if defined(__clang__) +#pragma clang diagnostic pop +#endif + +namespace doris::segment_v2 { + +namespace { + +constexpr uint64_t kBatchRows = SegmentIterator::kCountEmitBatchRows; // 65535 + +void reset_emit_counters() { + internal::count_emit_test_counters() = internal::CountEmitTestCounters {}; +} + +uint64_t emit_hits() { + return internal::count_emit_test_counters().count_emit_shortcut_hits; +} + +uint64_t emit_batches() { + return internal::count_emit_test_counters().count_emit_shortcut_batches; +} + +// DUP-key schema shaped like a COUNT_ON_INDEX scan: an INT key plus the +// (nullable STRING) match column. Both columns end up defaults-filled by +// today's path once the index consumed the MATCH predicate. +TabletSchemaSPtr make_tablet_schema() { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + auto* key_col = schema_pb.add_column(); + key_col->set_unique_id(0); + key_col->set_name("k1"); + key_col->set_type("INT"); + key_col->set_is_key(true); + key_col->set_is_nullable(true); + auto* match_col = schema_pb.add_column(); + match_col->set_unique_id(1); + match_col->set_name("content"); + match_col->set_type("STRING"); + match_col->set_is_key(false); + match_col->set_is_nullable(true); + auto tablet_schema = std::make_shared(); + tablet_schema->init_from_pb(schema_pb); + return tablet_schema; +} + +SchemaSPtr make_read_schema(const TabletSchemaSPtr& tablet_schema) { + std::vector read_column_ids(tablet_schema->num_columns()); + for (uint32_t cid = 0; cid < read_column_ids.size(); ++cid) { + read_column_ids[cid] = cid; + } + return std::make_shared(tablet_schema->columns(), read_column_ids); +} + +Block make_block_for(const Schema& schema) { + Block block; + for (size_t i = 0; i < schema.num_column_ids(); ++i) { + const auto* col_desc = schema.column(schema.column_id(i)); + auto data_type = Schema::get_data_type_ptr(*col_desc); + block.insert( + ColumnWithTypeAndName(data_type->create_column(), data_type, col_desc->name())); + } + return block; +} + +struct Fixture { + TabletSchemaSPtr tablet_schema; + SchemaSPtr read_schema; + std::unique_ptr iter; + OlapReaderStatistics stats; + + Fixture() { + tablet_schema = make_tablet_schema(); + read_schema = make_read_schema(tablet_schema); + iter = std::make_unique(nullptr, read_schema); + iter->_opts.tablet_schema = tablet_schema; + iter->_opts.push_down_agg_type_opt = TPushAggOp::COUNT_ON_INDEX; + iter->_opts.stats = &stats; + // State _lazy_init/_vec_init_lazy_materialization would have produced + // for a count-fastpath scan: no predicate columns, no lazy + // materialization, index fully answered every column's conditions. + iter->_is_pred_column.resize(read_schema->columns().size(), false); + iter->_lazy_materialization_read = false; + iter->_storage_name_and_type.resize(read_schema->columns().size()); + for (size_t i = 0; i < read_schema->num_column_ids(); ++i) { + ColumnId cid = read_schema->column_id(i); + iter->_storage_name_and_type[cid] = + std::make_pair(read_schema->column(cid)->name(), + Schema::get_data_type_ptr(*read_schema->column(cid))); + iter->_need_read_data_indices[cid] = false; + } + } + + // Simulates the reader having fabricated a count bitmap (G02 hit). + void set_hit() { iter->_count_fastpath_hit = true; } + + // Simulates the _lazy_init engage step for a fabricated bitmap of + // cardinality `count`. + void engage_with_count(uint64_t count) { + iter->_count_emit_shortcut = true; + iter->_count_emit_rows_remaining = count; + } + + // Drives _emit_count_shortcut_batch until EOF; returns per-batch rows. + std::vector drain(Block* block) { + std::vector batch_rows; + while (true) { + Status st = iter->_emit_count_shortcut_batch(block); + if (st.is()) { + EXPECT_EQ(block->rows(), 0U) << "EOF must deliver an empty block"; + break; + } + EXPECT_TRUE(st.ok()) << st.to_string(); + batch_rows.push_back(block->rows()); + } + return batch_rows; + } +}; + +uint64_t total_rows(const std::vector& batches) { + uint64_t total = 0; + for (size_t rows : batches) { + total += rows; + } + return total; +} + +} // namespace + +// The emitted total must EQUAL the fabricated count for counts spanning +// several batches plus a tail: this is the count-equality contract with +// today's per-rowid loop (which emits exactly |_row_bitmap| rows). +TEST(CountEmitShortcut, EmitsExactCountAcrossBatchesThenEof) { + reset_emit_counters(); + Fixture fx; + const uint64_t count = 3 * kBatchRows + 3395; // 200000 + fx.engage_with_count(count); + + Block block = make_block_for(*fx.read_schema); + const auto batches = fx.drain(&block); + + ASSERT_EQ(batches.size(), 4U); + EXPECT_EQ(batches[0], kBatchRows); + EXPECT_EQ(batches[1], kBatchRows); + EXPECT_EQ(batches[2], kBatchRows); + EXPECT_EQ(batches[3], 3395U); + EXPECT_EQ(total_rows(batches), count); + EXPECT_EQ(fx.stats.raw_rows_read, static_cast(count)); + EXPECT_EQ(emit_batches(), 4U); + + // A second call after EOF stays EOF (the scanner may re-poll). + Status st = fx.iter->_emit_count_shortcut_batch(&block); + EXPECT_TRUE(st.is()); + EXPECT_EQ(block.rows(), 0U); +} + +// Batch-boundary counts: exact batch multiples and off-by-one neighbours all +// sum to the requested count with ceil(count / batch) batches. +TEST(CountEmitShortcut, BatchBoundaryCountsAreExact) { + for (const uint64_t count : {uint64_t(1), kBatchRows - 1, kBatchRows, kBatchRows + 1}) { + reset_emit_counters(); + Fixture fx; + fx.engage_with_count(count); + Block block = make_block_for(*fx.read_schema); + const auto batches = fx.drain(&block); + EXPECT_EQ(total_rows(batches), count) << "count: " << count; + const uint64_t expected_batches = (count + kBatchRows - 1) / kBatchRows; + EXPECT_EQ(batches.size(), expected_batches) << "count: " << count; + EXPECT_EQ(emit_batches(), expected_batches) << "count: " << count; + for (size_t rows : batches) { + EXPECT_LE(rows, kBatchRows); + } + } +} + +// Empty result (df == 0, e.g. the term missed the dict): immediate EOF with +// zero emitted rows -- identical to today's empty-bitmap first batch. +TEST(CountEmitShortcut, EmptyResultIsImmediateEof) { + reset_emit_counters(); + Fixture fx; + fx.engage_with_count(0); + Block block = make_block_for(*fx.read_schema); + Status st = fx.iter->_emit_count_shortcut_batch(&block); + EXPECT_TRUE(st.is()); + EXPECT_EQ(block.rows(), 0U); + EXPECT_EQ(fx.stats.raw_rows_read, 0); + EXPECT_EQ(emit_batches(), 0U); +} + +// Nullable columns must be filled with NOT-NULL defaults exactly like the +// _no_need_read_key_data/_prune_column split fill. A raw +// ColumnNullable::insert_many_defaults would insert NULLs, and count(col) +// (planned under COUNT_ON_INDEX for count(match_col)) counts non-null values: +// NULL-filled rows would collapse the count to zero. +TEST(CountEmitShortcut, NullableColumnsEmitNotNullDefaults) { + Fixture fx; + fx.engage_with_count(1000); + Block block = make_block_for(*fx.read_schema); + ASSERT_TRUE(fx.iter->_emit_count_shortcut_batch(&block).ok()); + ASSERT_EQ(block.rows(), 1000U); + + // INT key column: not-null zeros. + const auto* key_col = assert_cast(block.get_by_position(0).column.get()); + EXPECT_FALSE(key_col->has_null()); + const auto& key_data = + assert_cast&>(key_col->get_nested_column()).get_data(); + for (size_t j = 0; j < key_data.size(); ++j) { + ASSERT_EQ(key_data[j], 0) << "row " << j; + } + + // STRING match column: not-null empty strings with valid offsets. + const auto* match_col = + assert_cast(block.get_by_position(1).column.get()); + EXPECT_FALSE(match_col->has_null()); + const auto& match_data = assert_cast(match_col->get_nested_column()); + ASSERT_EQ(match_data.size(), 1000U); + for (size_t j = 0; j < match_data.size(); ++j) { + ASSERT_EQ(match_data.get_data_at(j).size, 0U) << "row " << j; + } +} + +// The shortcut consumes only the CARDINALITY of the post-apply bitmap; the id +// positions are irrelevant. Pins the null-bearing G02 shape (null-disjoint +// fabrication produces a SHIFTED range like [100, 150), not [0, 50)). +TEST(CountEmitShortcut, NullDisjointFabricatedShapeCountsByCardinality) { + Fixture fx; + fx.iter->_row_bitmap = roaring::Roaring {}; + fx.iter->_row_bitmap.addRange(100, 150); + // The exact engage math from _lazy_init. + fx.engage_with_count(fx.iter->_row_bitmap.cardinality()); + Block block = make_block_for(*fx.read_schema); + const auto batches = fx.drain(&block); + EXPECT_EQ(total_rows(batches), 50U); +} + +// Segment iterators keep independent countdowns: a multi-segment scan is the +// sum of per-segment counts, each emitted by its own iterator. +TEST(CountEmitShortcut, MultipleIteratorsEmitIndependently) { + reset_emit_counters(); + Fixture fx1; + Fixture fx2; + fx1.engage_with_count(kBatchRows + 5000); // 2 batches + fx2.engage_with_count(3); // 1 batch + Block block1 = make_block_for(*fx1.read_schema); + Block block2 = make_block_for(*fx2.read_schema); + + const auto batches1 = fx1.drain(&block1); + const auto batches2 = fx2.drain(&block2); + EXPECT_EQ(total_rows(batches1), kBatchRows + 5000); + EXPECT_EQ(total_rows(batches2), 3U); + EXPECT_EQ(batches1.size(), 2U); + EXPECT_EQ(batches2.size(), 1U); + EXPECT_EQ(emit_batches(), 3U); +} + +// G02->G03 handshake teardown: the reply flag is captured into the iterator +// and BOTH context flags are cleared so no later read_from_index call can +// observe or forge them. +TEST(CountEmitShortcut, CaptureHitRecordsReplyAndResetsHandshake) { + Fixture fx; + fx.iter->_index_query_context = std::make_shared(); + fx.iter->_index_query_context->count_on_index_fastpath = true; + fx.iter->_index_query_context->count_on_index_fastpath_hit = true; + + fx.iter->_capture_count_fastpath_hit(); + EXPECT_TRUE(fx.iter->_count_fastpath_hit); + EXPECT_FALSE(fx.iter->_index_query_context->count_on_index_fastpath); + EXPECT_FALSE(fx.iter->_index_query_context->count_on_index_fastpath_hit); + + // No reply (row-accurate decode / cache hit): captured false. + fx.iter->_index_query_context->count_on_index_fastpath = true; + fx.iter->_capture_count_fastpath_hit(); + EXPECT_FALSE(fx.iter->_count_fastpath_hit); + EXPECT_FALSE(fx.iter->_index_query_context->count_on_index_fastpath); + + // Null context (index machinery never initialized): no crash, no hit. + fx.iter->_index_query_context = nullptr; + fx.iter->_capture_count_fastpath_hit(); + EXPECT_FALSE(fx.iter->_count_fastpath_hit); +} + +// Engage admits the one emission-only configuration and counts a seam hit. +TEST(CountEmitShortcut, EngageAdmitsCleanCountFastpathState) { + reset_emit_counters(); + Fixture fx; + fx.set_hit(); + Block block = make_block_for(*fx.read_schema); + EXPECT_TRUE(fx.iter->_should_engage_count_emit_shortcut(&block)); + EXPECT_EQ(emit_hits(), 1U); +} + +// Engage refusals: every deviation keeps today's per-rowid loop. The seam +// stays untouched on refusals. +TEST(CountEmitShortcut, EngageRefusalTruthTable) { + reset_emit_counters(); + // Without the reader hit (row-accurate bitmap): refuse. + { + Fixture fx; + Block block = make_block_for(*fx.read_schema); + EXPECT_FALSE(fx.iter->_should_engage_count_emit_shortcut(&block)); + } + // A surviving evaluation stage (e.g. index-eval downgrade kept the expr). + { + Fixture fx; + fx.set_hit(); + fx.iter->_is_need_expr_eval = true; + Block block = make_block_for(*fx.read_schema); + EXPECT_FALSE(fx.iter->_should_engage_count_emit_shortcut(&block)); + } + // A pushed-down read limit must keep the limit-aware loop. + { + Fixture fx; + fx.set_hit(); + fx.iter->_opts.read_limit = 10; + Block block = make_block_for(*fx.read_schema); + EXPECT_FALSE(fx.iter->_should_engage_count_emit_shortcut(&block)); + } + // Condition-cache writes are only produced by the real batch loop. + { + Fixture fx; + fx.set_hit(); + fx.iter->_opts.condition_cache_digest = 7; + Block block = make_block_for(*fx.read_schema); + EXPECT_FALSE(fx.iter->_should_engage_count_emit_shortcut(&block)); + } + // Rowid consumers need real row ids. + { + Fixture fx; + fx.set_hit(); + fx.iter->_opts.record_rowids = true; + Block block = make_block_for(*fx.read_schema); + EXPECT_FALSE(fx.iter->_should_engage_count_emit_shortcut(&block)); + } + // Block shape != read schema (e.g. delete-condition column beyond block). + { + Fixture fx; + fx.set_hit(); + Block block = make_block_for(*fx.read_schema); + block.erase(1); + EXPECT_FALSE(fx.iter->_should_engage_count_emit_shortcut(&block)); + } + // A column today's path would REALLY read (index did not answer its + // conditions): values would come from column data, not defaults. + { + Fixture fx; + fx.set_hit(); + fx.iter->_need_read_data_indices.erase(1); + Block block = make_block_for(*fx.read_schema); + EXPECT_FALSE(fx.iter->_should_engage_count_emit_shortcut(&block)); + } + // A storage->schema cast column: today's path emits CAST(file-type + // default), which the shortcut does not reproduce. Simulated by giving + // the INT key column the STRING column's storage type. + { + Fixture fx; + fx.set_hit(); + fx.iter->_storage_name_and_type[0].second = + Schema::get_data_type_ptr(*fx.read_schema->column(1)); + Block block = make_block_for(*fx.read_schema); + EXPECT_FALSE(fx.iter->_should_engage_count_emit_shortcut(&block)); + } + EXPECT_EQ(emit_hits(), 0U); +} + +} // namespace doris::segment_v2 diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java index 449e0333066f14..d1e95dfcf5747c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java @@ -105,6 +105,12 @@ public static void checkInvertedIndexParser(String indexColName, PrimitiveType c checkInvertedIndexProperties(properties, colType, invertedIndexFileStorageFormat); } + if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.SNII + && !colType.isStringType() && !colType.isArrayType()) { + throw new AnalysisException("SNII inverted index storage format only supports string columns, column: " + + indexColName + " type: " + colType); + } + // default is "none" if not set if (parser == null) { parser = INVERTED_INDEX_PARSER_NONE; diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index ee3091ec8e1751..51049bd28c8a94 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -384,11 +384,15 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V2); } else if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.V3) { schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V3); + } else if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.SNII) { + schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.SNII); } else if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.DEFAULT) { if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) { schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V1); } else if (Config.inverted_index_storage_format.equalsIgnoreCase("V2")) { schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V2); + } else if (Config.inverted_index_storage_format.equalsIgnoreCase("SNII")) { + schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.SNII); } else { schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V3); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 305fcf19064603..424d25d98de6d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -1231,6 +1231,8 @@ public static TInvertedIndexFileStorageFormat analyzeInvertedIndexFileStorageFor return TInvertedIndexFileStorageFormat.V1; } else if (Config.inverted_index_storage_format.equalsIgnoreCase("V2")) { return TInvertedIndexFileStorageFormat.V2; + } else if (Config.inverted_index_storage_format.equalsIgnoreCase("SNII")) { + return TInvertedIndexFileStorageFormat.SNII; } else { return TInvertedIndexFileStorageFormat.V3; } @@ -1242,11 +1244,15 @@ public static TInvertedIndexFileStorageFormat analyzeInvertedIndexFileStorageFor return TInvertedIndexFileStorageFormat.V2; } else if (invertedIndexFileStorageFormat.equalsIgnoreCase("v3")) { return TInvertedIndexFileStorageFormat.V3; + } else if (invertedIndexFileStorageFormat.equalsIgnoreCase("snii")) { + return TInvertedIndexFileStorageFormat.SNII; } else if (invertedIndexFileStorageFormat.equalsIgnoreCase("default")) { if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) { return TInvertedIndexFileStorageFormat.V1; } else if (Config.inverted_index_storage_format.equalsIgnoreCase("V2")) { return TInvertedIndexFileStorageFormat.V2; + } else if (Config.inverted_index_storage_format.equalsIgnoreCase("SNII")) { + return TInvertedIndexFileStorageFormat.SNII; } else { return TInvertedIndexFileStorageFormat.V3; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BuildIndexOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BuildIndexOp.java index 494e756538b112..bf5aac95225629 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BuildIndexOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BuildIndexOp.java @@ -31,6 +31,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.UserException; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; @@ -134,6 +135,10 @@ public void validate(ConnectContext ctx) throws UserException { } IndexType indexType = existedIdx.getIndexType(); + OlapTable olapTable = (OlapTable) table; + if (olapTable.getInvertedIndexFileStorageFormat() == TInvertedIndexFileStorageFormat.SNII) { + throw new AnalysisException("BUILD INDEX is not supported for SNII inverted index storage format yet"); + } if ((Config.isNotCloudMode() && indexType == IndexType.NGRAM_BF) || indexType == IndexType.BLOOMFILTER || (Config.isCloudMode() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index fd30dacc9d1d8e..a0a2718a232c3e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -857,8 +857,10 @@ public void validate(ConnectContext ctx) { } if (indexDef.getIndexType() == IndexType.ANN) { if (invertedIndexFileStorageFormat != null - && invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.V1) { - throw new AnalysisException("ANN index is not supported in index format V1"); + && (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.V1 + || invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.SNII)) { + throw new AnalysisException("ANN index is not supported in index format " + + invertedIndexFileStorageFormat); } } for (String indexColName : indexDef.getColumnNames()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java index 8630d80b7dc0ab..36f256994a7116 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java @@ -164,6 +164,11 @@ public void checkColumn(ColumnDefinition column, KeysType keysType, "ANN index can only be used in DUP_KEYS table or UNIQUE_KEYS table with" + " merge-on-write enabled"); } + if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.V1 + || invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.SNII) { + throw new AnalysisException("ANN index is not supported in index format " + + invertedIndexFileStorageFormat); + } return; } @@ -177,6 +182,17 @@ public void checkColumn(ColumnDefinition column, KeysType keysType, throw new AnalysisException(colType + " is not supported in " + indexType.toString() + " index. " + "invalid index: " + name); } + if (indexType == IndexType.INVERTED + && invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.SNII) { + boolean isStringIndex = colType.isStringLikeType() + || (colType.isArrayType() + && ((ArrayType) colType).getItemType().isStringLikeType()); + if (!isStringIndex) { + throw new AnalysisException( + "SNII inverted index storage format does not support BKD index on column: " + + indexColName); + } + } // In inverted index format v1, each subcolumn of a variant has its own index file, leading to high IOPS. // when the subcolumn type changes, it may result in missing files, causing link file failure. @@ -264,8 +280,10 @@ public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKe "ANN index can only be used in DUP_KEYS table or UNIQUE_KEYS table with" + " merge-on-write enabled"); } - if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.V1) { - throw new AnalysisException("ANN index is not supported in index format V1"); + if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.V1 + || invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.SNII) { + throw new AnalysisException("ANN index is not supported in index format " + + invertedIndexFileStorageFormat); } return; } @@ -280,9 +298,16 @@ public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKe throw new AnalysisException(colType + " is not supported in " + indexType.toString() + " index. " + "invalid index: " + name); } - - if (indexType == IndexType.ANN && !colType.isArrayType()) { - throw new AnalysisException("ANN index column must be array type"); + if (indexType == IndexType.INVERTED + && invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.SNII) { + boolean isStringIndex = colType.isStringType() + || (colType.isArrayType() + && ((org.apache.doris.catalog.ArrayType) columnType).getItemType().isStringType()); + if (!isStringIndex) { + throw new AnalysisException( + "SNII inverted index storage format does not support BKD index on column: " + + indexColName); + } } // In inverted index format v1, each subcolumn of a variant has its own index file, leading to high IOPS. diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java index fa6260d19f7a8d..8a836b6b5d6f2c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java @@ -46,6 +46,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.task.AgentTask; import org.apache.doris.task.AgentTaskQueue; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import org.apache.doris.thrift.TStatusCode; import org.apache.doris.thrift.TTaskType; import org.apache.doris.transaction.FakeTransactionIDGenerator; @@ -195,6 +196,47 @@ public void testBuildIndexIndexChange() throws UserException { Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState()); } + @Test + public void testBuildIndexRejectedForSniiStorageFormat() throws UserException { + if (fakeEnv != null) { + fakeEnv.close(); + } + fakeEnv = new FakeEnv(); + if (fakeEditLog != null) { + fakeEditLog.close(); + } + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + SchemaChangeHandler schemaChangeHandler = Env.getCurrentEnv().getSchemaChangeHandler(); + ArrayList alterOps = new ArrayList<>(); + Database db = masterEnv.getInternalCatalog().getDbOrDdlException(CatalogTestUtil.testDbId1); + OlapTable olapTable = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId1); + String indexName = "index1"; + TableNameInfo tableNameInfo = new TableNameInfo(masterEnv.getInternalCatalog().getName(), db.getName(), + olapTable.getName()); + IndexDefinition indexDefinition = new IndexDefinition(indexName, false, + Lists.newArrayList(olapTable.getBaseSchema().get(1).getName()), + "INVERTED", + Maps.newHashMap(), "balabala"); + CreateIndexOp createIndexClause = new CreateIndexOp(tableNameInfo, indexDefinition, false); + ConnectContext connectContext = new ConnectContext(); + createIndexClause.validate(connectContext); + alterOps.add(createIndexClause); + schemaChangeHandler.process(alterOps, db, olapTable); + TInvertedIndexFileStorageFormat originalFormat = olapTable.getInvertedIndexFileStorageFormat(); + try { + olapTable.setInvertedIndexFileStorageFormat(TInvertedIndexFileStorageFormat.SNII); + BuildIndexOp buildIndexClause = new BuildIndexOp(tableNameInfo, indexName, null, false); + buildIndexClause.validate(connectContext); + Assert.fail("BUILD INDEX should be rejected for SNII inverted index storage format."); + } catch (AnalysisException e) { + Assert.assertTrue(e.getMessage().contains( + "BUILD INDEX is not supported for SNII inverted index storage format yet")); + } finally { + olapTable.setInvertedIndexFileStorageFormat(originalFormat); + } + } + @Test public void testDropIndexIndexChange() throws UserException { if (fakeEnv != null) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java index 7b41ddc95cf840..060e687b495242 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java @@ -18,7 +18,9 @@ package org.apache.doris.nereids.trees.plans.commands; import org.apache.doris.catalog.AggregateType; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.Type; import org.apache.doris.catalog.info.IndexType; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition; @@ -57,6 +59,68 @@ void testVariantIndexFormatV1() throws AnalysisException { } } + @Test + void testSniiInvertedIndexColumnTypes() throws AnalysisException { + IndexDefinition def = new IndexDefinition("snii_index", false, Lists.newArrayList("col1"), + "INVERTED", null, "comment"); + + def.checkColumn(new ColumnDefinition("col1", StringType.INSTANCE, false, AggregateType.NONE, true, + null, "comment"), KeysType.DUP_KEYS, false, TInvertedIndexFileStorageFormat.SNII); + def.checkColumn(new ColumnDefinition("col1", ArrayType.of(StringType.INSTANCE), false, + AggregateType.NONE, true, null, "comment"), KeysType.DUP_KEYS, false, + TInvertedIndexFileStorageFormat.SNII); + + AnalysisException intException = Assertions.assertThrows(AnalysisException.class, () -> + def.checkColumn(new ColumnDefinition("col1", IntegerType.INSTANCE, false, AggregateType.NONE, + true, null, "comment"), KeysType.DUP_KEYS, false, + TInvertedIndexFileStorageFormat.SNII)); + Assertions.assertTrue(intException.getMessage().contains("does not support BKD index")); + + AnalysisException arrayIntException = Assertions.assertThrows(AnalysisException.class, () -> + def.checkColumn(new ColumnDefinition("col1", ArrayType.of(IntegerType.INSTANCE), false, + AggregateType.NONE, true, null, "comment"), KeysType.DUP_KEYS, false, + TInvertedIndexFileStorageFormat.SNII)); + Assertions.assertTrue(arrayIntException.getMessage().contains("does not support BKD index")); + } + + @Test + void testSniiInvertedIndexCatalogColumnTypes() throws AnalysisException { + IndexDefinition def = new IndexDefinition("snii_index", false, Lists.newArrayList("col1"), + "INVERTED", null, "comment"); + + def.checkColumn(new Column("col1", Type.STRING, true), KeysType.DUP_KEYS, false, + TInvertedIndexFileStorageFormat.SNII); + def.checkColumn(new Column("col1", org.apache.doris.catalog.ArrayType.create(Type.STRING), true), + KeysType.DUP_KEYS, false, TInvertedIndexFileStorageFormat.SNII); + + AnalysisException intException = Assertions.assertThrows(AnalysisException.class, () -> + def.checkColumn(new Column("col1", Type.INT, true), KeysType.DUP_KEYS, false, + TInvertedIndexFileStorageFormat.SNII)); + Assertions.assertTrue(intException.getMessage().contains("does not support BKD index")); + + AnalysisException arrayIntException = Assertions.assertThrows(AnalysisException.class, () -> + def.checkColumn(new Column("col1", org.apache.doris.catalog.ArrayType.create(Type.INT), true), + KeysType.DUP_KEYS, false, TInvertedIndexFileStorageFormat.SNII)); + Assertions.assertTrue(arrayIntException.getMessage().contains("does not support BKD index")); + } + + @Test + void testSniiRejectsAnnIndex() { + IndexDefinition def = new IndexDefinition("ann_index", false, Lists.newArrayList("col1"), + "ANN", null, "comment"); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, () -> + def.checkColumn(new ColumnDefinition("col1", ArrayType.of(FloatType.INSTANCE), false, + AggregateType.NONE, false, null, "comment"), KeysType.DUP_KEYS, false, + TInvertedIndexFileStorageFormat.SNII)); + Assertions.assertTrue(exception.getMessage().contains("ANN index is not supported in index format SNII")); + + AnalysisException catalogException = Assertions.assertThrows(AnalysisException.class, () -> + def.checkColumn(new Column("col1", org.apache.doris.catalog.ArrayType.create(Type.FLOAT), false), + KeysType.DUP_KEYS, false, TInvertedIndexFileStorageFormat.SNII)); + Assertions.assertTrue(catalogException.getMessage().contains( + "ANN index is not supported in index format SNII")); + } + void testArrayTypeSupport() throws AnalysisException { IndexDefinition def = new IndexDefinition("array_index", false, Lists.newArrayList("col1"), "INVERTED", null, "array test"); diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index dfc008ee05124e..eb270f9f4eab76 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -446,6 +446,7 @@ enum InvertedIndexStorageFormatPB { V1 = 0; V2 = 1; V3 = 2; + SNII = 3; } // Tablet-level storage format. Values match TStorageFormat (Thrift) integer values so diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index 07a4be72e8330b..5c977cea79bffd 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -197,7 +197,8 @@ enum TCompressionType { enum TInvertedIndexStorageFormat { DEFAULT = 0, // Default format, unspecified storage method. V1 = 1, // Index per idx: Each index is stored separately based on its identifier. - V2 = 2 // Segment id per idx: Indexes are organized based on segment identifiers, grouping indexes by their associated segment. + V2 = 2, // Segment id per idx: Indexes are organized based on segment identifiers, grouping indexes by their associated segment. + SNII = 4 // SNII native inverted index storage format } enum TBinlogFormat { diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index c6b9c705307380..d088a936b9e05f 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -130,7 +130,8 @@ enum TInvertedIndexFileStorageFormat { DEFAULT = 0, // Default format, unspecified storage method. V1 = 1, // Index per idx: Each index is stored separately based on its identifier. V2 = 2, // Segment id per idx: Indexes are organized based on segment identifiers, grouping indexes by their associated segment. - V3 = 3 // Position and dictionary compression + V3 = 3, // Position and dictionary compression + SNII = 4 // SNII native inverted index storage format } struct TScalarType { diff --git a/regression-test/data/inverted_index_p0/storage_format/test_storage_format_snii.out b/regression-test/data/inverted_index_p0/storage_format/test_storage_format_snii.out new file mode 100644 index 00000000000000..33e05cf4214d2f --- /dev/null +++ b/regression-test/data/inverted_index_p0/storage_format/test_storage_format_snii.out @@ -0,0 +1,16 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !match_any -- +1 +2 + +-- !match_all -- +1 + +-- !match_phrase -- +5 + +-- !null_bitmap -- +4 + +-- !array_contains -- +1 diff --git a/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_snii.groovy b/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_snii.groovy new file mode 100644 index 00000000000000..7800350fb6b753 --- /dev/null +++ b/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_snii.groovy @@ -0,0 +1,212 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_storage_format_snii", "p0, nonConcurrent") { + sql "DROP TABLE IF EXISTS test_storage_format_snii" + sql "DROP TABLE IF EXISTS test_storage_format_snii_array" + sql "DROP TABLE IF EXISTS test_storage_format_snii_add_index" + sql "DROP TABLE IF EXISTS test_storage_format_snii_bkd" + sql "DROP TABLE IF EXISTS test_storage_format_snii_array_bkd" + sql "DROP TABLE IF EXISTS test_storage_format_snii_ann" + + sql """ + CREATE TABLE test_storage_format_snii ( + id INT NULL, + body TEXT NULL, + INDEX idx_body (`body`) USING INVERTED PROPERTIES( + "parser" = "english", + "support_phrase" = "true", + "lower_case" = "true" + ) COMMENT '' + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "disable_auto_compaction" = "true", + "inverted_index_storage_format" = "SNII" + ); + """ + + sql """ + INSERT INTO test_storage_format_snii VALUES + (1, 'alpha beta gamma'), + (2, 'alpha delta'), + (3, 'beta epsilon'), + (4, NULL), + (5, 'quick brown fox'), + (6, 'quick fox'); + """ + sql "sync" + + order_qt_match_any """ + SELECT id FROM test_storage_format_snii + WHERE body MATCH_ANY 'alpha' + ORDER BY id + """ + order_qt_match_all """ + SELECT id FROM test_storage_format_snii + WHERE body MATCH_ALL 'alpha beta' + ORDER BY id + """ + order_qt_match_phrase """ + SELECT id FROM test_storage_format_snii + WHERE body MATCH_PHRASE 'quick brown' + ORDER BY id + """ + order_qt_null_bitmap """ + SELECT id FROM test_storage_format_snii + WHERE body IS NULL + ORDER BY id + """ + + sql """ + CREATE TABLE test_storage_format_snii_array ( + id INT NULL, + tags ARRAY NULL, + INDEX idx_tags (`tags`) USING INVERTED COMMENT '' + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "inverted_index_storage_format" = "SNII" + ); + """ + + sql """ + INSERT INTO test_storage_format_snii_array VALUES + (1, '["alpha", "beta"]'), + (2, '["gamma"]'), + (3, NULL); + """ + sql "sync" + + order_qt_array_contains """ + SELECT id FROM test_storage_format_snii_array + WHERE array_contains(tags, 'alpha') + ORDER BY id + """ + + test { + if (isCloudMode()) { + sql "BUILD INDEX ON test_storage_format_snii" + } else { + sql "BUILD INDEX idx_body ON test_storage_format_snii" + } + exception "BUILD INDEX is not supported for SNII inverted index storage format yet" + } + + sql """ + CREATE TABLE test_storage_format_snii_add_index ( + id INT NULL, + body TEXT NULL, + score INT NULL, + scores ARRAY NULL, + embedding ARRAY NOT NULL, + INDEX idx_body_added_table (`body`) USING INVERTED COMMENT '' + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "inverted_index_storage_format" = "SNII" + ); + """ + + test { + sql """ + ALTER TABLE test_storage_format_snii_add_index + ADD INDEX idx_score_added (`score`) USING INVERTED COMMENT '' + """ + exception "SNII inverted index storage format" + } + + test { + sql """ + ALTER TABLE test_storage_format_snii_add_index + ADD INDEX idx_scores_added (`scores`) USING INVERTED COMMENT '' + """ + exception "SNII inverted index storage format" + } + + test { + sql """ + CREATE INDEX idx_ann_added ON test_storage_format_snii_add_index (`embedding`) USING ANN PROPERTIES( + "index_type" = "hnsw", + "metric_type" = "l2_distance", + "dim" = "1" + ) + """ + exception "ANN index is not supported in index format SNII" + } + + test { + sql """ + CREATE TABLE test_storage_format_snii_bkd ( + id INT NULL, + score INT NULL, + INDEX idx_score (`score`) USING INVERTED COMMENT '' + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "inverted_index_storage_format" = "SNII" + ); + """ + exception "SNII inverted index storage format" + } + + test { + sql """ + CREATE TABLE test_storage_format_snii_array_bkd ( + id INT NULL, + scores ARRAY NULL, + INDEX idx_scores (`scores`) USING INVERTED COMMENT '' + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "inverted_index_storage_format" = "SNII" + ); + """ + exception "SNII inverted index storage format" + } + + test { + sql """ + CREATE TABLE test_storage_format_snii_ann ( + id INT NULL, + embedding ARRAY NOT NULL, + INDEX idx_ann (`embedding`) USING ANN PROPERTIES( + "index_type" = "hnsw", + "metric_type" = "l2_distance", + "dim" = "1" + ) + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "inverted_index_storage_format" = "SNII" + ); + """ + exception "ANN index is not supported in index format SNII" + } +}