From 13d6f8a7cdfea3ce92a3b0c0b3f7a8e4bf0c75ef Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Fri, 4 Sep 2026 14:25:35 +0800 Subject: [PATCH 01/12] [feature](lance) add cache in lance scan node --- be/src/common/config.cpp | 9 + be/src/common/config.h | 8 + .../format_v2/lance/lance_session_manager.cpp | 220 ++++++++++++++++++ .../format_v2/lance/lance_session_manager.h | 71 ++++++ be/src/format_v2/table/lance_reader.cpp | 47 +++- be/src/format_v2/table/lance_reader.h | 5 + .../lance/lance_session_manager_test.cpp | 130 +++++++++++ be/test/format_v2/table/lance_reader_test.cpp | 6 +- 8 files changed, 490 insertions(+), 6 deletions(-) create mode 100644 be/src/format_v2/lance/lance_session_manager.cpp create mode 100644 be/src/format_v2/lance/lance_session_manager.h create mode 100644 be/test/format_v2/lance/lance_session_manager_test.cpp diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 29ff784a2f2098..54b4e29ccf0e14 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1210,6 +1210,15 @@ DEFINE_Validator(variant_max_json_key_length, DEFINE_Validator(variant_storage_parse_mode, [](const int config) -> bool { return config >= 0 && config <= 2; }); +// Lance uses one BE-wide session so metadata/index caches and the optional Foyer data-file cache +// can be shared by all Lance dataset readers. +DEFINE_Int64(lance_index_cache_size_bytes, "6442450944"); // 6GB +DEFINE_Int64(lance_metadata_cache_size_bytes, "1073741824"); // 1GB +DEFINE_Bool(enable_lance_data_cache, "true"); +DEFINE_String(lance_data_cache_path, "${DORIS_HOME}/lance_data_cache"); +DEFINE_Int64(lance_data_cache_disk_capacity_bytes, "107374182400"); // 100GB +DEFINE_Int64(lance_data_cache_read_block_size_bytes, "1048576"); // 1MB + // block file cache DEFINE_Bool(enable_file_cache, "true"); // ATTENTION: For test only. Keep this enabled in production. diff --git a/be/src/common/config.h b/be/src/common/config.h index 7963d3331a5661..03ae5962d66146 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1251,6 +1251,14 @@ DECLARE_Bool(enable_debug_points); DECLARE_Int32(pipeline_executor_size); DECLARE_Int32(blocking_pipeline_executor_size); +// Lance shared session and optional Foyer data-file cache. +DECLARE_Int64(lance_index_cache_size_bytes); +DECLARE_Int64(lance_metadata_cache_size_bytes); +DECLARE_Bool(enable_lance_data_cache); +DECLARE_String(lance_data_cache_path); +DECLARE_Int64(lance_data_cache_disk_capacity_bytes); +DECLARE_Int64(lance_data_cache_read_block_size_bytes); + // block file cache DECLARE_Bool(enable_file_cache); DECLARE_mBool(enable_file_cache_write_from_s3_file_writer); diff --git a/be/src/format_v2/lance/lance_session_manager.cpp b/be/src/format_v2/lance/lance_session_manager.cpp new file mode 100644 index 00000000000000..c10142f9e05f45 --- /dev/null +++ b/be/src/format_v2/lance/lance_session_manager.cpp @@ -0,0 +1,220 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/lance/lance_session_manager.h" + +#include + +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/logging.h" +#include "common/metrics/doris_metrics.h" +#include "common/metrics/metrics.h" +#include "format_v2/lance/lance_reader_helper.h" + +namespace doris::format::lance { +namespace { + +DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(lance_session_index_cache_capacity_bytes, MetricUnit::BYTES); +DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(lance_session_index_cache_usage_bytes, MetricUnit::BYTES); +DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(lance_session_index_cache_entries, MetricUnit::NOUNIT); +DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(lance_session_index_cache_hits_total, MetricUnit::OPERATIONS); +DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(lance_session_index_cache_misses_total, + MetricUnit::OPERATIONS); +DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(lance_session_metadata_cache_capacity_bytes, MetricUnit::BYTES); +DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(lance_session_metadata_cache_usage_bytes, MetricUnit::BYTES); +DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(lance_session_metadata_cache_entries, MetricUnit::NOUNIT); +DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(lance_session_metadata_cache_hits_total, + MetricUnit::OPERATIONS); +DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(lance_session_metadata_cache_misses_total, + MetricUnit::OPERATIONS); + +constexpr std::string_view LANCE_SESSION_CACHE_METRICS_HOOK = "lance_session_cache"; + +int64_t metric_value(uint64_t value) { + return static_cast( + std::min(value, static_cast(std::numeric_limits::max()))); +} + +LanceSessionManager::Config load_lance_session_config() { + return { + .lance_index_cache_size_bytes = config::lance_index_cache_size_bytes, + .lance_metadata_cache_size_bytes = config::lance_metadata_cache_size_bytes, + .enable_lance_data_cache = config::enable_lance_data_cache, + .lance_data_cache_path = config::lance_data_cache_path, + .lance_data_cache_disk_capacity_bytes = config::lance_data_cache_disk_capacity_bytes, + .lance_data_cache_read_block_size_bytes = + config::lance_data_cache_read_block_size_bytes, + }; +} + +} // namespace + +class LanceSessionMetrics final { +public: + LanceSessionMetrics(LanceSession* session, const LanceSessionManager::Config& config) + : _session(session), _entity(DorisMetrics::instance()->server_entity()) { + INT_GAUGE_METRIC_REGISTER(_entity, lance_session_index_cache_capacity_bytes); + INT_GAUGE_METRIC_REGISTER(_entity, lance_session_index_cache_usage_bytes); + INT_GAUGE_METRIC_REGISTER(_entity, lance_session_index_cache_entries); + INT_COUNTER_METRIC_REGISTER(_entity, lance_session_index_cache_hits_total); + INT_COUNTER_METRIC_REGISTER(_entity, lance_session_index_cache_misses_total); + INT_GAUGE_METRIC_REGISTER(_entity, lance_session_metadata_cache_capacity_bytes); + INT_GAUGE_METRIC_REGISTER(_entity, lance_session_metadata_cache_usage_bytes); + INT_GAUGE_METRIC_REGISTER(_entity, lance_session_metadata_cache_entries); + INT_COUNTER_METRIC_REGISTER(_entity, lance_session_metadata_cache_hits_total); + INT_COUNTER_METRIC_REGISTER(_entity, lance_session_metadata_cache_misses_total); + + lance_session_index_cache_capacity_bytes->set_value(config.lance_index_cache_size_bytes); + lance_session_metadata_cache_capacity_bytes->set_value( + config.lance_metadata_cache_size_bytes); + _entity->register_hook(std::string(LANCE_SESSION_CACHE_METRICS_HOOK), + [this]() { update(); }); + update(); + } + + ~LanceSessionMetrics() { + _entity->deregister_hook(std::string(LANCE_SESSION_CACHE_METRICS_HOOK)); + METRIC_DEREGISTER(_entity, lance_session_index_cache_capacity_bytes); + METRIC_DEREGISTER(_entity, lance_session_index_cache_usage_bytes); + METRIC_DEREGISTER(_entity, lance_session_index_cache_entries); + METRIC_DEREGISTER(_entity, lance_session_index_cache_hits_total); + METRIC_DEREGISTER(_entity, lance_session_index_cache_misses_total); + METRIC_DEREGISTER(_entity, lance_session_metadata_cache_capacity_bytes); + METRIC_DEREGISTER(_entity, lance_session_metadata_cache_usage_bytes); + METRIC_DEREGISTER(_entity, lance_session_metadata_cache_entries); + METRIC_DEREGISTER(_entity, lance_session_metadata_cache_hits_total); + METRIC_DEREGISTER(_entity, lance_session_metadata_cache_misses_total); + } + +private: + void update() { + // Session caches are shared across queries, so publish one process-wide snapshot instead + // of attributing concurrent cache activity to an individual query profile. + LanceSessionCacheStats stats {}; + if (lance_session_get_cache_stats(_session, &stats) != 0) { + LOG_EVERY_N(WARNING, 100) + << lance_error("collect Lance session cache statistics").to_string(); + return; + } + lance_session_index_cache_usage_bytes->set_value( + metric_value(stats.index_cache_size_bytes)); + lance_session_index_cache_entries->set_value(metric_value(stats.index_cache_entries)); + lance_session_index_cache_hits_total->set_value(metric_value(stats.index_cache_hits)); + lance_session_index_cache_misses_total->set_value(metric_value(stats.index_cache_misses)); + lance_session_metadata_cache_usage_bytes->set_value( + metric_value(stats.metadata_cache_size_bytes)); + lance_session_metadata_cache_entries->set_value(metric_value(stats.metadata_cache_entries)); + lance_session_metadata_cache_hits_total->set_value(metric_value(stats.metadata_cache_hits)); + lance_session_metadata_cache_misses_total->set_value( + metric_value(stats.metadata_cache_misses)); + } + + LanceSession* _session; + MetricEntity* _entity; + IntGauge* lance_session_index_cache_capacity_bytes = nullptr; + IntGauge* lance_session_index_cache_usage_bytes = nullptr; + IntGauge* lance_session_index_cache_entries = nullptr; + IntCounter* lance_session_index_cache_hits_total = nullptr; + IntCounter* lance_session_index_cache_misses_total = nullptr; + IntGauge* lance_session_metadata_cache_capacity_bytes = nullptr; + IntGauge* lance_session_metadata_cache_usage_bytes = nullptr; + IntGauge* lance_session_metadata_cache_entries = nullptr; + IntCounter* lance_session_metadata_cache_hits_total = nullptr; + IntCounter* lance_session_metadata_cache_misses_total = nullptr; +}; + +LanceSessionManager& LanceSessionManager::instance() { + // Function-local static initialization is thread safe. Cache configuration is process scoped, + // so changing it requires a BE restart. + static LanceSessionManager manager(load_lance_session_config()); + return manager; +} + +LanceSessionManager::LanceSessionManager(Config config) : _config(std::move(config)) { + LOG(INFO) << "Creating BE-wide Lance session manager: lance_index_cache_size_bytes=" + << _config.lance_index_cache_size_bytes + << ", lance_metadata_cache_size_bytes=" + << _config.lance_metadata_cache_size_bytes + << ", enable_lance_data_cache=" << _config.enable_lance_data_cache + << ", lance_data_cache_path=" << _config.lance_data_cache_path + << ", lance_data_cache_disk_capacity_bytes=" + << _config.lance_data_cache_disk_capacity_bytes + << ", lance_data_cache_read_block_size_bytes=" + << _config.lance_data_cache_read_block_size_bytes + << ", foyer_memory_capacity_bytes=" + << _config.lance_data_cache_read_block_size_bytes; +} + +LanceSessionManager::~LanceSessionManager() { + _metrics.reset(); + lance_session_close(_session); +} + +Status LanceSessionManager::_initialize() { + if (_config.enable_lance_data_cache) { + const LanceDataCacheOptions data_cache_options { + .directory = _config.lance_data_cache_path.c_str(), + // Foyer's HybridCache requires a memory tier. Keep it at the minimum useful + // capacity of exactly one range-cache block; entries use WriteOnInsertion and + // are persisted to the disk tier immediately. + .memory_capacity_bytes = + static_cast(_config.lance_data_cache_read_block_size_bytes), + .disk_capacity_bytes = + static_cast(_config.lance_data_cache_disk_capacity_bytes), + .read_block_size_bytes = + static_cast(_config.lance_data_cache_read_block_size_bytes), + }; + _session = lance_session_new_with_data_cache( + static_cast(_config.lance_index_cache_size_bytes), + static_cast(_config.lance_metadata_cache_size_bytes), + &data_cache_options); + } else { + _session = lance_session_new( + static_cast(_config.lance_index_cache_size_bytes), + static_cast(_config.lance_metadata_cache_size_bytes)); + } + if (_session == nullptr) { + return lance_error("create shared Lance session"); + } + _metrics = std::make_unique(_session, _config); + return Status::OK(); +} + +Status LanceSessionManager::open_dataset(const char* uri, const char* const* storage_options, + uint64_t version, LanceDataset** dataset) { + if (uri == nullptr || dataset == nullptr) { + return Status::InvalidArgument("Lance dataset URI and output must not be null"); + } + *dataset = nullptr; + + std::call_once(_initialize_once, [this] { _initialize_status = _initialize(); }); + RETURN_IF_ERROR(_initialize_status); + + *dataset = lance_dataset_open_with_session(uri, storage_options, version, _session); + if (*dataset == nullptr) { + return lance_error("open Lance dataset with shared session"); + } + return Status::OK(); +} + +} // namespace doris::format::lance diff --git a/be/src/format_v2/lance/lance_session_manager.h b/be/src/format_v2/lance/lance_session_manager.h new file mode 100644 index 00000000000000..45dc62acad4442 --- /dev/null +++ b/be/src/format_v2/lance/lance_session_manager.h @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" + +struct LanceDataset; +struct LanceSession; + +namespace doris::format::lance { + +class LanceSessionMetrics; + +// Owns the single Lance session shared by all queries in one BE process. The session always owns +// Lance's metadata/index caches and optionally installs the Foyer data-file cache. Readers only +// open datasets through this class and do not depend on the selected data-cache implementation. +class LanceSessionManager final { +public: + struct Config { + int64_t lance_index_cache_size_bytes = 0; + int64_t lance_metadata_cache_size_bytes = 0; + bool enable_lance_data_cache = false; + std::string lance_data_cache_path; + int64_t lance_data_cache_disk_capacity_bytes = 0; + int64_t lance_data_cache_read_block_size_bytes = 0; + }; + + static LanceSessionManager& instance(); + + // The explicit configuration constructor keeps the process-global config out of focused + // manager tests. Production readers use instance(). + explicit LanceSessionManager(Config config); + ~LanceSessionManager(); + + LanceSessionManager(const LanceSessionManager&) = delete; + LanceSessionManager& operator=(const LanceSessionManager&) = delete; + + Status open_dataset(const char* uri, const char* const* storage_options, uint64_t version, + LanceDataset** dataset); + +private: + Status _initialize(); + + Config _config; + std::once_flag _initialize_once; + LanceSession* _session = nullptr; + std::unique_ptr _metrics; + Status _initialize_status = Status::OK(); +}; + +} // namespace doris::format::lance diff --git a/be/src/format_v2/table/lance_reader.cpp b/be/src/format_v2/table/lance_reader.cpp index a9fc65cdc0837f..2ae4aede5da172 100644 --- a/be/src/format_v2/table/lance_reader.cpp +++ b/be/src/format_v2/table/lance_reader.cpp @@ -38,6 +38,7 @@ #include "format_v2/lance/lance_reader_helper.h" #include "format_v2/lance/lance_runtime_filter_helper.h" #include "runtime/exec_env.h" +#include "format_v2/lance/lance_session_manager.h" #include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "storage/utils.h" @@ -139,6 +140,12 @@ Status LanceTableReader::init(TableReadOptions&& options) { TUnit::UNIT, LANCE_READER_PROFILE, 1); _execution_bytes_read = ADD_CHILD_COUNTER_WITH_LEVEL( _scanner_profile, "LanceExecutionIOBytesRead", TUnit::BYTES, LANCE_READER_PROFILE, 1); + _data_cache_bytes_read_from_cache = ADD_CHILD_COUNTER_WITH_LEVEL( + _scanner_profile, "LanceDataCacheBytesReadFromCache", TUnit::BYTES, + LANCE_READER_PROFILE, 1); + _data_cache_bytes_read_from_remote = ADD_CHILD_COUNTER_WITH_LEVEL( + _scanner_profile, "LanceDataCacheBytesReadFromRemote", TUnit::BYTES, + LANCE_READER_PROFILE, 1); _index_partition_cache_miss_loads = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LanceIndexPartitionCacheMissLoads", TUnit::UNIT, LANCE_READER_PROFILE, 1); @@ -686,12 +693,11 @@ Status LanceTableReader::_open_dataset(const DatasetKey& key) { std::unique_ptr dataset; { SCOPED_TIMER(_dataset_open_time); - dataset.reset(lance_dataset_open( + LanceDataset* raw_dataset = nullptr; + RETURN_IF_ERROR(LanceSessionManager::instance().open_dataset( key.uri.c_str(), key.storage_options.empty() ? nullptr : storage_option_ptrs.data(), - static_cast(key.version))); - } - if (dataset == nullptr) { - return lance_error("open Lance dataset"); + static_cast(key.version), &raw_dataset)); + dataset.reset(raw_dataset); } std::shared_ptr schema; RETURN_IF_ERROR(import_dataset_schema(dataset.get(), &schema)); @@ -1114,12 +1120,43 @@ void LanceTableReader::_close_dataset() { _fts_query_context = nullptr; } if (_dataset != nullptr) { + _collect_data_cache_statistics(); lance_dataset_close(_dataset); _dataset = nullptr; } _dataset_schema.reset(); } +void LanceTableReader::_collect_data_cache_statistics() { + if (_dataset == nullptr) { + return; + } + + LanceDataCacheStatistics statistics {}; + if (lance_dataset_get_data_cache_statistics(_dataset, &statistics) != 0) { + const auto status = lance_error("get Lance data cache statistics"); + LOG(WARNING) << "Failed to collect Lance data cache statistics: " << status.to_string(); + return; + } + + const auto set_counter = [](RuntimeProfile::Counter* counter, uint64_t value, + std::string_view metric_name) { + if (counter == nullptr) { + return; + } + if (value > static_cast(std::numeric_limits::max())) { + LOG(WARNING) << "Ignoring Lance data cache metric '" << metric_name << "' with value " + << value << " because it exceeds INT64_MAX"; + return; + } + COUNTER_SET(counter, static_cast(value)); + }; + set_counter(_data_cache_bytes_read_from_cache, statistics.bytes_read_from_cache, + "bytes_read_from_cache"); + set_counter(_data_cache_bytes_read_from_remote, statistics.bytes_read_from_remote, + "bytes_read_from_remote"); +} + Status LanceTableReader::_fill_block_from_lance_batch(LanceBatch* batch, Block* block, size_t* rows) { DORIS_CHECK(batch != nullptr); diff --git a/be/src/format_v2/table/lance_reader.h b/be/src/format_v2/table/lance_reader.h index c12eef397d5ff1..65828fc33f7c37 100644 --- a/be/src/format_v2/table/lance_reader.h +++ b/be/src/format_v2/table/lance_reader.h @@ -92,6 +92,9 @@ class LanceTableReader final : public TableReader { const TLanceFileDesc& lance_params) const; Status _configure_full_text_search(LanceScanner* scanner, const TLanceFileDesc& lance_params) const; + // Collect the cumulative statistics owned by this dataset handle. The Lance-C API returns an + // absolute snapshot, so this method replaces rather than increments the profile counters. + void _collect_data_cache_statistics(); // Keep lance-c's anonymous statistics typedef out of this header. _open_scanner installs the // strongly typed C callback adapter before forwarding the borrowed value here. static void _collect_scan_statistics(void* callback_ctx, const void* opaque_statistics); @@ -126,6 +129,8 @@ class LanceTableReader final : public TableReader { RuntimeProfile::Counter* _execution_iops = nullptr; RuntimeProfile::Counter* _execution_requests = nullptr; RuntimeProfile::Counter* _execution_bytes_read = nullptr; + RuntimeProfile::Counter* _data_cache_bytes_read_from_cache = nullptr; + RuntimeProfile::Counter* _data_cache_bytes_read_from_remote = nullptr; RuntimeProfile::Counter* _index_partition_cache_miss_loads = nullptr; RuntimeProfile::Counter* _index_comparisons = nullptr; std::unordered_map _lance_count_metrics; diff --git a/be/test/format_v2/lance/lance_session_manager_test.cpp b/be/test/format_v2/lance/lance_session_manager_test.cpp new file mode 100644 index 00000000000000..9e873f61d8c938 --- /dev/null +++ b/be/test/format_v2/lance/lance_session_manager_test.cpp @@ -0,0 +1,130 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/lance/lance_session_manager.h" + +#include +#include + +#include +#include +#include +#include +#include + +#include "common/metrics/doris_metrics.h" +#include "common/metrics/metrics.h" + +namespace doris::format::lance { +namespace { + +using LanceDatasetPtr = std::unique_ptr; + +std::filesystem::path lance_fixture_path() { + return std::filesystem::path(__FILE__).parent_path().parent_path() / + "table/lance/data/all_types.lance"; +} + +std::filesystem::path unique_cache_path() { + const auto suffix = std::chrono::steady_clock::now().time_since_epoch().count(); + return std::filesystem::temp_directory_path() / + ("doris_lance_foyer_cache_" + std::to_string(suffix)); +} + +TEST(LanceSessionManagerTest, SessionAndDataCacheConfigurationsAreIndependent) { + LanceSessionManager::Config config { + .lance_index_cache_size_bytes = 0, + .lance_metadata_cache_size_bytes = 0, + .enable_lance_data_cache = false, + // These are deliberately invalid and must be ignored while the data cache is off. + .lance_data_cache_path = "", + .lance_data_cache_disk_capacity_bytes = -1, + .lance_data_cache_read_block_size_bytes = -1, + }; + LanceSessionManager manager(std::move(config)); + LanceDataset* raw_dataset = nullptr; + ASSERT_TRUE(manager + .open_dataset(lance_fixture_path().c_str(), nullptr, 0, &raw_dataset) + .ok()); + LanceDatasetPtr dataset(raw_dataset, lance_dataset_close); + ASSERT_NE(dataset, nullptr); +} + +TEST(LanceSessionManagerTest, PublishesSessionCacheMetrics) { + constexpr int64_t INDEX_CACHE_CAPACITY = 8 * 1024 * 1024; + constexpr int64_t METADATA_CACHE_CAPACITY = 4 * 1024 * 1024; + LanceSessionManager::Config config { + .lance_index_cache_size_bytes = INDEX_CACHE_CAPACITY, + .lance_metadata_cache_size_bytes = METADATA_CACHE_CAPACITY, + .enable_lance_data_cache = false, + }; + LanceSessionManager manager(std::move(config)); + LanceDataset* raw_dataset = nullptr; + ASSERT_TRUE(manager.open_dataset(lance_fixture_path().c_str(), nullptr, 0, &raw_dataset).ok()); + LanceDatasetPtr dataset(raw_dataset, lance_dataset_close); + ASSERT_NE(dataset, nullptr); + + auto* entity = DorisMetrics::instance()->server_entity(); + const auto gauge = [entity](const char* name) { + return static_cast(entity->get_metric(name)); + }; + auto* index_capacity = gauge("lance_session_index_cache_capacity_bytes"); + auto* metadata_capacity = gauge("lance_session_metadata_cache_capacity_bytes"); + ASSERT_NE(index_capacity, nullptr); + ASSERT_NE(metadata_capacity, nullptr); + EXPECT_EQ(index_capacity->value(), INDEX_CACHE_CAPACITY); + EXPECT_EQ(metadata_capacity->value(), METADATA_CACHE_CAPACITY); + EXPECT_NE(gauge("lance_session_index_cache_usage_bytes"), nullptr); + EXPECT_NE(gauge("lance_session_index_cache_entries"), nullptr); + EXPECT_NE(gauge("lance_session_metadata_cache_usage_bytes"), nullptr); + EXPECT_NE(gauge("lance_session_metadata_cache_entries"), nullptr); + EXPECT_NE(entity->get_metric("lance_session_index_cache_hits_total"), nullptr); + EXPECT_NE(entity->get_metric("lance_session_index_cache_misses_total"), nullptr); + EXPECT_NE(entity->get_metric("lance_session_metadata_cache_hits_total"), nullptr); + EXPECT_NE(entity->get_metric("lance_session_metadata_cache_misses_total"), nullptr); +} + +TEST(LanceSessionManagerTest, CreatesFoyerBackedSession) { + const auto cache_path = unique_cache_path(); + std::filesystem::create_directories(cache_path); + const auto cleanup = [&cache_path] { + std::error_code error; + std::filesystem::remove_all(cache_path, error); + }; + + LanceSessionManager::Config config { + .lance_index_cache_size_bytes = 0, + .lance_metadata_cache_size_bytes = 0, + .enable_lance_data_cache = true, + .lance_data_cache_path = cache_path.string(), + .lance_data_cache_disk_capacity_bytes = 32 * 1024 * 1024, + .lance_data_cache_read_block_size_bytes = 64 * 1024, + }; + { + LanceSessionManager manager(std::move(config)); + LanceDataset* raw_dataset = nullptr; + const auto status = manager.open_dataset(lance_fixture_path().c_str(), nullptr, 0, + &raw_dataset); + LanceDatasetPtr dataset(raw_dataset, lance_dataset_close); + EXPECT_TRUE(status.ok()) << status; + EXPECT_NE(dataset, nullptr); + } + cleanup(); +} + +} // namespace +} // namespace doris::format::lance diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp index e23d7c2ea6bd21..763ffdb9db6215 100644 --- a/be/test/format_v2/table/lance_reader_test.cpp +++ b/be/test/format_v2/table/lance_reader_test.cpp @@ -933,6 +933,8 @@ TEST(LanceTableReaderVectorSearchTest, SearchesMultipleFragmentSplits) { "LanceExecutionIOOps", "LanceExecutionIORequests", "LanceExecutionIOBytesRead", + "LanceDataCacheBytesReadFromCache", + "LanceDataCacheBytesReadFromRemote", "LanceIndexPartitionCacheMissLoads", "LanceIndexComparisons", "LanceFragmentsScanned", @@ -1055,7 +1057,9 @@ TEST(LanceTableReaderVectorSearchTest, ReturnsStableGlobalRowIdsAndFetchesPayloa EXPECT_NE(fetch_profile.get_counter("LanceRowIdFetchTotalTime"), nullptr); expect_lance_profile_hierarchy(&fetch_profile, {"LanceDatasetOpenTime", "LanceRowIdTakeReadTime", - "LanceArrowToDorisBlockTime", "LanceRowIdFetchTotalTime"}); + "LanceArrowToDorisBlockTime", "LanceRowIdFetchTotalTime", + "LanceDataCacheBytesReadFromCache", + "LanceDataCacheBytesReadFromRemote"}); EXPECT_TRUE(payload_reader.close().ok()); } From 96e515c0e9db23b5da02453a5ccd5f6e7b3990f5 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Mon, 7 Sep 2026 20:55:00 +0800 Subject: [PATCH 02/12] scalar index filter --- .../lance/LanceExternalCatalog.java | 15 +-- .../datasource/lance/LanceMetadataLoader.java | 34 ++--- .../lance/source/IndexSegmentSplitPlan.java | 120 ++++++++++++++--- .../lance/source/LanceScalarIndexPlanner.java | 122 ++++++++++++++++++ .../lance/source/LanceScanNode.java | 94 ++++++++------ .../datasource/lance/source/LanceSplit.java | 14 +- .../org/apache/doris/qe/SessionVariable.java | 18 +++ .../lance/source/LanceScanNodeTest.java | 86 +++++++++++- .../lance/test_lance_fragment_grouping.groovy | 83 ++++++++++++ 9 files changed, 496 insertions(+), 90 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlanner.java create mode 100644 regression-test/suites/external_table_p0/lance/test_lance_fragment_grouping.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java index 2e23a5ef1f0316..ae0f8f717e0226 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java @@ -332,20 +332,15 @@ public boolean tableExist(SessionContext ctx, String dbName, String tblName) { } public LanceTableMetadata loadTableMetadata(String dbName, String tableName) { - return loadTableMetadata(dbName, tableName, Optional.empty(), false); + return loadTableMetadata(dbName, tableName, Optional.empty()); } public LanceTableMetadata loadTableMetadataForSearch(String dbName, String tableName) { - return loadTableMetadata(dbName, tableName, Optional.empty(), true); + return loadTableMetadata(dbName, tableName, Optional.empty()); } public LanceTableMetadata loadTableMetadata(String dbName, String tableName, Optional tableSnapshot) { - return loadTableMetadata(dbName, tableName, tableSnapshot, false); - } - - private LanceTableMetadata loadTableMetadata(String dbName, String tableName, - Optional tableSnapshot, boolean loadIndexSegments) { makeSureInitialized(); ResolvedTableAccess tableAccess = resolveTableAccess(dbName, tableName); try { @@ -366,11 +361,7 @@ private LanceTableMetadata loadTableMetadata(String dbName, String tableName, return LanceMetadataLoader.loadVersion( tableAccess.datasetUri, tableAccess.storageOptions, version, allocator); } - return loadIndexSegments - ? LanceMetadataLoader.loadLatestWithIndexSegments(tableAccess.datasetUri, - tableAccess.storageOptions, allocator) - : LanceMetadataLoader.loadLatest( - tableAccess.datasetUri, tableAccess.storageOptions, allocator); + return LanceMetadataLoader.loadLatest(tableAccess.datasetUri, tableAccess.storageOptions, allocator); } catch (Exception e) { throw new RuntimeException("Failed to load Lance table metadata for " + dbName + "." + tableName + ": " + sanitizedRootCauseMessage(e), safeCause(e)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceMetadataLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceMetadataLoader.java index d223cf98f36c71..1e9c6aa83b677c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceMetadataLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceMetadataLoader.java @@ -53,8 +53,10 @@ public static LanceTableMetadata loadLatestForTvf( String datasetUri, List storageProperties) throws Exception { try (BufferAllocator allocator = new RootAllocator(ALLOCATOR_LIMIT)) { - return loadLatest(datasetUri, - LanceStorageOptions.fromDorisStorageProperties(datasetUri, storageProperties), allocator); + // S3 TVFs do not plan FE index-segment groups; only the dataset snapshot is needed. + return loadInternal(datasetUri, + LanceStorageOptions.fromDorisStorageProperties(datasetUri, storageProperties), + OptionalLong.empty(), allocator, false); } } @@ -64,17 +66,10 @@ public static LanceTableMetadata loadLatestForTvf( *

Called by * {@link LanceExternalCatalog#loadTableMetadata(String, String, java.util.Optional)} when no * time-travel version is requested. Schema, version, and fragments are read from the same - * opened dataset snapshot. + * opened dataset snapshot, together with index coverage for fragment grouping and external searches. */ public static LanceTableMetadata loadLatest(String datasetUri, Map lanceStorageOptions, BufferAllocator allocator) throws Exception { - return loadInternal( - datasetUri, lanceStorageOptions, OptionalLong.empty(), allocator, false); - } - - /** Loads the latest fixed snapshot together with search-index segment coverage. */ - public static LanceTableMetadata loadLatestWithIndexSegments( - String datasetUri, Map lanceStorageOptions, BufferAllocator allocator) throws Exception { return loadInternal( datasetUri, lanceStorageOptions, OptionalLong.empty(), allocator, true); } @@ -90,13 +85,13 @@ public static LanceTableMetadata loadVersion(String datasetUri, Map lanceStorageOptions, long version, BufferAllocator allocator) throws Exception { return loadInternal( - datasetUri, lanceStorageOptions, OptionalLong.of(version), allocator, false); + datasetUri, lanceStorageOptions, OptionalLong.of(version), allocator, true); } /** Shared implementation for the latest-version and explicit-version public entry points. */ private static LanceTableMetadata loadInternal(String datasetUri, Map lanceStorageOptions, OptionalLong version, - BufferAllocator allocator, boolean loadIndexSegments) throws Exception { + BufferAllocator allocator, boolean includeIndexSegments) throws Exception { try (Dataset dataset = Dataset.open().allocator(allocator).uri(datasetUri) .readOptions(LanceReadOptions.build(lanceStorageOptions, version)).build()) { long resolvedVersion = dataset.version(); @@ -106,11 +101,11 @@ private static LanceTableMetadata loadInternal(String datasetUri, Integer.toUnsignedLong(fragment.getId()), fragment.metadata().getNumRows(), fragment.metadata().getPhysicalRows())); } - Map lanceFieldIds = loadIndexSegments + Map lanceFieldIds = includeIndexSegments ? loadTopLevelFieldIds(dataset) : Collections.emptyMap(); - List indexSegments = loadIndexSegments - ? loadSearchIndexSegments(dataset) : Collections.emptyList(); - return loadIndexSegments + List indexSegments = includeIndexSegments + ? loadIndexSegments(dataset) : Collections.emptyList(); + return includeIndexSegments ? LanceTableMetadata.withIndexSegments(datasetUri, resolvedVersion, dataset.getSchema(), fragments, lanceFieldIds, indexSegments, lanceStorageOptions) @@ -134,13 +129,12 @@ private static Map loadTopLevelFieldIds(Dataset dataset) { return result; } - private static List loadSearchIndexSegments(Dataset dataset) { + private static List loadIndexSegments(Dataset dataset) { List result = new ArrayList<>(); - for (IndexDescription description : dataset.describeIndices()) { + for (IndexDescription description : LanceIndexMetadataLoader.describeUserIndexes(dataset)) { String metric = parseMetric(description.getDetailsJson()); for (Index segment : description.getSegments()) { - if (segment.indexType() == null || (segment.indexType().getValue() < 100 - && segment.indexType() != org.lance.index.IndexType.INVERTED)) { + if (segment.indexType() == null) { continue; } List fragmentIds = segment.fragments() diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/IndexSegmentSplitPlan.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/IndexSegmentSplitPlan.java index b890242ddff16c..7faa276d1579ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/IndexSegmentSplitPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/IndexSegmentSplitPlan.java @@ -21,18 +21,19 @@ import org.apache.doris.spi.Split; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; -/** Builds external-search splits from physical Lance index segments and optional fragments. */ +/** Builds splits from index-segment coverage and optional uncovered fragments. */ final class IndexSegmentSplitPlan { private final String datasetUri; private final long version; - private final List splits; + private final List splits; private final Set indexSegmentFragmentIds = new HashSet<>(); - private long maxPhysicalRows = 1; IndexSegmentSplitPlan(String datasetUri, long version, int expectedIndexSegments) { this.datasetUri = datasetUri; @@ -59,25 +60,112 @@ boolean isCoveredByIndexSegment(long fragmentId) { void addIndexSegmentSplit(UUID indexSegmentUuid, List fragmentIds, long physicalRows) { indexSegmentFragmentIds.addAll(fragmentIds); - addSplit(LanceSplit.forIndexSegment( - datasetUri, version, indexSegmentUuid, fragmentIds, physicalRows), physicalRows); + splits.add(LanceSplit.forIndexSegment( + datasetUri, version, indexSegmentUuid, fragmentIds, physicalRows)); } - void addUnindexedFragmentSplit(LanceFragmentInfo fragment) { - long physicalRows = Math.max(fragment.getPhysicalRows(), 1); - addSplit(LanceSplit.forFragment( - datasetUri, version, fragment.getId(), physicalRows), physicalRows); + // Ordinary scans use segment coverage only to group fragments. No UUID is sent to Lance. + void addIndexSegmentFragmentGroup(List fragmentIds, long physicalRows) { + indexSegmentFragmentIds.addAll(fragmentIds); + addFragmentGroup(fragmentIds, physicalRows); } - List buildSplits() { - for (Split split : splits) { - split.setTargetSplitSize(maxPhysicalRows); + // Manifest order and row-based split weights are shared by ordinary scans and fallbacks. + // An empty index coverage set groups all fragments; vector fallbacks use a group size of 1. + void addUncoveredFragments(Iterable fragments, int fragmentsPerSplit) { + if (fragmentsPerSplit < 1) { + throw new IllegalArgumentException("fragmentsPerSplit must be positive"); + } + List fragmentIds = new ArrayList<>(); + long physicalRows = 0; + for (LanceFragmentInfo fragment : fragments) { + if (isCoveredByIndexSegment(fragment.getId())) { + continue; + } + fragmentIds.add(fragment.getId()); + physicalRows += Math.max(fragment.getPhysicalRows(), 1); + if (fragmentIds.size() == fragmentsPerSplit) { + addFragmentGroup(fragmentIds, physicalRows); + fragmentIds.clear(); + physicalRows = 0; + } + } + if (!fragmentIds.isEmpty()) { + addFragmentGroup(fragmentIds, physicalRows); + } + } + + private void addFragmentGroup(List fragmentIds, long physicalRows) { + splits.add(LanceSplit.forFragments(datasetUri, version, fragmentIds, physicalRows)); + } + + /** Subdivides ordinary fragment groups to provide work for the available backends. */ + List buildFragmentSplits(int numBackends, Map visibleFragments) { + int targetSplits = Math.min(numBackends, visibleFragments.size()); + if (splits.size() >= targetSplits) { + return buildSplits(); + } + int[] groupCounts = new int[splits.size()]; + Arrays.fill(groupCounts, 1); + for (int count = splits.size(); count < targetSplits; count++) { + int largestGroup = -1; + double largestWeight = -1; + for (int i = 0; i < splits.size(); i++) { + LanceSplit split = splits.get(i); + // Never duplicate a search segment UUID or assign a fragment to two splits. + if (split.hasIndexSegmentUuids() || groupCounts[i] >= split.getFragmentIds().size()) { + continue; + } + double weight = (double) split.getSelfSplitWeight() / groupCounts[i]; + if (weight > largestWeight) { + largestGroup = i; + largestWeight = weight; + } + } + if (largestGroup < 0) { + break; + } + groupCounts[largestGroup]++; + } + List originalSplits = new ArrayList<>(splits); + splits.clear(); + for (int i = 0; i < originalSplits.size(); i++) { + LanceSplit split = originalSplits.get(i); + if (groupCounts[i] == 1) { + splits.add(split); + } else { + subdivideFragmentGroup(split, groupCounts[i], visibleFragments); + } } - return splits; + return buildSplits(); } - private void addSplit(LanceSplit split, long physicalRows) { - splits.add(split); - maxPhysicalRows = Math.max(maxPhysicalRows, physicalRows); + private void subdivideFragmentGroup(LanceSplit split, int groups, + Map visibleFragments) { + List fragments = split.getFragmentIds(); + long remainingRows = split.getSelfSplitWeight(); + int start = 0; + for (int remainingGroups = groups; remainingGroups > 0; remainingGroups--) { + long targetRows = remainingRows / remainingGroups; + long physicalRows = 0; + int end = start; + // Retain fragment order and leave at least one fragment for each remaining group. + int lastEnd = fragments.size() - (remainingGroups - 1); + do { + physicalRows += Math.max(visibleFragments.get(fragments.get(end++)).getPhysicalRows(), 1); + } while (end < lastEnd && physicalRows < targetRows); + addFragmentGroup(fragments.subList(start, end), physicalRows); + start = end; + remainingRows -= physicalRows; + } + } + + List buildSplits() { + // Recompute after subdivision; the old largest group's weight is no longer applicable. + long maxPhysicalRows = splits.stream().mapToLong(LanceSplit::getSelfSplitWeight).max().orElse(1); + for (LanceSplit split : splits) { + split.setTargetSplitSize(maxPhysicalRows); + } + return new ArrayList<>(splits); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlanner.java new file mode 100644 index 00000000000000..1b1e4295577d02 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlanner.java @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.source; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.datasource.lance.LanceFragmentInfo; +import org.apache.doris.datasource.lance.LanceIndexSegmentInfo; +import org.apache.doris.datasource.lance.LanceTableMetadata; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.stream.Collectors; + +/** Groups fragments using scalar-index coverage. Index search planning remains entirely in Lance. */ +final class LanceScalarIndexPlanner { + static final class Plan { + final String indexName; + final IndexSegmentSplitPlan splits; + private final long coveredRows; + + Plan(String indexName, IndexSegmentSplitPlan splits, long coveredRows) { + this.indexName = indexName; + this.splits = splits; + this.coveredRows = coveredRows; + } + } + + static Plan plan(LanceTableMetadata metadata, List pushedConjuncts, + Map visibleFragments) { + if (metadata.getVersion() <= 0) { + return null; + } + Set filterFields = collectFilterFields(metadata, pushedConjuncts); + if (filterFields.isEmpty()) { + return null; + } + // Group all segments of each logical index before checking coverage. Name order + // provides a stable winner when multiple indices cover the same number of rows. + Map> indices = metadata.getIndexSegments().stream() + .collect(Collectors.groupingBy(LanceIndexSegmentInfo::getIndexName, + TreeMap::new, Collectors.toList())); + Plan selected = null; + for (List segments : indices.values()) { + // The loader copies logical-index field metadata into every segment. An index + // is a candidate when any of its fields occurs in a pushed filter. + // Non-vector indices include INVERTED (FTS); this selects grouping, not search semantics. + LanceIndexSegmentInfo index = segments.get(0); + if (index.isVectorIndex() || Collections.disjoint(index.getFieldIds(), filterFields)) { + continue; + } + Plan candidate = groupFragments(metadata, segments, visibleFragments); + if (candidate != null && (selected == null || candidate.coveredRows > selected.coveredRows)) { + selected = candidate; + } + } + return selected; + } + + private static Set collectFilterFields(LanceTableMetadata metadata, List pushedConjuncts) { + Set slots = new HashSet<>(); + pushedConjuncts.forEach(expr -> expr.collect(SlotRef.class, slots)); + Set fields = new HashSet<>(); + for (SlotRef slot : slots) { + metadata.getLanceFieldId(slot.getColumnName()).ifPresent(fields::add); + } + return fields; + } + + private static Plan groupFragments(LanceTableMetadata metadata, List segments, + Map visibleFragments) { + IndexSegmentSplitPlan splits = new IndexSegmentSplitPlan( + metadata.getDatasetUri(), metadata.getVersion(), segments.size()); + Set coveredFragments = new HashSet<>(); + long coveredRows = 0; + for (LanceIndexSegmentInfo segment : segments) { + if (!segment.getFragmentIds().isPresent()) { + return null; + } + List fragments = new ArrayList<>(); + long physicalRows = 0; + for (Long fragmentId : segment.getFragmentIds().get()) { + LanceFragmentInfo fragment = visibleFragments.get(fragmentId); + if (fragment == null) { + continue; + } + // A visible fragment must have one output owner. Reject overlapping + // coverage for this candidate and let the caller try another index. + if (!coveredFragments.add(fragmentId)) { + return null; + } + fragments.add(fragmentId); + physicalRows += Math.max(fragment.getPhysicalRows(), 1); + coveredRows += Math.max(fragment.getPhysicalRows(), 0); + } + if (!fragments.isEmpty()) { + splits.addIndexSegmentFragmentGroup(fragments, physicalRows); + } + } + return splits.isEmpty() ? null : new Plan(segments.get(0).getIndexName(), splits, coveredRows); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java index 7c0b90f26b1c5f..6e426d6e2c2cfc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.lance.source; +import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.Column; @@ -73,7 +74,8 @@ * *

These modes share dataset metadata, storage properties, and BE scan-range serialization. * Keeping them in one node prevents those common parts from drifting apart. The search request is - * also an explicit mode marker. Ordinary scans are split by fragment. Indexed vector searches are + * also an explicit mode marker. Ordinary scans group fragments using index coverage or a fixed + * fragment count. Indexed vector searches are * split by physical index segment, with uncovered fragments retained as flat-search fallbacks. * Full-text searches are split only by committed inverted-index segments, with coverage governed * by the request's STRICT or INDEX_ONLY mode. Each search split produces local candidates; a Doris @@ -101,6 +103,9 @@ private enum SearchKind { private final Set lazyMaterializedColumns = new HashSet<>(); private long plannedVersion = -1; private int plannedFragments; + private int plannedFragmentsPerSplit = 0; + private final List lancePushedConjuncts = new ArrayList<>(); + private LanceScalarIndexPlanner.Plan scalarIndexPlan; private int plannedUnindexedFragments; private int plannedIndexSegments; private int plannedIndexFragments; @@ -214,14 +219,14 @@ private TLanceScanParams getOrCreateLanceScanParams() { return params.getLanceScanParams(); } - // A fragment-level LIMIT can be pushed into an ordinary Lance scan only when every predicate + // A split-level LIMIT can be pushed into an ordinary Lance scan only when every predicate // is already pushed into Lance (conjuncts is empty). Otherwise Doris re-filters the returned - // rows and truncating a fragment early could drop valid results. + // rows and truncating a split early could drop valid results. // // OFFSET needs no special handling: the Nereids SplitLimit rule rewrites Limit(limit, offset) // into a global Limit(limit, offset) over a local Limit(limit + offset, 0), and the local // bound is what lands on this scan node. So getLimit() already accounts for the offset and - // getOffset() is always 0 here; each fragment fetches up to limit + offset rows and the upper + // getOffset() is always 0 here; each split fetches up to limit + offset rows and the upper // global LIMIT still applies the offset and the final bound. private boolean canPushDownLimit() { return hasLimit() && conjuncts.isEmpty(); @@ -248,6 +253,8 @@ protected void convertPredicate() { new LancePredicateConverter(plannedMetadata.getSchema()).convert(conjuncts); lanceSubstraitFilter = result.getSubstraitFilter(); lancePushdownPredicate = result.getDebugPredicate(); + lancePushedConjuncts.clear(); + lancePushedConjuncts.addAll(result.getPushedConjuncts()); conjuncts.removeAll(result.getPushedConjuncts()); } } @@ -271,9 +278,14 @@ public List getSplits(int numBackends) throws UserException { LanceTableMetadata metadata = plannedMetadata; plannedVersion = metadata.getVersion(); plannedFragments = metadata.getFragments().size(); + plannedFragmentsPerSplit = searchKind == SearchKind.NORMAL ? sessionVariable.lanceFragmentsPerSplit : 1; + if (plannedFragmentsPerSplit < 0) { + throw new UserException("lance_fragments_per_split must be non-negative"); + } plannedUnindexedFragments = searchKind == SearchKind.NORMAL ? 0 : plannedFragments; plannedIndexSegments = 0; plannedIndexFragments = 0; + scalarIndexPlan = null; if (searchKind != SearchKind.NORMAL && plannedVersion <= 0) { throw new UserException( "Lance external search requires a fixed positive dataset version"); @@ -295,13 +307,15 @@ public List getSplits(int numBackends) throws UserException { return indexSplits.get(); } } - break; + IndexSegmentSplitPlan plan = new IndexSegmentSplitPlan( + metadata.getDatasetUri(), metadata.getVersion(), 0); + plan.addUncoveredFragments(visibleFragments.values(), 1); + return plan.buildSplits(); case NORMAL: - break; + return createNormalFragmentSplits(metadata, visibleFragments, numBackends); default: throw new IllegalStateException("Unsupported Lance search kind " + searchKind); } - return createFragmentSplits(metadata, visibleFragments); } // COUNT(*)/COUNT(1) with no filter is answered from Lance metadata. Each carrier contains a @@ -354,24 +368,26 @@ private Map getVisibleFragments(LanceTableMetadata meta return visible; } - private List createFragmentSplits(LanceTableMetadata metadata, - Map visibleFragments) { - long targetRows = 1; - for (LanceFragmentInfo fragment : visibleFragments.values()) { - targetRows = Math.max(targetRows, Math.max(fragment.getPhysicalRows(), 1)); + private List createNormalFragmentSplits(LanceTableMetadata metadata, + Map visibleFragments, int numBackends) { + if (plannedFragmentsPerSplit > 0) { + // Keep the debug grouping exact, even when it produces fewer splits than BEs. + IndexSegmentSplitPlan plan = new IndexSegmentSplitPlan(metadata.getDatasetUri(), metadata.getVersion(), 0); + plan.addUncoveredFragments(visibleFragments.values(), plannedFragmentsPerSplit); + return plan.buildSplits(); } - - // Keep one fragment per split. Use the largest fragment's physical row count as the - // normalization baseline for split weights, so backend scheduling reflects the relative - // amount of physical data each fragment scans, including rows covered by deletion metadata. - List splits = new ArrayList<>(visibleFragments.size()); - for (LanceFragmentInfo fragment : visibleFragments.values()) { - LanceSplit split = LanceSplit.forFragment(metadata.getDatasetUri(), metadata.getVersion(), - fragment.getId(), fragment.getPhysicalRows()); - split.setTargetSplitSize(targetRows); - splits.add(split); + scalarIndexPlan = LanceScalarIndexPlanner.plan(metadata, lancePushedConjuncts, visibleFragments); + IndexSegmentSplitPlan plan; + if (scalarIndexPlan != null) { + plan = scalarIndexPlan.splits; + plannedIndexSegments = plan.splitCount(); + plannedIndexFragments = plan.indexSegmentFragmentCount(); + plannedUnindexedFragments = plannedFragments - plannedIndexFragments; + } else { + plan = new IndexSegmentSplitPlan(metadata.getDatasetUri(), metadata.getVersion(), 0); } - return splits; + plan.addUncoveredFragments(visibleFragments.values(), 1); + return plan.buildFragmentSplits(numBackends, visibleFragments); } private Optional> createVectorIndexSegmentSplits(LanceTableMetadata metadata, @@ -531,15 +547,6 @@ private static long sumPhysicalRows(List fragmentIds, return physicalRows; } - private static void appendUnindexedFragmentSplits(IndexSegmentSplitPlan plan, - Map visibleFragments) { - for (LanceFragmentInfo fragment : visibleFragments.values()) { - if (!plan.isCoveredByIndexSegment(fragment.getId())) { - plan.addUnindexedFragmentSplit(fragment); - } - } - } - private boolean isVectorIndexEnabled() { // default use_index is true if (!externalSearchRequest.isSetVectorSearchOptions()) { @@ -574,11 +581,9 @@ protected void setScanParams(TFileRangeDesc rangeDesc, Split split) { lanceParams.setDatasetUri(lanceSplit.getDatasetUri()); lanceParams.setVersion(lanceSplit.getVersion()); if (lanceSplit.hasFragmentIds()) { - if (searchKind == SearchKind.NORMAL && lanceSplit.getTableLevelRowCount() < 0 - && (lanceSplit.getFragmentIds().size() != 1 - || lanceSplit.hasIndexSegmentUuids())) { + if (searchKind == SearchKind.NORMAL && lanceSplit.hasIndexSegmentUuids()) { throw new IllegalArgumentException( - "Ordinary Lance scan split must contain one fragment and no index segment"); + "Ordinary Lance scan split must not contain index segment UUIDs"); } if (searchKind == SearchKind.FULL_TEXT && !lanceSplit.hasIndexSegmentUuids()) { throw new IllegalArgumentException( @@ -601,8 +606,8 @@ protected void setScanParams(TFileRangeDesc rangeDesc, Split split) { // BE serves the row count from table_level_row_count below, leaving fragment_ids unset. throw new IllegalArgumentException("Lance scan split must contain fragments"); } - // Push LIMIT into each ordinary fragment scanner only when it is safe to truncate that - // fragment early. External searches use their own per-split candidate bound. + // Push LIMIT into each ordinary split scanner only when it is safe to truncate that + // split early. External searches use their own per-split candidate bound. if (searchKind == SearchKind.NORMAL && canPushDownLimit()) { lanceParams.setLimit(getLimit()); } @@ -695,6 +700,19 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { .append(((LanceExternalCatalog) lanceTable.getCatalog()).getLanceCatalogType()).append("\n"); result.append(prefix).append("lanceVersion=").append(plannedVersion).append("\n"); result.append(prefix).append("lanceFragments=").append(plannedFragments).append("\n"); + if (plannedFragmentsPerSplit > 0) { + result.append(prefix).append("lanceFragmentGrouping=DEBUG\n"); + result.append(prefix).append("lanceFragmentsPerSplit=").append(plannedFragmentsPerSplit).append("\n"); + } else if (scalarIndexPlan == null) { + result.append(prefix).append("lanceFragmentGrouping=FRAGMENT\n"); + } else { + result.append(prefix).append("lanceFragmentGrouping=INDEX_SEGMENT\n"); + result.append(prefix).append("lanceGroupingIndex=").append(scalarIndexPlan.indexName).append("\n"); + result.append(prefix).append("lanceGroupingIndexSegments=").append(plannedIndexSegments).append("\n"); + result.append(prefix).append("lanceGroupingIndexedFragments=").append(plannedIndexFragments).append("\n"); + result.append(prefix).append("lanceGroupingUnindexedFragments=") + .append(plannedUnindexedFragments).append("\n"); + } if (canPushDownLimit()) { result.append(prefix).append("lanceLimit=").append(getLimit()).append("\n"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceSplit.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceSplit.java index 5125ba6c749ce5..1335062259ab9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceSplit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceSplit.java @@ -27,8 +27,8 @@ import java.util.UUID; /** - * A Lance scan split. Catalog and S3 scans normally use one fixed-version fragment per split. - * Indexed vector search uses one physical index segment and its covered fragments per split. + * A Lance scan split. Ordinary catalog scans use one or more fixed-version fragments per split. + * Indexed scans can use one physical index segment and its covered fragments per split. * Backend-local TVFs use one whole-dataset latest-version split. */ public class LanceSplit extends FileSplit { @@ -42,7 +42,15 @@ public class LanceSplit extends FileSplit { public static LanceSplit forFragment( String datasetUri, long version, long fragmentId, long physicalRows) { - return new LanceSplit(datasetUri, version, Collections.singletonList(fragmentId), + return forFragments(datasetUri, version, Collections.singletonList(fragmentId), physicalRows); + } + + public static LanceSplit forFragments( + String datasetUri, long version, List fragmentIds, long physicalRows) { + if (fragmentIds == null || fragmentIds.isEmpty()) { + throw new IllegalArgumentException("Lance fragment split must contain fragments"); + } + return new LanceSplit(datasetUri, version, fragmentIds, Collections.emptyList(), physicalRows); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index c10bebc476ac62..51d0ed5e36c45b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -578,6 +578,8 @@ public String toString() { // Split size for ExternalFileScanNode. Default value 0 means use the block size of HDFS/S3. public static final String FILE_SPLIT_SIZE = "file_split_size"; + public static final String LANCE_FRAGMENTS_PER_SPLIT = "lance_fragments_per_split"; + public static final String FILE_SPLIT_SIZE_ON_FE = "file_split_size_on_fe"; public static final String FILE_SPLIT_SIZE_ON_BE = "file_split_size_on_be"; @@ -2573,6 +2575,16 @@ public Map getForceEagerAggHintMap() { @VariableMgr.VarAttr(name = FILE_SPLIT_SIZE, needForward = true) public long fileSplitSize = 0; + @VariableMgr.VarAttr(name = LANCE_FRAGMENTS_PER_SPLIT, needForward = true, + flag = VariableMgr.INVISIBLE, fuzzy = false, + checker = "checkLanceFragmentsPerSplit", description = { + "普通 Lance 扫描的调试参数。默认 0 自动划分；正数强制按指定 fragment 数分组，" + + "跳过索引分组和补足 BE 数量的逻辑。不影响 vector/FTS 查询。", + "Debug override for ordinary Lance scans. Default 0 uses automatic splitting; " + + "a positive value groups that many fragments per split, bypassing index grouping " + + "and minimum BE parallelism. Does not affect vector/FTS queries."}) + public int lanceFragmentsPerSplit = 0; + @VariableMgr.VarAttr(name = FILE_SPLIT_SIZE_ON_FE, needForward = true, description = { "支持 BE 细粒度切分时，FE 粗粒度文件分片的目标大小，单位为字节，默认为 512MB", "Target size in bytes for FE coarse-grained file splits when BE refinement is supported. " @@ -6417,6 +6429,12 @@ public void checkBatchSize(String batchSize) { } + public void checkLanceFragmentsPerSplit(String value) { + if (Integer.parseInt(value) < 0) { + throw new InvalidParameterException("lance_fragments_per_split must be non-negative"); + } + } + private static final long PREFERRED_BLOCK_SIZE_BYTES_MIN = 1048576L; // 1MB private static final long PREFERRED_BLOCK_SIZE_BYTES_MAX = 536870912L; // 512MB diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java index 99636231145adb..c144af39f5f683 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java @@ -49,6 +49,7 @@ import org.mockito.Mockito; import java.nio.ByteBuffer; +import java.security.InvalidParameterException; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -75,6 +76,84 @@ public void testAdditionalTypesRejectSmoothUpgradeSourceBackend() throws Excepti Assert.assertTrue(exception.getMessage().contains("10001")); } + @Test + public void testGroupedFragmentsPreserveCoverageWeightsAndScanParams() throws Exception { + SessionVariable sessionVariable = new SessionVariable(); + sessionVariable.lanceFragmentsPerSplit = 2; + LanceScanNode node = newNode(sessionVariable); + setMetadata(node, LanceTableMetadata.withoutIndexSegments( + "s3://bucket/table.lance", 42, new Schema(Collections.emptyList()), + Arrays.asList( + new LanceFragmentInfo(7, 10, 1000), + new LanceFragmentInfo(11, 250, 250), + new LanceFragmentInfo(13, 0, 0), + new LanceFragmentInfo(17, 499, 499), + new LanceFragmentInfo(23, 125, 125)), + Collections.emptyMap())); + + List splits = node.getSplits(2); + + Assert.assertEquals(3, splits.size()); + List> expectedIds = Arrays.asList(Arrays.asList(7L, 11L), + Arrays.asList(13L, 17L), Collections.singletonList(23L)); + long[] expectedRows = {1250, 500, 125}; + long[] expectedWeights = {100, 40, 10}; + for (int i = 0; i < splits.size(); i++) { + LanceSplit split = (LanceSplit) splits.get(i); + Assert.assertEquals(expectedIds.get(i), split.getFragmentIds()); + Assert.assertEquals(expectedRows[i], split.getSelfSplitWeight()); + Assert.assertEquals(1250L, split.getTargetSplitSize().longValue()); + Assert.assertEquals(expectedWeights[i], split.getSplitWeight().getRawValue()); + TFileRangeDesc range = new TFileRangeDesc(); + node.setScanParams(range, split); + Assert.assertEquals(expectedIds.get(i), range.getTableFormatParams().getLanceParams().getFragmentIds()); + Assert.assertEquals(42L, range.getTableFormatParams().getLanceParams().getVersion()); + Assert.assertEquals("s3://bucket/table.lance", + range.getTableFormatParams().getLanceParams().getDatasetUri()); + Assert.assertFalse(range.getTableFormatParams().getLanceParams().isSetIndexSegmentUuids()); + Assert.assertFalse(range.getTableFormatParams().getLanceParams().isSetLimit()); + } + + node.setLimit(10); + TFileRangeDesc limitedRange = new TFileRangeDesc(); + node.setScanParams(limitedRange, splits.get(0)); + Assert.assertEquals(10L, limitedRange.getTableFormatParams().getLanceParams().getLimit()); + } + + @Test + public void testFragmentGroupLargerThanDatasetAndEmptyDataset() throws Exception { + SessionVariable sessionVariable = new SessionVariable(); + sessionVariable.lanceFragmentsPerSplit = Integer.MAX_VALUE; + LanceScanNode node = newNode(sessionVariable); + setMetadata(node, LanceTableMetadata.withoutIndexSegments( + "s3://bucket/table.lance", 42, new Schema(Collections.emptyList()), + Arrays.asList(new LanceFragmentInfo(7, 0, 0), new LanceFragmentInfo(11, 0, 0)), + Collections.emptyMap())); + + List splits = node.getSplits(2); + + Assert.assertEquals(1, splits.size()); + Assert.assertEquals(Arrays.asList(7L, 11L), ((LanceSplit) splits.get(0)).getFragmentIds()); + Assert.assertEquals(2L, ((LanceSplit) splits.get(0)).getSelfSplitWeight()); + Assert.assertEquals(100L, splits.get(0).getSplitWeight().getRawValue()); + + setMetadata(node, LanceTableMetadata.withoutIndexSegments( + "s3://bucket/table.lance", 43, new Schema(Collections.emptyList()), + Collections.emptyList(), Collections.emptyMap())); + Assert.assertTrue(node.getSplits(2).isEmpty()); + } + + @Test + public void testFragmentGroupSizeValidation() { + SessionVariable sessionVariable = new SessionVariable(); + Assert.assertEquals(0, sessionVariable.lanceFragmentsPerSplit); + sessionVariable.checkLanceFragmentsPerSplit("0"); + sessionVariable.checkLanceFragmentsPerSplit("1"); + sessionVariable.checkLanceFragmentsPerSplit("8"); + Assert.assertThrows(InvalidParameterException.class, + () -> sessionVariable.checkLanceFragmentsPerSplit("-1")); + } + @Test public void testFragmentRowsDetermineSplitWeights() throws Exception { LanceTableMetadata metadata = LanceTableMetadata.withoutIndexSegments( @@ -608,6 +687,8 @@ public void testSplitSearchRetainsTopKPlusOffsetCandidates() { @Test public void testLanceSplitRejectsInvalidRangeFieldsInFrontend() { + assertInvalidSplit(() -> LanceSplit.forFragments("s3://bucket/table.lance", 42, + Collections.emptyList(), 1), "Lance fragment split must contain fragments"); assertInvalidSplit(() -> LanceSplit.forFragment("", 42, 1, 1), "Lance dataset URI must not be empty"); assertInvalidSplit(() -> LanceSplit.forFragment("s3://bucket/table.lance", -1, 1, 1), @@ -635,9 +716,12 @@ private static LanceScanNode newSearchNode( ? request.getSearchQuery().getVectorSearch().getColumn() : request.getSearchQuery().getFullTextSearch().getColumn(); int searchFieldId = metadata.getLanceFieldId(searchColumn).orElse(-1); + // Ordinary-scan grouping must not change any vector or full-text split expectations. + SessionVariable sessionVariable = new SessionVariable(); + sessionVariable.lanceFragmentsPerSplit = 8; return LanceScanNode.forExternalSearch( new PlanNodeId(0), new TupleDescriptor(new TupleId(0)), null, - metadata, searchFieldId, request, new SessionVariable()); + metadata, searchFieldId, request, sessionVariable); } private static void setMetadata(LanceScanNode node, LanceTableMetadata metadata) throws Exception { diff --git a/regression-test/suites/external_table_p0/lance/test_lance_fragment_grouping.groovy b/regression-test/suites/external_table_p0/lance/test_lance_fragment_grouping.groovy new file mode 100644 index 00000000000000..de5ce49afd1969 --- /dev/null +++ b/regression-test/suites/external_table_p0/lance/test_lance_fragment_grouping.groovy @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_lance_fragment_grouping", "p0,external") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable Lance test because the Iceberg MinIO environment is disabled.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_lance_fragment_grouping" + // Reuse the immutable multi-fragment fixture from the vector-search suites for ordinary scans. + String tableName = "${catalogName}.doris.vs_ivf_pq_f32" + def originalGroupSize = sql("SELECT @@session.lance_fragments_per_split")[0][0] + + sql """DROP CATALOG IF EXISTS `${catalogName}`""" + try { + sql """ + CREATE CATALOG `${catalogName}` PROPERTIES ( + "type" = "lance", + "lance.catalog.type" = "filesystem", + "warehouse" = "s3://warehouse/lance", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.region" = "us-east-1", + "use_path_style" = "true" + ) + """ + + List queries = [ + "SELECT row_id, category FROM ${tableName} ORDER BY row_id", + "SELECT row_id FROM ${tableName} WHERE row_id BETWEEN 250 AND 770 " + + "AND category = 'odd' ORDER BY row_id", + "SELECT row_id FROM ${tableName} WHERE row_id < 10 OR row_id > 1000 ORDER BY row_id", + "SELECT row_id FROM ${tableName} WHERE row_id > 250 " + + "AND MOD(row_id, 3) = 1 ORDER BY row_id LIMIT 17 OFFSET 3", + "SELECT row_id FROM ${tableName} WHERE row_id < 0 ORDER BY row_id", + "SELECT COUNT(*) FROM ${tableName}" + ] + sql "SET lance_fragments_per_split = 1" + def expected = queries.collect { query -> sql(query) } + assertTrue(expected[0].size() > 0) + String baselinePlan = sql("EXPLAIN ${queries[0]}").collect { it[0] }.join("\n") + def splitMatcher = baselinePlan =~ /inputSplitNum=(\d+)/ + assertTrue(splitMatcher.find()) + int fragmentCount = splitMatcher.group(1).toInteger() + assertTrue(fragmentCount > 1) + + [2, 3, fragmentCount + 1].each { groupSize -> + sql "SET lance_fragments_per_split = ${groupSize}" + int expectedSplits = (fragmentCount + groupSize - 1).intdiv(groupSize) + explain { + sql(queries[0]) + contains "lanceFragmentsPerSplit=${groupSize}" + contains "inputSplitNum=${expectedSplits}," + contains "lanceFragments=${fragmentCount}" + } + queries.eachWithIndex { query, index -> + assertEquals(expected[index], sql(query)) + } + } + } finally { + sql "SET lance_fragments_per_split = ${originalGroupSize}" + sql """DROP CATALOG IF EXISTS `${catalogName}`""" + } +} From af5f3b5a7108f242f00c4c401b3198074b7db7d9 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Tue, 8 Sep 2026 16:33:03 +0800 Subject: [PATCH 03/12] cache and scan options --- be/src/common/config.cpp | 13 +- be/src/common/config.h | 7 + be/src/format_v2/table/lance_reader.cpp | 28 + be/src/format_v2/table/lance_reader.h | 1 + .../processor/post/PlanPostProcessors.java | 4 +- .../post/materialize/LazyMaterializeTopN.java | 10 +- .../materialize/MaterializeProbeVisitor.java | 7 +- .../org/apache/doris/qe/SessionVariable.java | 10 + .../test_lance_runtime_filter_pushdown.groovy | 2 + .../test_lance_vector_search_two_phase.groovy | 11 +- thirdparty/download-thirdparty.sh | 6 + .../patches/lance-c-0.1.9-pr-75-pr-78.patch | 2441 +++++++++++++++++ 12 files changed, 2530 insertions(+), 10 deletions(-) create mode 100644 thirdparty/patches/lance-c-0.1.9-pr-75-pr-78.patch diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 54b4e29ccf0e14..94ac90b0d2cf82 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1212,13 +1212,24 @@ DEFINE_Validator(variant_storage_parse_mode, // Lance uses one BE-wide session so metadata/index caches and the optional Foyer data-file cache // can be shared by all Lance dataset readers. -DEFINE_Int64(lance_index_cache_size_bytes, "6442450944"); // 6GB +DEFINE_Int64(lance_index_cache_size_bytes, "10737418240"); // 10 GiB DEFINE_Int64(lance_metadata_cache_size_bytes, "1073741824"); // 1GB DEFINE_Bool(enable_lance_data_cache, "true"); DEFINE_String(lance_data_cache_path, "${DORIS_HOME}/lance_data_cache"); DEFINE_Int64(lance_data_cache_disk_capacity_bytes, "107374182400"); // 100GB DEFINE_Int64(lance_data_cache_read_block_size_bytes, "1048576"); // 1MB +// I/O buffering budget per Lance scanner, not a cap on its total memory usage. +// Runtime changes apply to newly created scanners. +DEFINE_mInt64(lance_io_buffer_size_bytes, "2147483648"); // 2 GiB +DEFINE_Validator(lance_io_buffer_size_bytes, [](int64_t value) { return value > 0; }); + +// Read-ahead limits per Lance scanner. Runtime changes apply to newly created scanners. +DEFINE_mInt32(lance_batch_readahead, "5"); +DEFINE_Validator(lance_batch_readahead, [](int32_t value) { return value > 0; }); +DEFINE_mInt32(lance_fragment_readahead, "5"); +DEFINE_Validator(lance_fragment_readahead, [](int32_t value) { return value > 0; }); + // block file cache DEFINE_Bool(enable_file_cache, "true"); // ATTENTION: For test only. Keep this enabled in production. diff --git a/be/src/common/config.h b/be/src/common/config.h index 03ae5962d66146..fca3d1c6534cf5 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1259,6 +1259,13 @@ DECLARE_String(lance_data_cache_path); DECLARE_Int64(lance_data_cache_disk_capacity_bytes); DECLARE_Int64(lance_data_cache_read_block_size_bytes); +// I/O buffering budget per Lance scanner, applied when a new scanner is created. +DECLARE_mInt64(lance_io_buffer_size_bytes); + +// Read-ahead limits per Lance scanner, applied when a new scanner is created. +DECLARE_mInt32(lance_batch_readahead); +DECLARE_mInt32(lance_fragment_readahead); + // block file cache DECLARE_Bool(enable_file_cache); DECLARE_mBool(enable_file_cache_write_from_s3_file_writer); diff --git a/be/src/format_v2/table/lance_reader.cpp b/be/src/format_v2/table/lance_reader.cpp index 2ae4aede5da172..88572bea63b0f3 100644 --- a/be/src/format_v2/table/lance_reader.cpp +++ b/be/src/format_v2/table/lance_reader.cpp @@ -30,6 +30,7 @@ #include #include +#include "common/config.h" #include "common/consts.h" #include "common/logging.h" #include "core/column/column_nullable.h" @@ -815,6 +816,7 @@ Status LanceTableReader::_open_scanner(const TFileRangeDesc& range) { if (lance_scanner_set_batch_size(scanner, static_cast(batch_size)) != 0) { return lance_error("set Lance scanner batch size"); } + RETURN_IF_ERROR(_configure_scan_options(scanner)); const auto& lance_params = range.table_format_params.lance_params; switch (_search_kind) { @@ -833,6 +835,32 @@ Status LanceTableReader::_open_scanner(const TFileRangeDesc& range) { return Status::OK(); } +Status LanceTableReader::_configure_scan_options(LanceScanner* scanner) const { + DORIS_CHECK(scanner != nullptr); + // Doris runs multiple scanners concurrently. Limit each scanner's read-ahead; + // the I/O budget does not cap its total memory usage. + const auto io_buffer_size = static_cast(config::lance_io_buffer_size_bytes); + const auto batch_readahead = static_cast(config::lance_batch_readahead); + const auto fragment_readahead = static_cast(config::lance_fragment_readahead); + constexpr bool scan_in_order = false; + + if (lance_scanner_set_io_buffer_size(scanner, io_buffer_size) != 0) { + return lance_error("set Lance scanner I/O buffer size"); + } + if (lance_scanner_set_batch_readahead(scanner, batch_readahead) != 0) { + return lance_error("set Lance scanner batch readahead"); + } + if (lance_scanner_set_fragment_readahead(scanner, fragment_readahead) != 0) { + return lance_error("set Lance scanner fragment readahead"); + } + // Storage order is not required; query ordering is enforced by Sort/TopN operators. + if (lance_scanner_set_scan_in_order(scanner, scan_in_order) != 0) { + return lance_error("set Lance scanner scan order"); + } + + return Status::OK(); +} + Status LanceTableReader::_configure_normal_scan(LanceScanner* scanner, const TLanceFileDesc& lance_params) const { DORIS_CHECK(scanner != nullptr); diff --git a/be/src/format_v2/table/lance_reader.h b/be/src/format_v2/table/lance_reader.h index 65828fc33f7c37..944041bed3d950 100644 --- a/be/src/format_v2/table/lance_reader.h +++ b/be/src/format_v2/table/lance_reader.h @@ -87,6 +87,7 @@ class LanceTableReader final : public TableReader { Status _open_dataset(const DatasetKey& key); Status _prepare_fts_query_context(); Status _open_scanner(const TFileRangeDesc& range); + Status _configure_scan_options(LanceScanner* scanner) const; Status _configure_normal_scan(LanceScanner* scanner, const TLanceFileDesc& lance_params) const; Status _configure_vector_search(LanceScanner* scanner, const TLanceFileDesc& lance_params) const; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java index bd21a6883d454f..4a94aacdc00a6a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.util.MoreFieldsThread; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; import org.apache.doris.thrift.TRuntimeFilterMode; import com.google.common.collect.ImmutableList; @@ -73,7 +74,8 @@ public List getProcessors() { 2. LazyMaterializeTopN should be applied after RecomputeLogicalPropertiesProcessor PhysicalLazyMaterialize.materializedSlots should be subsequence of topN.getOutput(). */ - if (cascadesContext.getConnectContext().getSessionVariable().enableTopnLazyMaterialization()) { + SessionVariable sessionVariable = cascadesContext.getConnectContext().getSessionVariable(); + if (sessionVariable.enableTopnLazyMaterialization() || sessionVariable.enableLanceLazyMaterialization) { builder.add(new LazyMaterializeTopN()); } builder.add(new MergeProjectPostProcessor()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java index 26da0c0dbba758..94e176f83d2fec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java @@ -77,7 +77,10 @@ public Plan visitPhysicalTopN(PhysicalTopN topN, CascadesContext ctx) { if (hasMaterialized) { return topN; } - if (SessionVariable.getTopNLazyMaterializationThreshold() < topN.getLimit()) { + SessionVariable sessionVariable = ctx.getConnectContext().getSessionVariable(); + boolean enableOtherTables = sessionVariable.topNLazyMaterializationThreshold > 0 + && topN.getLimit() <= sessionVariable.topNLazyMaterializationThreshold; + if (!sessionVariable.enableLanceLazyMaterialization && !enableOtherTables) { return topN; } /* @@ -91,7 +94,10 @@ public Plan visitPhysicalTopN(PhysicalTopN topN, CascadesContext ctx) { List materializedSlots = new ArrayList<>(); // find the slots which can be lazy materialized for (Slot slot : topN.getOutput()) { - Optional source = computeMaterializeSource(topN, (SlotReference) slot); + // Decide per source so a Lance relation does not bypass the threshold for other tables. + Optional source = computeMaterializeSource(topN, (SlotReference) slot) + .filter(candidate -> MaterializeProbeVisitor.isLanceExternalSearch(candidate.relation) + ? sessionVariable.enableLanceLazyMaterialization : enableOtherTables); if (source.isPresent()) { SlotReference baseSlot = source.get().baseSlot; if (source.get().baseSlot.hasSubColPath()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java index 177c697df4f037..effc2bc69ccc11 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java @@ -175,8 +175,11 @@ boolean checkTVFRelationTableSupportedType(PhysicalTVFRelation tvfRelation) { return false; } - private boolean isLanceExternalSearch(PhysicalTVFRelation tvfRelation) { - String functionName = tvfRelation.getFunction().getName(); + static boolean isLanceExternalSearch(Relation relation) { + if (!(relation instanceof PhysicalTVFRelation)) { + return false; + } + String functionName = ((PhysicalTVFRelation) relation).getFunction().getName(); return VectorSearchTableValuedFunction.NAME.equals(functionName) || FullTextSearchTableValuedFunction.NAME.equals(functionName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 51d0ed5e36c45b..12073b9ffbff7e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -580,6 +580,8 @@ public String toString() { public static final String LANCE_FRAGMENTS_PER_SPLIT = "lance_fragments_per_split"; + public static final String ENABLE_LANCE_LAZY_MATERIALIZATION = "enable_lance_lazy_materialization"; + public static final String FILE_SPLIT_SIZE_ON_FE = "file_split_size_on_fe"; public static final String FILE_SPLIT_SIZE_ON_BE = "file_split_size_on_be"; @@ -2585,6 +2587,14 @@ public Map getForceEagerAggHintMap() { + "and minimum BE parallelism. Does not affect vector/FTS queries."}) public int lanceFragmentsPerSplit = 0; + @VariableMgr.VarAttr(name = ENABLE_LANCE_LAZY_MATERIALIZATION, needForward = true, + description = { + "是否启用 Lance 两阶段延迟读取，默认开启，不受 topn_lazy_materialization_threshold 控制。" + + "当前支持 vector_search 和 full_text_search 中可安全延迟读取的列。", + "Enable Lance two-phase lazy materialization, independently of topn_lazy_materialization_threshold. " + + "Enabled by default for eligible columns in vector_search and full_text_search."}) + public boolean enableLanceLazyMaterialization = true; + @VariableMgr.VarAttr(name = FILE_SPLIT_SIZE_ON_FE, needForward = true, description = { "支持 BE 细粒度切分时，FE 粗粒度文件分片的目标大小，单位为字节，默认为 512MB", "Target size in bytes for FE coarse-grained file splits when BE refinement is supported. " diff --git a/regression-test/suites/external_table_p0/lance/test_lance_runtime_filter_pushdown.groovy b/regression-test/suites/external_table_p0/lance/test_lance_runtime_filter_pushdown.groovy index 870f51cd7b2b9a..b3a2ad9bc93848 100644 --- a/regression-test/suites/external_table_p0/lance/test_lance_runtime_filter_pushdown.groovy +++ b/regression-test/suites/external_table_p0/lance/test_lance_runtime_filter_pushdown.groovy @@ -118,6 +118,7 @@ suite("test_lance_runtime_filter_pushdown", "p0,external") { """ sql "SET topn_lazy_materialization_threshold = 1024" + sql "SET enable_lance_lazy_materialization = true" explain { sql "verbose ${twoPhaseQuery}" contains "VMaterializeNode" @@ -139,6 +140,7 @@ suite("test_lance_runtime_filter_pushdown", "p0,external") { """ sql "SET topn_lazy_materialization_threshold = -1" + sql "SET enable_lance_lazy_materialization = false" sql "SET disable_join_reorder = true" explain { sql "verbose ${explicitJoinQuery}" diff --git a/regression-test/suites/external_table_p0/lance/test_lance_vector_search_two_phase.groovy b/regression-test/suites/external_table_p0/lance/test_lance_vector_search_two_phase.groovy index d7dde8053a82e9..f268c49e444879 100644 --- a/regression-test/suites/external_table_p0/lance/test_lance_vector_search_two_phase.groovy +++ b/regression-test/suites/external_table_p0/lance/test_lance_vector_search_two_phase.groovy @@ -71,7 +71,9 @@ suite("test_lance_vector_search_two_phase", "p0,external") { // phase 1: Lance reads embedding for ANN, but outputs only _distance and the hidden row ID // global TopN: OFFSET 1 / LIMIT 5 // phase 2: fetch row_id, category and label by the hidden row ID - sql "SET topn_lazy_materialization_threshold = 1024" + // Lance remains enabled even when the generic TopN optimization is disabled. + sql "SET topn_lazy_materialization_threshold = -1" + sql "SET enable_lance_lazy_materialization = true" explain { sql "verbose ${resultQuery}" check { explainString -> @@ -100,9 +102,10 @@ suite("test_lance_vector_search_two_phase", "p0,external") { qt_two_phase_execution "${resultQuery}" - // Turning the threshold off removes both the Materialization node and the hidden row ID. + // The Lance switch disables materialization even when the generic threshold allows it. // The user-visible result must remain identical to the two-phase result. - sql "SET topn_lazy_materialization_threshold = -1" + sql "SET topn_lazy_materialization_threshold = 1024" + sql "SET enable_lance_lazy_materialization = false" explain { sql "verbose ${resultQuery}" notContains "VMaterializeNode" @@ -115,7 +118,7 @@ suite("test_lance_vector_search_two_phase", "p0,external") { } qt_one_phase_execution "${resultQuery}" - sql "SET topn_lazy_materialization_threshold = 1024" + sql "SET enable_lance_lazy_materialization = true" // TVF filter is a Lance prefilter. Lance reads category while selecting ANN candidates // and removes ineligible rows before nearest(), so the three nearest category='odd' rows diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index 23f3be88f18e34..b8f0374a4d4cfc 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -729,6 +729,12 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then -p1 <"${TP_PATCH_DIR}/lance-c-0.1.9-pr-74.patch" touch "${PATCHED_MARK}" fi + # Also update source trees that already have PR #73 and PR #74 applied. + if [[ ! -f "${PATCHED_MARK}_pr_75_pr_78" ]]; then + patch --batch --forward --reject-file=- --fuzz=0 --no-backup-if-mismatch -s \ + -p1 <"${TP_PATCH_DIR}/lance-c-0.1.9-pr-75-pr-78.patch" + touch "${PATCHED_MARK}_pr_75_pr_78" + fi cd - fi echo "Finished patching ${LANCE_C_SOURCE}" diff --git a/thirdparty/patches/lance-c-0.1.9-pr-75-pr-78.patch b/thirdparty/patches/lance-c-0.1.9-pr-75-pr-78.patch new file mode 100644 index 00000000000000..3e2240a1e6f46b --- /dev/null +++ b/thirdparty/patches/lance-c-0.1.9-pr-75-pr-78.patch @@ -0,0 +1,2441 @@ +Lance-C v0.1.9 scanner options: PR #75 followed by PR #78. +Apply after lance-c-0.1.9-pr-73.patch and lance-c-0.1.9-pr-74.patch. +The upstream mail patches below are concatenated without modification. + +PR #75: https://github.com/lance-format/lance-c/pull/75 +Head: 043a1f7eac253d8ac6be3f970b60fcbf615295aa +PR #78: https://github.com/lance-format/lance-c/pull/78 +Head: e894f591aef358cd36fdbf915c4d5b95fd0e8348 + +From 0752592cc4bbc69f8f9333362a00f3bceee73100 Mon Sep 17 00:00:00 2001 +From: zhangstar333 +Date: Fri, 4 Sep 2026 23:35:29 +0800 +Subject: [PATCH 1/2] add some setter + +--- + include/lance/lance.h | 81 +++++++++++ + include/lance/lance.hpp | 47 +++++++ + src/scanner.rs | 279 +++++++++++++++++++++++++++++++++++++ + tests/c_api_test.rs | 230 ++++++++++++++++++++++++++++++ + tests/cpp/test_cpp_api.cpp | 7 + + 5 files changed, 644 insertions(+) + +diff --git a/include/lance/lance.h b/include/lance/lance.h +index 3bf291f..00f415b 100644 +--- a/include/lance/lance.h ++++ b/include/lance/lance.h +@@ -934,6 +934,74 @@ LanceScanner* lance_scanner_new( + int32_t lance_scanner_set_limit(LanceScanner* scanner, int64_t limit); + int32_t lance_scanner_set_offset(LanceScanner* scanner, int64_t offset); + int32_t lance_scanner_set_batch_size(LanceScanner* scanner, int64_t batch_size); ++ ++/** ++ * Set the target output batch size in bytes. ++ * ++ * When set, this takes precedence over the row-based batch size. The value ++ * must be greater than zero and must be set before scanning starts. ++ */ ++int32_t lance_scanner_set_batch_size_bytes( ++ LanceScanner* scanner, ++ uint64_t batch_size_bytes ++); ++ ++/** ++ * Set the scanner I/O buffer size in bytes. ++ * ++ * The value must be greater than zero and must be set before scanning starts. ++ * This bounds buffered I/O received from storage, but is not a hard limit on ++ * all memory used by the scanner. ++ * ++ * @param scanner Scanner handle. Must not be NULL. ++ * @param io_buffer_size_bytes I/O buffer size in bytes. Must be greater than zero. ++ * @return 0 on success, -1 on error. ++ */ ++int32_t lance_scanner_set_io_buffer_size( ++ LanceScanner* scanner, ++ uint64_t io_buffer_size_bytes ++); ++ ++/** ++ * Set the maximum number of batches decoded concurrently. ++ * ++ * @param batch_readahead Number of in-flight batch decode tasks. Must be greater than zero. ++ */ ++int32_t lance_scanner_set_batch_readahead( ++ LanceScanner* scanner, ++ size_t batch_readahead ++); ++ ++/** ++ * Set fragment readahead for unordered scans. ++ * ++ * This setting is only used when scan-in-order is disabled. The value must be ++ * greater than zero. ++ */ ++int32_t lance_scanner_set_fragment_readahead( ++ LanceScanner* scanner, ++ size_t fragment_readahead ++); ++ ++/** ++ * Set the target number of physical execution partitions. ++ * ++ * This controls the partition count used by the physical optimizer and can be ++ * used to bound scan CPU parallelism. The value must be greater than zero and ++ * must be set before scanning starts. ++ */ ++int32_t lance_scanner_set_target_parallelism( ++ LanceScanner* scanner, ++ size_t target_parallelism ++); ++ ++/** ++ * Configure whether batches are returned in storage order (default: true). ++ * ++ * Disabling ordering can improve throughput by returning batches as soon as ++ * they are ready. ++ */ ++int32_t lance_scanner_set_scan_in_order(LanceScanner* scanner, bool scan_in_order); + int32_t lance_scanner_with_row_id(LanceScanner* scanner, bool enable); + + /** +@@ -1656,6 +1724,19 @@ int32_t lance_scanner_nearest( + ); + + int32_t lance_scanner_set_nprobes(LanceScanner* scanner, uint32_t n); ++ ++/** ++ * Set vector index partition-search concurrency for each query. ++ * ++ * A value of -1 uses the CPU pool size, 0 selects Lance's automatic policy, ++ * 1 uses the sequential path, and values greater than 1 request parallel ++ * partition search. The effective value is capped by available parallelism. ++ * Values below -1 are rejected. Must be set before scanning starts. ++ */ ++int32_t lance_scanner_set_query_parallelism( ++ LanceScanner* scanner, ++ int32_t query_parallelism ++); + int32_t lance_scanner_set_refine_factor(LanceScanner* scanner, uint32_t f); + int32_t lance_scanner_set_ef(LanceScanner* scanner, uint32_t e); + int32_t lance_scanner_set_metric(LanceScanner* scanner, LanceMetricType metric); +diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp +index 6cf245f..3c03f86 100644 +--- a/include/lance/lance.hpp ++++ b/include/lance/lance.hpp +@@ -1196,6 +1196,48 @@ class Scanner { + return *this; + } + ++ /// Set the target output batch size in bytes. ++ Scanner& batch_size_bytes(uint64_t bytes) { ++ if (lance_scanner_set_batch_size_bytes(handle_.get(), bytes) != 0) ++ check_error(); ++ return *this; ++ } ++ ++ /// Set the scanner I/O buffer size in bytes. ++ Scanner& io_buffer_size(uint64_t bytes) { ++ if (lance_scanner_set_io_buffer_size(handle_.get(), bytes) != 0) ++ check_error(); ++ return *this; ++ } ++ ++ /// Set the number of batches decoded concurrently. ++ Scanner& batch_readahead(size_t batches) { ++ if (lance_scanner_set_batch_readahead(handle_.get(), batches) != 0) ++ check_error(); ++ return *this; ++ } ++ ++ /// Set fragment readahead for unordered scans. ++ Scanner& fragment_readahead(size_t fragments) { ++ if (lance_scanner_set_fragment_readahead(handle_.get(), fragments) != 0) ++ check_error(); ++ return *this; ++ } ++ ++ /// Set the target number of physical execution partitions. ++ Scanner& target_parallelism(size_t partitions) { ++ if (lance_scanner_set_target_parallelism(handle_.get(), partitions) != 0) ++ check_error(); ++ return *this; ++ } ++ ++ /// Configure whether batches are returned in storage order. ++ Scanner& scan_in_order(bool ordered = true) { ++ if (lance_scanner_set_scan_in_order(handle_.get(), ordered) != 0) ++ check_error(); ++ return *this; ++ } ++ + /// Enable/disable row ID in output. + Scanner& with_row_id(bool enable = true) { + if (lance_scanner_with_row_id(handle_.get(), enable) != 0) +@@ -1313,6 +1355,11 @@ class Scanner { + if (lance_scanner_set_nprobes(handle_.get(), n) != 0) check_error(); + return *this; + } ++ Scanner& query_parallelism(int32_t parallelism) { ++ if (lance_scanner_set_query_parallelism(handle_.get(), parallelism) != 0) ++ check_error(); ++ return *this; ++ } + Scanner& refine_factor(uint32_t f) { + if (lance_scanner_set_refine_factor(handle_.get(), f) != 0) check_error(); + return *this; +diff --git a/src/scanner.rs b/src/scanner.rs +index 0c29b17..ebedafc 100644 +--- a/src/scanner.rs ++++ b/src/scanner.rs +@@ -60,11 +60,18 @@ pub struct LanceScanner { + limit: Option, + offset: Option, + batch_size: Option, ++ batch_size_bytes: Option, ++ io_buffer_size: Option, ++ batch_readahead: Option, ++ fragment_readahead: Option, ++ target_parallelism: Option, ++ scan_in_order: Option, + with_row_id: bool, + fragment_ids: Option>, + index_segments: Option>, + nearest: Option, + nprobes: Option, ++ query_parallelism: Option, + refine_factor: Option, + ef: Option, + metric_override: Option, +@@ -129,11 +136,18 @@ impl LanceScanner { + limit: None, + offset: None, + batch_size: None, ++ batch_size_bytes: None, ++ io_buffer_size: None, ++ batch_readahead: None, ++ fragment_readahead: None, ++ target_parallelism: None, ++ scan_in_order: None, + with_row_id: false, + fragment_ids: None, + index_segments: None, + nearest: None, + nprobes: None, ++ query_parallelism: None, + refine_factor: None, + ef: None, + metric_override: None, +@@ -164,6 +178,15 @@ impl LanceScanner { + Arc::clone(&self.poisoned) + } + ++ fn ensure_scan_not_started(&self, setting_name: &str) -> Result<()> { ++ if self.scan_started.load(Ordering::Acquire) { ++ return Err(lance_core::Error::invalid_input_source( ++ format!("{setting_name} must be set before the scan starts").into(), ++ )); ++ } ++ Ok(()) ++ } ++ + /// Apply fragment selection to a scanner builder if fragment_ids is set. + fn apply_fragment_filter(&self, scanner: &mut lance::dataset::scanner::Scanner) -> Result<()> { + if let Some(ids) = &self.fragment_ids { +@@ -231,6 +254,24 @@ impl LanceScanner { + if let Some(bs) = self.batch_size { + scanner.batch_size(bs); + } ++ if let Some(batch_size_bytes) = self.batch_size_bytes { ++ scanner.batch_size_bytes(batch_size_bytes); ++ } ++ if let Some(io_buffer_size) = self.io_buffer_size { ++ scanner.io_buffer_size(io_buffer_size); ++ } ++ if let Some(batch_readahead) = self.batch_readahead { ++ scanner.batch_readahead(batch_readahead); ++ } ++ if let Some(fragment_readahead) = self.fragment_readahead { ++ scanner.fragment_readahead(fragment_readahead); ++ } ++ if let Some(target_parallelism) = self.target_parallelism { ++ scanner.target_parallelism(target_parallelism); ++ } ++ if let Some(scan_in_order) = self.scan_in_order { ++ scanner.scan_in_order(scan_in_order); ++ } + if self.with_row_id { + scanner.with_row_id(); + } +@@ -260,6 +301,9 @@ impl LanceScanner { + if let Some(np) = self.nprobes { + scanner.nprobes(np as usize); + } ++ if let Some(query_parallelism) = self.query_parallelism { ++ scanner.query_parallelism(query_parallelism); ++ } + if let Some(rf) = self.refine_factor { + scanner.refine(rf); + } +@@ -757,6 +801,205 @@ unsafe fn scanner_set_batch_size_inner(scanner: *mut LanceScanner, batch_size: i + Ok(0) + } + ++/// Set the target output batch size in bytes. Returns 0 on success. ++/// ++/// The size must be greater than zero and must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_batch_size_bytes( ++ scanner: *mut LanceScanner, ++ batch_size_bytes: u64, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_batch_size_bytes_inner(scanner, batch_size_bytes) ++ }) ++} ++ ++unsafe fn scanner_set_batch_size_bytes_inner( ++ scanner: *mut LanceScanner, ++ batch_size_bytes: u64, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ if batch_size_bytes == 0 { ++ return Err(lance_core::Error::invalid_input_source( ++ "batch_size_bytes must be greater than 0, got 0".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("batch_size_bytes")?; ++ scanner.batch_size_bytes = Some(batch_size_bytes); ++ Ok(0) ++} ++ ++/// Set the scanner I/O buffer size in bytes. Returns 0 on success. ++/// ++/// The size must be greater than zero and must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_io_buffer_size( ++ scanner: *mut LanceScanner, ++ io_buffer_size_bytes: u64, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_io_buffer_size_inner(scanner, io_buffer_size_bytes) ++ }) ++} ++ ++unsafe fn scanner_set_io_buffer_size_inner( ++ scanner: *mut LanceScanner, ++ io_buffer_size_bytes: u64, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ if io_buffer_size_bytes == 0 { ++ return Err(lance_core::Error::invalid_input_source( ++ "io_buffer_size_bytes must be greater than 0, got 0".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("io_buffer_size_bytes")?; ++ scanner.io_buffer_size = Some(io_buffer_size_bytes); ++ Ok(0) ++} ++ ++/// Set the number of batches to decode concurrently. Returns 0 on success. ++/// ++/// The value must be greater than zero and must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_batch_readahead( ++ scanner: *mut LanceScanner, ++ batch_readahead: usize, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_batch_readahead_inner(scanner, batch_readahead) ++ }) ++} ++ ++unsafe fn scanner_set_batch_readahead_inner( ++ scanner: *mut LanceScanner, ++ batch_readahead: usize, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ if batch_readahead == 0 { ++ return Err(lance_core::Error::invalid_input_source( ++ "batch_readahead must be greater than 0, got 0".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("batch_readahead")?; ++ scanner.batch_readahead = Some(batch_readahead); ++ Ok(0) ++} ++ ++/// Set the number of fragments to read ahead for unordered scans. ++/// ++/// The value must be greater than zero and must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_fragment_readahead( ++ scanner: *mut LanceScanner, ++ fragment_readahead: usize, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_fragment_readahead_inner(scanner, fragment_readahead) ++ }) ++} ++ ++unsafe fn scanner_set_fragment_readahead_inner( ++ scanner: *mut LanceScanner, ++ fragment_readahead: usize, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ if fragment_readahead == 0 { ++ return Err(lance_core::Error::invalid_input_source( ++ "fragment_readahead must be greater than 0, got 0".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("fragment_readahead")?; ++ scanner.fragment_readahead = Some(fragment_readahead); ++ Ok(0) ++} ++ ++/// Set the target number of physical execution partitions. ++/// ++/// The value must be greater than zero and must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_target_parallelism( ++ scanner: *mut LanceScanner, ++ target_parallelism: usize, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_target_parallelism_inner(scanner, target_parallelism) ++ }) ++} ++ ++unsafe fn scanner_set_target_parallelism_inner( ++ scanner: *mut LanceScanner, ++ target_parallelism: usize, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ if target_parallelism == 0 { ++ return Err(lance_core::Error::invalid_input_source( ++ "target_parallelism must be greater than 0, got 0".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("target_parallelism")?; ++ scanner.target_parallelism = Some(target_parallelism); ++ Ok(0) ++} ++ ++/// Configure whether scan results are returned in storage order. ++/// ++/// Must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_scan_in_order( ++ scanner: *mut LanceScanner, ++ scan_in_order: bool, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_scan_in_order_inner(scanner, scan_in_order) ++ }) ++} ++ ++unsafe fn scanner_set_scan_in_order_inner( ++ scanner: *mut LanceScanner, ++ scan_in_order: bool, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("scan_in_order")?; ++ scanner.scan_in_order = Some(scan_in_order); ++ Ok(0) ++} ++ + /// Enable or disable row ID in scan output. Returns 0. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn lance_scanner_with_row_id( +@@ -1766,6 +2009,42 @@ scanner_set_u32!(lance_scanner_set_nprobes, nprobes); + scanner_set_u32!(lance_scanner_set_refine_factor, refine_factor); + scanner_set_u32!(lance_scanner_set_ef, ef); + ++/// Set vector index partition-search concurrency for each query. ++/// ++/// `-1` uses the CPU pool size, `0` selects Lance's automatic policy, and ++/// positive values request that many workers. Values below `-1` are invalid. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_query_parallelism( ++ scanner: *mut LanceScanner, ++ query_parallelism: i32, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_query_parallelism_inner(scanner, query_parallelism) ++ }) ++} ++ ++unsafe fn scanner_set_query_parallelism_inner( ++ scanner: *mut LanceScanner, ++ query_parallelism: i32, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ if query_parallelism < -1 { ++ return Err(lance_core::Error::invalid_input_source( ++ format!("query_parallelism must be -1, 0, or greater than 0, got {query_parallelism}") ++ .into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("query_parallelism")?; ++ scanner.query_parallelism = Some(query_parallelism); ++ Ok(0) ++} ++ + #[unsafe(no_mangle)] + pub unsafe extern "C" fn lance_scanner_set_metric(scanner: *mut LanceScanner, metric: i32) -> i32 { + scanner_poison_check!(scanner, -1); +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index 8805764..42f842d 100644 +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -1207,6 +1207,154 @@ fn test_scanner_batch_size() { + unsafe { lance_dataset_close(ds) }; + } + ++#[test] ++fn test_scanner_execution_tuning_options() { ++ let (_tmp, uri) = create_multi_fragment_dataset(); ++ let c_uri = c_str(&uri); ++ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; ++ assert!(!ds.is_null()); ++ ++ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; ++ assert!(!scanner.is_null()); ++ assert_eq!( ++ unsafe { lance_scanner_set_batch_size_bytes(scanner, 1024) }, ++ 0 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_io_buffer_size(scanner, 64 * 1024) }, ++ 0 ++ ); ++ assert_eq!(unsafe { lance_scanner_set_batch_readahead(scanner, 1) }, 0); ++ assert_eq!( ++ unsafe { lance_scanner_set_fragment_readahead(scanner, 1) }, ++ 0 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_target_parallelism(scanner, 1) }, ++ 0 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_scan_in_order(scanner, false) }, ++ 0 ++ ); ++ ++ let mut ffi_stream = FFI_ArrowArrayStream::empty(); ++ assert_eq!( ++ unsafe { lance_scanner_to_arrow_stream(scanner, &mut ffi_stream) }, ++ 0 ++ ); ++ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut ffi_stream) }.unwrap(); ++ let total_rows: usize = reader.map(|batch| batch.unwrap().num_rows()).sum(); ++ assert_eq!(total_rows, 10); ++ ++ unsafe { lance_scanner_close(scanner) }; ++ unsafe { lance_dataset_close(ds) }; ++} ++ ++#[test] ++fn test_scanner_execution_tuning_options_reject_zero() { ++ let (_tmp, uri) = create_test_dataset(); ++ let c_uri = c_str(&uri); ++ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; ++ assert!(!ds.is_null()); ++ ++ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; ++ assert!(!scanner.is_null()); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_batch_size_bytes(scanner, 0) }, ++ -1 ++ ); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ assert!(take_last_error_message().contains("batch_size_bytes must be greater than 0, got 0")); ++ ++ assert_eq!(unsafe { lance_scanner_set_io_buffer_size(scanner, 0) }, -1); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ assert!( ++ take_last_error_message().contains("io_buffer_size_bytes must be greater than 0, got 0") ++ ); ++ ++ assert_eq!(unsafe { lance_scanner_set_batch_readahead(scanner, 0) }, -1); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ assert!(take_last_error_message().contains("batch_readahead must be greater than 0, got 0")); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_fragment_readahead(scanner, 0) }, ++ -1 ++ ); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ assert!(take_last_error_message().contains("fragment_readahead must be greater than 0, got 0")); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_target_parallelism(scanner, 0) }, ++ -1 ++ ); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ assert!(take_last_error_message().contains("target_parallelism must be greater than 0, got 0")); ++ ++ unsafe { lance_scanner_close(scanner) }; ++ unsafe { lance_dataset_close(ds) }; ++} ++ ++#[test] ++fn test_scanner_execution_tuning_options_reject_after_scan_start() { ++ let (_tmp, uri) = create_test_dataset(); ++ let c_uri = c_str(&uri); ++ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; ++ assert!(!ds.is_null()); ++ ++ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; ++ assert!(!scanner.is_null()); ++ ++ let mut ffi_stream = FFI_ArrowArrayStream::empty(); ++ assert_eq!( ++ unsafe { lance_scanner_to_arrow_stream(scanner, &mut ffi_stream) }, ++ 0 ++ ); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_batch_size_bytes(scanner, 1024) }, ++ -1 ++ ); ++ assert!(take_last_error_message().contains("batch_size_bytes must be set before")); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_io_buffer_size(scanner, 64 * 1024) }, ++ -1 ++ ); ++ assert!(take_last_error_message().contains("io_buffer_size_bytes must be set before")); ++ ++ assert_eq!(unsafe { lance_scanner_set_batch_readahead(scanner, 1) }, -1); ++ assert!(take_last_error_message().contains("batch_readahead must be set before")); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_fragment_readahead(scanner, 1) }, ++ -1 ++ ); ++ assert!(take_last_error_message().contains("fragment_readahead must be set before")); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_target_parallelism(scanner, 1) }, ++ -1 ++ ); ++ assert!(take_last_error_message().contains("target_parallelism must be set before")); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_scan_in_order(scanner, false) }, ++ -1 ++ ); ++ assert!(take_last_error_message().contains("scan_in_order must be set before")); ++ ++ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut ffi_stream) }.unwrap(); ++ assert_eq!( ++ reader.map(|batch| batch.unwrap().num_rows()).sum::(), ++ 5 ++ ); ++ ++ unsafe { lance_scanner_close(scanner) }; ++ unsafe { lance_dataset_close(ds) }; ++} ++ + // --------------------------------------------------------------------------- + // Combined filter + projection + limit + // --------------------------------------------------------------------------- +@@ -1441,6 +1589,34 @@ fn test_null_safety_comprehensive() { + unsafe { lance_scanner_set_batch_size(ptr::null_mut(), 10) }, + -1 + ); ++ assert_eq!( ++ unsafe { lance_scanner_set_batch_size_bytes(ptr::null_mut(), 1024) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_io_buffer_size(ptr::null_mut(), 64 * 1024) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_batch_readahead(ptr::null_mut(), 1) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_fragment_readahead(ptr::null_mut(), 1) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_target_parallelism(ptr::null_mut(), 1) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_query_parallelism(ptr::null_mut(), 1) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_scan_in_order(ptr::null_mut(), true) }, ++ -1 ++ ); + assert_eq!( + unsafe { lance_scanner_with_row_id(ptr::null_mut(), true) }, + -1 +@@ -5216,6 +5392,7 @@ fn test_scanner_nearest_with_ivf_pq_index() { + 10, + ); + lance_scanner_set_nprobes(scanner, 4); ++ assert_eq!(lance_scanner_set_query_parallelism(scanner, 4), 0); + } + + let mut stream = FFI_ArrowArrayStream::empty(); +@@ -5234,6 +5411,59 @@ fn test_scanner_nearest_with_ivf_pq_index() { + unsafe { lance_dataset_close(ds) }; + } + ++#[test] ++fn test_scanner_query_parallelism_validation_and_lifecycle() { ++ let (_tmp, uri) = create_test_dataset(); ++ let uri_c = c_str(&uri); ++ let ds = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) }; ++ assert!(!ds.is_null()); ++ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; ++ assert!(!scanner.is_null()); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_query_parallelism(scanner, -1) }, ++ 0 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_query_parallelism(scanner, 0) }, ++ 0 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_query_parallelism(scanner, 2) }, ++ 0 ++ ); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_query_parallelism(scanner, -2) }, ++ -1 ++ ); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ assert!( ++ take_last_error_message() ++ .contains("query_parallelism must be -1, 0, or greater than 0, got -2") ++ ); ++ ++ let mut stream = FFI_ArrowArrayStream::empty(); ++ assert_eq!( ++ unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) }, ++ 0 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_query_parallelism(scanner, 1) }, ++ -1 ++ ); ++ assert!(take_last_error_message().contains("query_parallelism must be set before")); ++ ++ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap(); ++ assert_eq!( ++ reader.map(|batch| batch.unwrap().num_rows()).sum::(), ++ 5 ++ ); ++ ++ unsafe { lance_scanner_close(scanner) }; ++ unsafe { lance_dataset_close(ds) }; ++} ++ + #[test] + fn test_scanner_nearest_dim_mismatch() { + let (_tmp, uri) = create_vector_dataset(64, 8); +diff --git a/tests/cpp/test_cpp_api.cpp b/tests/cpp/test_cpp_api.cpp +index 17b1ab6..28762b8 100644 +--- a/tests/cpp/test_cpp_api.cpp ++++ b/tests/cpp/test_cpp_api.cpp +@@ -134,6 +134,12 @@ static void test_scanner_fluent(const std::string& uri) { + scanner.limit(5) + .offset(0) + .batch_size(2) ++ .batch_size_bytes(1024) ++ .io_buffer_size(64 * 1024) ++ .batch_readahead(1) ++ .fragment_readahead(1) ++ .target_parallelism(1) ++ .scan_in_order(false) + .statistics_callback(capture_scan_statistics, &captured); + + ArrowArrayStream stream; +@@ -370,6 +376,7 @@ static void test_nearest_smoke(const std::string& uri) { + try { + scanner.nearest("embedding", q, 8, 5) + .nprobes(2) ++ .query_parallelism(2) + .refine_factor(1) + .ef(50) + .metric(LANCE_METRIC_L2) + +From 043a1f7eac253d8ac6be3f970b60fcbf615295aa Mon Sep 17 00:00:00 2001 +From: zhangstar333 +Date: Sat, 5 Sep 2026 00:07:06 +0800 +Subject: [PATCH 2/2] add check + +--- + include/lance/lance.h | 8 ++++---- + include/lance/lance.hpp | 2 +- + src/scanner.rs | 11 ++++++++++- + tests/c_api_test.rs | 17 ++++++++++++++++- + 4 files changed, 31 insertions(+), 7 deletions(-) + +diff --git a/include/lance/lance.h b/include/lance/lance.h +index 00f415b..541134f 100644 +--- a/include/lance/lance.h ++++ b/include/lance/lance.h +@@ -949,12 +949,12 @@ int32_t lance_scanner_set_batch_size_bytes( + /** + * Set the scanner I/O buffer size in bytes. + * +- * The value must be greater than zero and must be set before scanning starts. +- * This bounds buffered I/O received from storage, but is not a hard limit on +- * all memory used by the scanner. ++ * The value must be between 1 and INT64_MAX, inclusive, and must be set before ++ * scanning starts. This bounds buffered I/O received from storage, but is not ++ * a hard limit on all memory used by the scanner. + * + * @param scanner Scanner handle. Must not be NULL. +- * @param io_buffer_size_bytes I/O buffer size in bytes. Must be greater than zero. ++ * @param io_buffer_size_bytes I/O buffer size in bytes, in the range [1, INT64_MAX]. + * @return 0 on success, -1 on error. + */ + int32_t lance_scanner_set_io_buffer_size( +diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp +index 3c03f86..a60dcd4 100644 +--- a/include/lance/lance.hpp ++++ b/include/lance/lance.hpp +@@ -1203,7 +1203,7 @@ class Scanner { + return *this; + } + +- /// Set the scanner I/O buffer size in bytes. ++ /// Set the scanner I/O buffer size in bytes, in the range [1, INT64_MAX]. + Scanner& io_buffer_size(uint64_t bytes) { + if (lance_scanner_set_io_buffer_size(handle_.get(), bytes) != 0) + check_error(); +diff --git a/src/scanner.rs b/src/scanner.rs +index ebedafc..6414cc0 100644 +--- a/src/scanner.rs ++++ b/src/scanner.rs +@@ -837,7 +837,7 @@ unsafe fn scanner_set_batch_size_bytes_inner( + + /// Set the scanner I/O buffer size in bytes. Returns 0 on success. + /// +-/// The size must be greater than zero and must be set before the scan starts. ++/// The size must be between 1 and [`i64::MAX`] and must be set before the scan starts. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn lance_scanner_set_io_buffer_size( + scanner: *mut LanceScanner, +@@ -863,6 +863,15 @@ unsafe fn scanner_set_io_buffer_size_inner( + "io_buffer_size_bytes must be greater than 0, got 0".into(), + )); + } ++ if io_buffer_size_bytes > i64::MAX as u64 { ++ return Err(lance_core::Error::invalid_input_source( ++ format!( ++ "io_buffer_size_bytes must be at most {}, got {io_buffer_size_bytes}", ++ i64::MAX ++ ) ++ .into(), ++ )); ++ } + let scanner = unsafe { &mut *scanner }; + scanner.ensure_scan_not_started("io_buffer_size_bytes")?; + scanner.io_buffer_size = Some(io_buffer_size_bytes); +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index 42f842d..ffdd915 100644 +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -1252,7 +1252,7 @@ fn test_scanner_execution_tuning_options() { + } + + #[test] +-fn test_scanner_execution_tuning_options_reject_zero() { ++fn test_scanner_execution_tuning_options_reject_invalid_values() { + let (_tmp, uri) = create_test_dataset(); + let c_uri = c_str(&uri); + let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; +@@ -1274,6 +1274,21 @@ fn test_scanner_execution_tuning_options_reject_zero() { + take_last_error_message().contains("io_buffer_size_bytes must be greater than 0, got 0") + ); + ++ assert_eq!( ++ unsafe { lance_scanner_set_io_buffer_size(scanner, i64::MAX as u64) }, ++ 0 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_io_buffer_size(scanner, u64::MAX) }, ++ -1 ++ ); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ assert!(take_last_error_message().contains(&format!( ++ "io_buffer_size_bytes must be at most {}, got {}", ++ i64::MAX, ++ u64::MAX ++ ))); ++ + assert_eq!(unsafe { lance_scanner_set_batch_readahead(scanner, 0) }, -1); + assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); + assert!(take_last_error_message().contains("batch_readahead must be greater than 0, got 0")); +From 057135cdd4ac6ac7b5348a1832caf7081a8541fb Mon Sep 17 00:00:00 2001 +From: zhangstar333 +Date: Mon, 7 Sep 2026 12:35:45 +0800 +Subject: [PATCH 1/2] feat: expose scanner use_scalar_index options + +--- + include/lance/lance.h | 83 +++++++++ + include/lance/lance.hpp | 47 ++++++ + src/scanner.rs | 335 +++++++++++++++++++++++++++++++++++++ + tests/c_api_test.rs | 296 ++++++++++++++++++++++++++++++++ + tests/cpp/test_cpp_api.cpp | 8 + + 5 files changed, 769 insertions(+) + +diff --git a/include/lance/lance.h b/include/lance/lance.h +index 31213da..6ace2cb 100644 +--- a/include/lance/lance.h ++++ b/include/lance/lance.h +@@ -147,6 +147,13 @@ typedef enum { + LANCE_METRIC_HAMMING = 3, + } LanceMetricType; + ++/** Speed / accuracy tradeoff for approximate vector search. */ ++typedef enum { ++ LANCE_APPROX_MODE_FAST = 0, ++ LANCE_APPROX_MODE_NORMAL = 1, ++ LANCE_APPROX_MODE_ACCURATE = 2, ++} LanceApproxMode; ++ + typedef enum { + LANCE_DTYPE_FLOAT32 = 0, + LANCE_DTYPE_FLOAT16 = 1, +@@ -1002,8 +1009,54 @@ int32_t lance_scanner_set_target_parallelism( + * they are ready. + */ + int32_t lance_scanner_set_scan_in_order(LanceScanner* scanner, bool scan_in_order); ++ ++/** ++ * Configure whether scalar indices may be used to optimize filters. ++ * ++ * Scalar indices are enabled by default. Disable this to force filter ++ * evaluation without scalar indices. This setting is independent of ++ * `lance_scanner_set_use_index`, which controls vector ANN index usage. ++ * Must be set before scanning starts. ++ */ ++int32_t lance_scanner_set_use_scalar_index( ++ LanceScanner* scanner, ++ bool use_scalar_index ++); ++ ++/** ++ * Configure whether row-based output batches are strict. ++ * ++ * When enabled, every batch except the last has exactly the configured row ++ * batch size. This may require copying and cannot be combined with a byte-based ++ * batch-size limit. Must be set before scanning starts. ++ */ ++int32_t lance_scanner_set_strict_batch_size( ++ LanceScanner* scanner, ++ bool strict_batch_size ++); ++ ++/** ++ * Configure whether file statistics may optimize the scan (default: true). ++ * Intended primarily for debugging and benchmarking. Must be set before ++ * scanning starts. ++ */ ++int32_t lance_scanner_set_use_stats(LanceScanner* scanner, bool use_stats); ++ + int32_t lance_scanner_with_row_id(LanceScanner* scanner, bool enable); + ++/** Include or omit the `_rowaddr` metadata column. Must be set before scanning. */ ++int32_t lance_scanner_with_row_address(LanceScanner* scanner, bool enable); ++ ++/** ++ * Configure whether deleted rows still present in storage are returned. ++ * Deleted rows have a NULL `_rowid`; callers should also enable row IDs. ++ * Must be set before scanning starts. ++ */ ++int32_t lance_scanner_set_include_deleted_rows( ++ LanceScanner* scanner, ++ bool include_deleted_rows ++); ++ + /** + * Restrict scan to the given fragment IDs. Must be called before iteration. + * @param ids Array of fragment IDs +@@ -1725,6 +1778,36 @@ int32_t lance_scanner_nearest( + + int32_t lance_scanner_set_nprobes(LanceScanner* scanner, uint32_t n); + ++/** ++ * Set the minimum number of vector-index partitions to search. ++ * Must be greater than zero and no greater than `maximum_nprobes` when set. ++ * Must be set before scanning starts. ++ */ ++int32_t lance_scanner_set_minimum_nprobes( ++ LanceScanner* scanner, ++ uint32_t minimum_nprobes ++); ++ ++/** ++ * Set the maximum number of vector-index partitions to search. ++ * Must be greater than zero and no less than `minimum_nprobes` when set. ++ * This only affects prefiltered searches that need more candidates. ++ * Must be set before scanning starts. ++ */ ++int32_t lance_scanner_set_maximum_nprobes( ++ LanceScanner* scanner, ++ uint32_t maximum_nprobes ++); ++ ++/** ++ * Configure the speed / accuracy tradeoff for approximate vector search. ++ * Must be set before scanning starts. ++ */ ++int32_t lance_scanner_set_approx_mode( ++ LanceScanner* scanner, ++ LanceApproxMode approx_mode ++); ++ + /** + * Set vector index partition-search concurrency for each query. + * +diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp +index 404d2df..e08f76a 100644 +--- a/include/lance/lance.hpp ++++ b/include/lance/lance.hpp +@@ -1269,6 +1269,27 @@ class Scanner { + return *this; + } + ++ /// Configure whether scalar indices may be used to optimize filters. ++ Scanner& use_scalar_index(bool enable = true) { ++ if (lance_scanner_set_use_scalar_index(handle_.get(), enable) != 0) ++ check_error(); ++ return *this; ++ } ++ ++ /// Configure whether row-based output batches are strict. ++ Scanner& strict_batch_size(bool strict_batch_size = true) { ++ if (lance_scanner_set_strict_batch_size(handle_.get(), strict_batch_size) != 0) ++ check_error(); ++ return *this; ++ } ++ ++ /// Configure whether file statistics may optimize the scan. ++ Scanner& use_stats(bool use_stats = true) { ++ if (lance_scanner_set_use_stats(handle_.get(), use_stats) != 0) ++ check_error(); ++ return *this; ++ } ++ + /// Enable/disable row ID in output. + Scanner& with_row_id(bool enable = true) { + if (lance_scanner_with_row_id(handle_.get(), enable) != 0) +@@ -1276,6 +1297,20 @@ class Scanner { + return *this; + } + ++ /// Include or omit the `_rowaddr` metadata column. ++ Scanner& with_row_address(bool enable = true) { ++ if (lance_scanner_with_row_address(handle_.get(), enable) != 0) ++ check_error(); ++ return *this; ++ } ++ ++ /// Configure whether deleted rows still present in storage are returned. ++ Scanner& include_deleted_rows(bool include_deleted_rows = true) { ++ if (lance_scanner_set_include_deleted_rows(handle_.get(), include_deleted_rows) != 0) ++ check_error(); ++ return *this; ++ } ++ + /// Restrict scan to specific fragment IDs. + Scanner& fragment_ids(const uint64_t* ids, size_t len) { + if (lance_scanner_set_fragment_ids(handle_.get(), ids, len) != 0) +@@ -1386,6 +1421,18 @@ class Scanner { + if (lance_scanner_set_nprobes(handle_.get(), n) != 0) check_error(); + return *this; + } ++ Scanner& minimum_nprobes(uint32_t minimum_nprobes) { ++ if (lance_scanner_set_minimum_nprobes(handle_.get(), minimum_nprobes) != 0) check_error(); ++ return *this; ++ } ++ Scanner& maximum_nprobes(uint32_t maximum_nprobes) { ++ if (lance_scanner_set_maximum_nprobes(handle_.get(), maximum_nprobes) != 0) check_error(); ++ return *this; ++ } ++ Scanner& approx_mode(LanceApproxMode approx_mode) { ++ if (lance_scanner_set_approx_mode(handle_.get(), approx_mode) != 0) check_error(); ++ return *this; ++ } + Scanner& query_parallelism(int32_t parallelism) { + if (lance_scanner_set_query_parallelism(handle_.get(), parallelism) != 0) + check_error(); +diff --git a/src/scanner.rs b/src/scanner.rs +index d3ef3be..53cd5b6 100644 +--- a/src/scanner.rs ++++ b/src/scanner.rs +@@ -21,6 +21,7 @@ use lance::dataset::scanner::{ + use lance::io::exec::fts::{FlatMatchQueryExec, MatchQueryExec, PhraseQueryExec}; + use lance_core::Result; + use lance_index::scalar::FullTextSearchQuery; ++use lance_index::vector::ApproxMode; + use lance_io::stream::RecordBatchStream; + use lance_table::format::IndexMetadata; + use uuid::Uuid; +@@ -51,6 +52,38 @@ pub enum LanceDataType { + Int8 = 4, + } + ++/// Speed / accuracy tradeoff for approximate vector search, mirroring the C ++/// enum `LanceApproxMode`. ++#[repr(i32)] ++#[derive(Clone, Copy, Debug, PartialEq, Eq)] ++pub enum LanceApproxMode { ++ Fast = 0, ++ Normal = 1, ++ Accurate = 2, ++} ++ ++impl LanceApproxMode { ++ fn from_i32(value: i32) -> Result { ++ match value { ++ 0 => Ok(Self::Fast), ++ 1 => Ok(Self::Normal), ++ 2 => Ok(Self::Accurate), ++ _ => Err(lance_core::Error::invalid_input_source( ++ format!("approx_mode must be 0 (FAST), 1 (NORMAL), or 2 (ACCURATE), got {value}") ++ .into(), ++ )), ++ } ++ } ++ ++ fn to_approx_mode(self) -> ApproxMode { ++ match self { ++ Self::Fast => ApproxMode::Fast, ++ Self::Normal => ApproxMode::Normal, ++ Self::Accurate => ApproxMode::Accurate, ++ } ++ } ++} ++ + /// Opaque scanner handle. Stores configuration until stream materialization. + pub struct LanceScanner { + dataset: Arc, +@@ -62,16 +95,24 @@ pub struct LanceScanner { + offset: Option, + batch_size: Option, + batch_size_bytes: Option, ++ strict_batch_size: Option, + io_buffer_size: Option, + batch_readahead: Option, + fragment_readahead: Option, + target_parallelism: Option, + scan_in_order: Option, ++ use_scalar_index: Option, ++ use_stats: Option, + with_row_id: bool, ++ with_row_address: bool, ++ include_deleted_rows: bool, + fragment_ids: Option>, + index_segments: Option>, + nearest: Option, + nprobes: Option, ++ minimum_nprobes: Option, ++ maximum_nprobes: Option, ++ approx_mode: Option, + query_parallelism: Option, + refine_factor: Option, + ef: Option, +@@ -138,16 +179,24 @@ impl LanceScanner { + offset: None, + batch_size: None, + batch_size_bytes: None, ++ strict_batch_size: None, + io_buffer_size: None, + batch_readahead: None, + fragment_readahead: None, + target_parallelism: None, + scan_in_order: None, ++ use_scalar_index: None, ++ use_stats: None, + with_row_id: false, ++ with_row_address: false, ++ include_deleted_rows: false, + fragment_ids: None, + index_segments: None, + nearest: None, + nprobes: None, ++ minimum_nprobes: None, ++ maximum_nprobes: None, ++ approx_mode: None, + query_parallelism: None, + refine_factor: None, + ef: None, +@@ -258,6 +307,9 @@ impl LanceScanner { + if let Some(batch_size_bytes) = self.batch_size_bytes { + scanner.batch_size_bytes(batch_size_bytes); + } ++ if let Some(strict_batch_size) = self.strict_batch_size { ++ scanner.strict_batch_size(strict_batch_size); ++ } + if let Some(io_buffer_size) = self.io_buffer_size { + scanner.io_buffer_size(io_buffer_size); + } +@@ -273,9 +325,21 @@ impl LanceScanner { + if let Some(scan_in_order) = self.scan_in_order { + scanner.scan_in_order(scan_in_order); + } ++ if let Some(use_scalar_index) = self.use_scalar_index { ++ scanner.use_scalar_index(use_scalar_index); ++ } ++ if let Some(use_stats) = self.use_stats { ++ scanner.use_stats(use_stats); ++ } + if self.with_row_id { + scanner.with_row_id(); + } ++ if self.with_row_address { ++ scanner.with_row_address(); ++ } ++ if self.include_deleted_rows { ++ scanner.include_deleted_rows(); ++ } + self.apply_fragment_filter(&mut scanner)?; + if self.index_segments.is_some() && self.nearest.is_none() { + return Err(lance_core::Error::invalid_input_source( +@@ -302,6 +366,15 @@ impl LanceScanner { + if let Some(np) = self.nprobes { + scanner.nprobes(np as usize); + } ++ if let Some(minimum_nprobes) = self.minimum_nprobes { ++ scanner.minimum_nprobes(minimum_nprobes as usize); ++ } ++ if let Some(maximum_nprobes) = self.maximum_nprobes { ++ scanner.maximum_nprobes(maximum_nprobes as usize); ++ } ++ if let Some(approx_mode) = self.approx_mode { ++ scanner.approx_mode(approx_mode.to_approx_mode()); ++ } + if let Some(query_parallelism) = self.query_parallelism { + scanner.query_parallelism(query_parallelism); + } +@@ -1035,6 +1108,92 @@ unsafe fn scanner_set_scan_in_order_inner( + Ok(0) + } + ++/// Configure whether scalar indices may be used to optimize filters. ++/// ++/// Scalar indices are enabled by default in Lance. Must be set before the scan ++/// starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_use_scalar_index( ++ scanner: *mut LanceScanner, ++ use_scalar_index: bool, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_use_scalar_index_inner(scanner, use_scalar_index) ++ }) ++} ++ ++unsafe fn scanner_set_use_scalar_index_inner( ++ scanner: *mut LanceScanner, ++ use_scalar_index: bool, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("use_scalar_index")?; ++ scanner.use_scalar_index = Some(use_scalar_index); ++ Ok(0) ++} ++ ++/// Configure whether output batches use the exact row-based batch size. ++/// ++/// Must be set before the scan starts. Lance rejects enabling this together ++/// with a byte-based batch-size limit when the scan is materialized. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_strict_batch_size( ++ scanner: *mut LanceScanner, ++ strict_batch_size: bool, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_strict_batch_size_inner(scanner, strict_batch_size) ++ }) ++} ++ ++unsafe fn scanner_set_strict_batch_size_inner( ++ scanner: *mut LanceScanner, ++ strict_batch_size: bool, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("strict_batch_size")?; ++ scanner.strict_batch_size = Some(strict_batch_size); ++ Ok(0) ++} ++ ++/// Configure whether file statistics may be used to optimize the scan. ++/// ++/// Statistics are enabled by default. Must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_use_stats( ++ scanner: *mut LanceScanner, ++ use_stats: bool, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_use_stats_inner(scanner, use_stats) ++ }) ++} ++ ++unsafe fn scanner_set_use_stats_inner(scanner: *mut LanceScanner, use_stats: bool) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("use_stats")?; ++ scanner.use_stats = Some(use_stats); ++ Ok(0) ++} ++ + /// Enable or disable row ID in scan output. Returns 0. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn lance_scanner_with_row_id( +@@ -1058,6 +1217,62 @@ unsafe fn scanner_with_row_id_inner(scanner: *mut LanceScanner, enable: bool) -> + Ok(0) + } + ++/// Enable or disable the `_rowaddr` metadata column in scan output. ++/// ++/// Must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_with_row_address( ++ scanner: *mut LanceScanner, ++ enable: bool, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_with_row_address_inner(scanner, enable) ++ }) ++} ++ ++unsafe fn scanner_with_row_address_inner(scanner: *mut LanceScanner, enable: bool) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("with_row_address")?; ++ scanner.with_row_address = enable; ++ Ok(0) ++} ++ ++/// Configure whether deleted rows still present in storage are returned. ++/// ++/// Deleted rows have a NULL `_rowid`, so callers should also enable row IDs. ++/// Must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_include_deleted_rows( ++ scanner: *mut LanceScanner, ++ include_deleted_rows: bool, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_include_deleted_rows_inner(scanner, include_deleted_rows) ++ }) ++} ++ ++unsafe fn scanner_set_include_deleted_rows_inner( ++ scanner: *mut LanceScanner, ++ include_deleted_rows: bool, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("include_deleted_rows")?; ++ scanner.include_deleted_rows = include_deleted_rows; ++ Ok(0) ++} ++ + /// Restrict the scan to the given fragment IDs. + /// Must be called before any iteration method. + /// +@@ -2044,6 +2259,126 @@ scanner_set_u32!(lance_scanner_set_nprobes, nprobes); + scanner_set_u32!(lance_scanner_set_refine_factor, refine_factor); + scanner_set_u32!(lance_scanner_set_ef, ef); + ++/// Set the minimum number of vector-index partitions to search. ++/// ++/// The value must be greater than zero, no greater than a configured ++/// `maximum_nprobes`, and must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_minimum_nprobes( ++ scanner: *mut LanceScanner, ++ minimum_nprobes: u32, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_minimum_nprobes_inner(scanner, minimum_nprobes) ++ }) ++} ++ ++unsafe fn scanner_set_minimum_nprobes_inner( ++ scanner: *mut LanceScanner, ++ minimum_nprobes: u32, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ if minimum_nprobes == 0 { ++ return Err(lance_core::Error::invalid_input_source( ++ "minimum_nprobes must be greater than 0, got 0".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("minimum_nprobes")?; ++ if let Some(maximum_nprobes) = scanner.maximum_nprobes ++ && minimum_nprobes > maximum_nprobes ++ { ++ return Err(lance_core::Error::invalid_input_source( ++ format!( ++ "minimum_nprobes ({minimum_nprobes}) must not exceed maximum_nprobes ({maximum_nprobes})" ++ ) ++ .into(), ++ )); ++ } ++ scanner.minimum_nprobes = Some(minimum_nprobes); ++ Ok(0) ++} ++ ++/// Set the maximum number of vector-index partitions to search. ++/// ++/// The value must be greater than zero, no less than a configured ++/// `minimum_nprobes`, and must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_maximum_nprobes( ++ scanner: *mut LanceScanner, ++ maximum_nprobes: u32, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_maximum_nprobes_inner(scanner, maximum_nprobes) ++ }) ++} ++ ++unsafe fn scanner_set_maximum_nprobes_inner( ++ scanner: *mut LanceScanner, ++ maximum_nprobes: u32, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ if maximum_nprobes == 0 { ++ return Err(lance_core::Error::invalid_input_source( ++ "maximum_nprobes must be greater than 0, got 0".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("maximum_nprobes")?; ++ if let Some(minimum_nprobes) = scanner.minimum_nprobes ++ && maximum_nprobes < minimum_nprobes ++ { ++ return Err(lance_core::Error::invalid_input_source( ++ format!( ++ "maximum_nprobes ({maximum_nprobes}) must not be less than minimum_nprobes ({minimum_nprobes})" ++ ) ++ .into(), ++ )); ++ } ++ scanner.maximum_nprobes = Some(maximum_nprobes); ++ Ok(0) ++} ++ ++/// Configure the speed / accuracy tradeoff for approximate vector search. ++/// ++/// Must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_approx_mode( ++ scanner: *mut LanceScanner, ++ approx_mode: i32, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_approx_mode_inner(scanner, approx_mode) ++ }) ++} ++ ++unsafe fn scanner_set_approx_mode_inner( ++ scanner: *mut LanceScanner, ++ approx_mode: i32, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ let approx_mode = LanceApproxMode::from_i32(approx_mode)?; ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("approx_mode")?; ++ scanner.approx_mode = Some(approx_mode); ++ Ok(0) ++} ++ + /// Set vector index partition-search concurrency for each query. + /// + /// `-1` uses the CPU pool size, `0` selects Lance's automatic policy, and +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index bde742d..a550a93 100644 +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -1237,6 +1237,12 @@ fn test_scanner_execution_tuning_options() { + unsafe { lance_scanner_set_scan_in_order(scanner, false) }, + 0 + ); ++ assert_eq!( ++ unsafe { lance_scanner_set_use_scalar_index(scanner, false) }, ++ 0 ++ ); ++ assert_eq!(unsafe { lance_scanner_set_use_stats(scanner, false) }, 0); ++ assert_eq!(unsafe { lance_scanner_with_row_address(scanner, true) }, 0); + + let mut ffi_stream = FFI_ArrowArrayStream::empty(); + assert_eq!( +@@ -1244,6 +1250,7 @@ fn test_scanner_execution_tuning_options() { + 0 + ); + let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut ffi_stream) }.unwrap(); ++ assert!(reader.schema().field_with_name("_rowaddr").is_ok()); + let total_rows: usize = reader.map(|batch| batch.unwrap().num_rows()).sum(); + assert_eq!(total_rows, 10); + +@@ -1360,6 +1367,30 @@ fn test_scanner_execution_tuning_options_reject_after_scan_start() { + ); + assert!(take_last_error_message().contains("scan_in_order must be set before")); + ++ assert_eq!( ++ unsafe { lance_scanner_set_use_scalar_index(scanner, false) }, ++ -1 ++ ); ++ assert!(take_last_error_message().contains("use_scalar_index must be set before")); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_strict_batch_size(scanner, true) }, ++ -1 ++ ); ++ assert!(take_last_error_message().contains("strict_batch_size must be set before")); ++ ++ assert_eq!(unsafe { lance_scanner_set_use_stats(scanner, false) }, -1); ++ assert!(take_last_error_message().contains("use_stats must be set before")); ++ ++ assert_eq!(unsafe { lance_scanner_with_row_address(scanner, true) }, -1); ++ assert!(take_last_error_message().contains("with_row_address must be set before")); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_include_deleted_rows(scanner, true) }, ++ -1 ++ ); ++ assert!(take_last_error_message().contains("include_deleted_rows must be set before")); ++ + let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut ffi_stream) }.unwrap(); + assert_eq!( + reader.map(|batch| batch.unwrap().num_rows()).sum::(), +@@ -1370,6 +1401,79 @@ fn test_scanner_execution_tuning_options_reject_after_scan_start() { + unsafe { lance_dataset_close(ds) }; + } + ++#[test] ++fn test_scanner_strict_batch_size_across_fragments() { ++ let (_tmp, uri) = create_multi_fragment_dataset(); ++ let c_uri = c_str(&uri); ++ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; ++ assert!(!ds.is_null()); ++ ++ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; ++ assert!(!scanner.is_null()); ++ assert_eq!(unsafe { lance_scanner_set_batch_size(scanner, 3) }, 0); ++ assert_eq!( ++ unsafe { lance_scanner_set_strict_batch_size(scanner, true) }, ++ 0 ++ ); ++ ++ let mut stream = FFI_ArrowArrayStream::empty(); ++ assert_eq!( ++ unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) }, ++ 0 ++ ); ++ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap(); ++ let batch_sizes = reader ++ .map(|batch| batch.unwrap().num_rows()) ++ .collect::>(); ++ assert_eq!(batch_sizes, vec![3, 3, 3, 1]); ++ ++ unsafe { lance_scanner_close(scanner) }; ++ unsafe { lance_dataset_close(ds) }; ++} ++ ++#[test] ++fn test_scanner_include_deleted_rows() { ++ let (_tmp, uri) = create_multi_fragment_dataset(); ++ let c_uri = c_str(&uri); ++ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; ++ assert!(!ds.is_null()); ++ ++ let predicate = c_str("id >= 8"); ++ let mut num_deleted = 0; ++ assert_eq!( ++ unsafe { lance_dataset_delete(ds, predicate.as_ptr(), &mut num_deleted) }, ++ 0 ++ ); ++ assert_eq!(num_deleted, 2); ++ ++ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; ++ assert!(!scanner.is_null()); ++ assert_eq!(unsafe { lance_scanner_with_row_id(scanner, true) }, 0); ++ assert_eq!( ++ unsafe { lance_scanner_set_include_deleted_rows(scanner, true) }, ++ 0 ++ ); ++ ++ let mut stream = FFI_ArrowArrayStream::empty(); ++ assert_eq!( ++ unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) }, ++ 0 ++ ); ++ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap(); ++ let batches = reader.map(|batch| batch.unwrap()).collect::>(); ++ assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 10); ++ assert_eq!( ++ batches ++ .iter() ++ .map(|batch| batch.column_by_name("_rowid").unwrap().null_count()) ++ .sum::(), ++ 2 ++ ); ++ ++ unsafe { lance_scanner_close(scanner) }; ++ unsafe { lance_dataset_close(ds) }; ++} ++ + // --------------------------------------------------------------------------- + // Combined filter + projection + limit + // --------------------------------------------------------------------------- +@@ -1632,10 +1736,42 @@ fn test_null_safety_comprehensive() { + unsafe { lance_scanner_set_scan_in_order(ptr::null_mut(), true) }, + -1 + ); ++ assert_eq!( ++ unsafe { lance_scanner_set_use_scalar_index(ptr::null_mut(), false) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_strict_batch_size(ptr::null_mut(), true) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_use_stats(ptr::null_mut(), false) }, ++ -1 ++ ); + assert_eq!( + unsafe { lance_scanner_with_row_id(ptr::null_mut(), true) }, + -1 + ); ++ assert_eq!( ++ unsafe { lance_scanner_with_row_address(ptr::null_mut(), true) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_include_deleted_rows(ptr::null_mut(), true) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_minimum_nprobes(ptr::null_mut(), 1) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_maximum_nprobes(ptr::null_mut(), 1) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_approx_mode(ptr::null_mut(), LanceApproxMode::Normal as i32,) }, ++ -1 ++ ); + + // Scanner iteration with NULL. + let mut ffi_stream2 = FFI_ArrowArrayStream::empty(); +@@ -3077,6 +3213,90 @@ fn test_create_scalar_index_btree() { + unsafe { lance_dataset_close(ds) }; + } + ++#[test] ++fn test_scanner_set_use_scalar_index_controls_filter_planning() { ++ let (_tmp, uri) = create_test_dataset(); ++ let uri_c = c_str(&uri); ++ let ds = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) }; ++ assert!(!ds.is_null()); ++ ++ let column = c_str("id"); ++ assert_eq!( ++ unsafe { ++ lance_dataset_create_scalar_index( ++ ds, ++ column.as_ptr(), ++ ptr::null(), ++ LanceScalarIndexType::BTree as i32, ++ ptr::null(), ++ false, ++ ) ++ }, ++ 0 ++ ); ++ ++ let run_scan = |use_scalar_index: bool| { ++ let filter = c_str("id = 3"); ++ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), filter.as_ptr()) }; ++ assert!(!scanner.is_null()); ++ assert_eq!( ++ unsafe { lance_scanner_set_use_scalar_index(scanner, use_scalar_index) }, ++ 0 ++ ); ++ ++ let mut captured = CapturedScanStatistics::default(); ++ assert_eq!( ++ unsafe { ++ lance_scanner_set_statistics_callback( ++ scanner, ++ Some(capture_scan_statistics), ++ (&mut captured as *mut CapturedScanStatistics).cast(), ++ ) ++ }, ++ 0 ++ ); ++ ++ let mut stream = FFI_ArrowArrayStream::empty(); ++ assert_eq!( ++ unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) }, ++ 0 ++ ); ++ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap(); ++ let ids = reader ++ .flat_map(|batch| { ++ let batch = batch.unwrap(); ++ batch ++ .column_by_name("id") ++ .unwrap() ++ .as_any() ++ .downcast_ref::() ++ .unwrap() ++ .values() ++ .to_vec() ++ }) ++ .collect::>(); ++ ++ assert_eq!(captured.calls, 1); ++ unsafe { lance_scanner_close(scanner) }; ++ (ids, captured) ++ }; ++ ++ let (indexed_ids, indexed_statistics) = run_scan(true); ++ let (unindexed_ids, unindexed_statistics) = run_scan(false); ++ assert_eq!(indexed_ids, vec![3]); ++ assert_eq!(unindexed_ids, indexed_ids); ++ assert!( ++ indexed_statistics.indices_loaded > 0, ++ "enabled scan should load the scalar index" ++ ); ++ assert_eq!( ++ unindexed_statistics.indices_loaded, 0, ++ "disabled scan should bypass the scalar index" ++ ); ++ ++ unsafe { lance_dataset_close(ds) }; ++} ++ + #[test] + fn test_scalar_index_segment_build_is_fragment_scoped_and_uncommitted() { + let (_tmp, uri) = create_many_small_fragments(2); +@@ -5409,6 +5629,12 @@ fn test_scanner_nearest_with_ivf_pq_index() { + 10, + ); + lance_scanner_set_nprobes(scanner, 4); ++ assert_eq!(lance_scanner_set_minimum_nprobes(scanner, 2), 0); ++ assert_eq!(lance_scanner_set_maximum_nprobes(scanner, 6), 0); ++ assert_eq!( ++ lance_scanner_set_approx_mode(scanner, LanceApproxMode::Accurate as i32), ++ 0 ++ ); + assert_eq!(lance_scanner_set_query_parallelism(scanner, 4), 0); + } + +@@ -5428,6 +5654,76 @@ fn test_scanner_nearest_with_ivf_pq_index() { + unsafe { lance_dataset_close(ds) }; + } + ++#[test] ++fn test_scanner_adaptive_nprobes_and_approx_mode_validation_and_lifecycle() { ++ let (_tmp, uri) = create_test_dataset(); ++ let uri_c = c_str(&uri); ++ let ds = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) }; ++ assert!(!ds.is_null()); ++ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; ++ assert!(!scanner.is_null()); ++ ++ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 0) }, -1); ++ assert!(take_last_error_message().contains("minimum_nprobes must be greater than 0, got 0")); ++ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 0) }, -1); ++ assert!(take_last_error_message().contains("maximum_nprobes must be greater than 0, got 0")); ++ assert_eq!(unsafe { lance_scanner_set_approx_mode(scanner, 3) }, -1); ++ assert!( ++ take_last_error_message() ++ .contains("approx_mode must be 0 (FAST), 1 (NORMAL), or 2 (ACCURATE), got 3") ++ ); ++ ++ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 2) }, 0); ++ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 3) }, -1); ++ assert!( ++ take_last_error_message() ++ .contains("minimum_nprobes (3) must not exceed maximum_nprobes (2)") ++ ); ++ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 1) }, 0); ++ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 1) }, 0); ++ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 2) }, -1); ++ assert!( ++ take_last_error_message() ++ .contains("minimum_nprobes (2) must not exceed maximum_nprobes (1)") ++ ); ++ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 2) }, 0); ++ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 2) }, 0); ++ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 1) }, -1); ++ assert!( ++ take_last_error_message() ++ .contains("maximum_nprobes (1) must not be less than minimum_nprobes (2)") ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_approx_mode(scanner, LanceApproxMode::Fast as i32) }, ++ 0 ++ ); ++ ++ let mut stream = FFI_ArrowArrayStream::empty(); ++ assert_eq!( ++ unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) }, ++ 0 ++ ); ++ ++ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 1) }, -1); ++ assert!(take_last_error_message().contains("minimum_nprobes must be set before")); ++ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 1) }, -1); ++ assert!(take_last_error_message().contains("maximum_nprobes must be set before")); ++ assert_eq!( ++ unsafe { lance_scanner_set_approx_mode(scanner, LanceApproxMode::Normal as i32) }, ++ -1 ++ ); ++ assert!(take_last_error_message().contains("approx_mode must be set before")); ++ ++ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap(); ++ assert_eq!( ++ reader.map(|batch| batch.unwrap().num_rows()).sum::(), ++ 5 ++ ); ++ ++ unsafe { lance_scanner_close(scanner) }; ++ unsafe { lance_dataset_close(ds) }; ++} ++ + #[test] + fn test_scanner_query_parallelism_validation_and_lifecycle() { + let (_tmp, uri) = create_test_dataset(); +diff --git a/tests/cpp/test_cpp_api.cpp b/tests/cpp/test_cpp_api.cpp +index 28762b8..60b8fc4 100644 +--- a/tests/cpp/test_cpp_api.cpp ++++ b/tests/cpp/test_cpp_api.cpp +@@ -140,6 +140,11 @@ static void test_scanner_fluent(const std::string& uri) { + .fragment_readahead(1) + .target_parallelism(1) + .scan_in_order(false) ++ .use_scalar_index(false) ++ .strict_batch_size(false) ++ .use_stats(false) ++ .with_row_address(true) ++ .include_deleted_rows(false) + .statistics_callback(capture_scan_statistics, &captured); + + ArrowArrayStream stream; +@@ -376,6 +381,9 @@ static void test_nearest_smoke(const std::string& uri) { + try { + scanner.nearest("embedding", q, 8, 5) + .nprobes(2) ++ .minimum_nprobes(1) ++ .maximum_nprobes(2) ++ .approx_mode(LANCE_APPROX_MODE_NORMAL) + .query_parallelism(2) + .refine_factor(1) + .ef(50) + +From e894f591aef358cd36fdbf915c4d5b95fd0e8348 Mon Sep 17 00:00:00 2001 +From: zhangstar333 +Date: Mon, 7 Sep 2026 14:27:10 +0800 +Subject: [PATCH 2/2] update + +--- + include/lance/lance.h | 18 +++- + include/lance/lance.hpp | 7 +- + src/scanner.rs | 210 +++++++++++++++++++++++++++++++--------- + tests/c_api_test.rs | 77 +++++++++++++++ + 4 files changed, 261 insertions(+), 51 deletions(-) + +diff --git a/include/lance/lance.h b/include/lance/lance.h +index 6ace2cb..8173ae5 100644 +--- a/include/lance/lance.h ++++ b/include/lance/lance.h +@@ -946,7 +946,8 @@ int32_t lance_scanner_set_batch_size(LanceScanner* scanner, int64_t batch_size); + * Set the target output batch size in bytes. + * + * When set, this takes precedence over the row-based batch size. The value +- * must be greater than zero and must be set before scanning starts. ++ * must be greater than zero and must be set before scanning starts. The call ++ * is rejected without changing scanner state if strict batch sizing is enabled. + */ + int32_t lance_scanner_set_batch_size_bytes( + LanceScanner* scanner, +@@ -1028,7 +1029,8 @@ int32_t lance_scanner_set_use_scalar_index( + * + * When enabled, every batch except the last has exactly the configured row + * batch size. This may require copying and cannot be combined with a byte-based +- * batch-size limit. Must be set before scanning starts. ++ * batch-size limit. The call is rejected without changing scanner state if a ++ * byte limit is already set. Must be set before scanning starts. + */ + int32_t lance_scanner_set_strict_batch_size( + LanceScanner* scanner, +@@ -1776,11 +1778,19 @@ int32_t lance_scanner_nearest( + uint32_t k + ); + +-int32_t lance_scanner_set_nprobes(LanceScanner* scanner, uint32_t n); ++/** ++ * Set both the minimum and maximum vector-index partition-search bounds. ++ * ++ * This replaces both bounds configured by earlier calls to any nprobes ++ * setter. The value must be greater than zero. Must be set before scanning. ++ */ ++int32_t lance_scanner_set_nprobes(LanceScanner* scanner, uint32_t nprobes); + + /** + * Set the minimum number of vector-index partitions to search. ++ * This replaces only the minimum bound; the current maximum is preserved. + * Must be greater than zero and no greater than `maximum_nprobes` when set. ++ * An invalid resulting range is rejected without changing either bound. + * Must be set before scanning starts. + */ + int32_t lance_scanner_set_minimum_nprobes( +@@ -1790,8 +1800,10 @@ int32_t lance_scanner_set_minimum_nprobes( + + /** + * Set the maximum number of vector-index partitions to search. ++ * This replaces only the maximum bound; the current minimum is preserved. + * Must be greater than zero and no less than `minimum_nprobes` when set. + * This only affects prefiltered searches that need more candidates. ++ * An invalid resulting range is rejected without changing either bound. + * Must be set before scanning starts. + */ + int32_t lance_scanner_set_maximum_nprobes( +diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp +index e08f76a..c12c0c6 100644 +--- a/include/lance/lance.hpp ++++ b/include/lance/lance.hpp +@@ -1417,14 +1417,17 @@ class Scanner { + return *this; + } + +- Scanner& nprobes(uint32_t n) { +- if (lance_scanner_set_nprobes(handle_.get(), n) != 0) check_error(); ++ /// Replace both minimum and maximum partition-search bounds. ++ Scanner& nprobes(uint32_t nprobes) { ++ if (lance_scanner_set_nprobes(handle_.get(), nprobes) != 0) check_error(); + return *this; + } ++ /// Replace only the minimum partition-search bound. + Scanner& minimum_nprobes(uint32_t minimum_nprobes) { + if (lance_scanner_set_minimum_nprobes(handle_.get(), minimum_nprobes) != 0) check_error(); + return *this; + } ++ /// Replace only the maximum partition-search bound. + Scanner& maximum_nprobes(uint32_t maximum_nprobes) { + if (lance_scanner_set_maximum_nprobes(handle_.get(), maximum_nprobes) != 0) check_error(); + return *this; +diff --git a/src/scanner.rs b/src/scanner.rs +index 53cd5b6..4ceeb0e 100644 +--- a/src/scanner.rs ++++ b/src/scanner.rs +@@ -109,9 +109,7 @@ pub struct LanceScanner { + fragment_ids: Option>, + index_segments: Option>, + nearest: Option, +- nprobes: Option, +- minimum_nprobes: Option, +- maximum_nprobes: Option, ++ nprobes: NprobesRange, + approx_mode: Option, + query_parallelism: Option, + refine_factor: Option, +@@ -148,6 +146,72 @@ struct NearestQuery { + k: u32, + } + ++/// The effective adaptive partition-search range shared by all three nprobes ++/// setters. Updates are computed and validated before replacing this state. ++#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] ++struct NprobesRange { ++ minimum: Option, ++ maximum: Option, ++} ++ ++impl NprobesRange { ++ fn exact(nprobes: u32) -> Result { ++ if nprobes == 0 { ++ return Err(lance_core::Error::invalid_input_source( ++ "nprobes must be greater than 0, got 0".into(), ++ )); ++ } ++ Ok(Self { ++ minimum: Some(nprobes), ++ maximum: Some(nprobes), ++ }) ++ } ++ ++ fn with_minimum(self, minimum_nprobes: u32) -> Result { ++ if minimum_nprobes == 0 { ++ return Err(lance_core::Error::invalid_input_source( ++ "minimum_nprobes must be greater than 0, got 0".into(), ++ )); ++ } ++ if let Some(maximum_nprobes) = self.maximum ++ && minimum_nprobes > maximum_nprobes ++ { ++ return Err(lance_core::Error::invalid_input_source( ++ format!( ++ "minimum_nprobes ({minimum_nprobes}) must not exceed maximum_nprobes ({maximum_nprobes})" ++ ) ++ .into(), ++ )); ++ } ++ Ok(Self { ++ minimum: Some(minimum_nprobes), ++ ..self ++ }) ++ } ++ ++ fn with_maximum(self, maximum_nprobes: u32) -> Result { ++ if maximum_nprobes == 0 { ++ return Err(lance_core::Error::invalid_input_source( ++ "maximum_nprobes must be greater than 0, got 0".into(), ++ )); ++ } ++ if let Some(minimum_nprobes) = self.minimum ++ && maximum_nprobes < minimum_nprobes ++ { ++ return Err(lance_core::Error::invalid_input_source( ++ format!( ++ "maximum_nprobes ({maximum_nprobes}) must not be less than minimum_nprobes ({minimum_nprobes})" ++ ) ++ .into(), ++ )); ++ } ++ Ok(Self { ++ maximum: Some(maximum_nprobes), ++ ..self ++ }) ++ } ++} ++ + /// Poll status for `lance_scanner_poll_next`. + #[repr(C)] + #[derive(Debug, PartialEq, Eq)] +@@ -193,9 +257,7 @@ impl LanceScanner { + fragment_ids: None, + index_segments: None, + nearest: None, +- nprobes: None, +- minimum_nprobes: None, +- maximum_nprobes: None, ++ nprobes: NprobesRange::default(), + approx_mode: None, + query_parallelism: None, + refine_factor: None, +@@ -363,13 +425,10 @@ impl LanceScanner { + } + if let Some(n) = &self.nearest { + scanner.nearest(&n.column, n.query.as_ref(), n.k as usize)?; +- if let Some(np) = self.nprobes { +- scanner.nprobes(np as usize); +- } +- if let Some(minimum_nprobes) = self.minimum_nprobes { ++ if let Some(minimum_nprobes) = self.nprobes.minimum { + scanner.minimum_nprobes(minimum_nprobes as usize); + } +- if let Some(maximum_nprobes) = self.maximum_nprobes { ++ if let Some(maximum_nprobes) = self.nprobes.maximum { + scanner.maximum_nprobes(maximum_nprobes as usize); + } + if let Some(approx_mode) = self.approx_mode { +@@ -930,6 +989,14 @@ unsafe fn scanner_set_batch_size_bytes_inner( + } + let scanner = unsafe { &mut *scanner }; + scanner.ensure_scan_not_started("batch_size_bytes")?; ++ if scanner.strict_batch_size == Some(true) { ++ return Err(lance_core::Error::invalid_input_source( ++ format!( ++ "strict_batch_size=true cannot be combined with batch_size_bytes={batch_size_bytes}" ++ ) ++ .into(), ++ )); ++ } + scanner.batch_size_bytes = Some(batch_size_bytes); + Ok(0) + } +@@ -1140,8 +1207,8 @@ unsafe fn scanner_set_use_scalar_index_inner( + + /// Configure whether output batches use the exact row-based batch size. + /// +-/// Must be set before the scan starts. Lance rejects enabling this together +-/// with a byte-based batch-size limit when the scan is materialized. ++/// Must be set before the scan starts. Enabling this together with a ++/// byte-based batch-size limit is rejected without changing scanner state. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn lance_scanner_set_strict_batch_size( + scanner: *mut LanceScanner, +@@ -1164,6 +1231,14 @@ unsafe fn scanner_set_strict_batch_size_inner( + } + let scanner = unsafe { &mut *scanner }; + scanner.ensure_scan_not_started("strict_batch_size")?; ++ if strict_batch_size && let Some(batch_size_bytes) = scanner.batch_size_bytes { ++ return Err(lance_core::Error::invalid_input_source( ++ format!( ++ "strict_batch_size=true cannot be combined with batch_size_bytes={batch_size_bytes}" ++ ) ++ .into(), ++ )); ++ } + scanner.strict_batch_size = Some(strict_batch_size); + Ok(0) + } +@@ -2255,10 +2330,38 @@ macro_rules! scanner_set_u32 { + }; + } + +-scanner_set_u32!(lance_scanner_set_nprobes, nprobes); + scanner_set_u32!(lance_scanner_set_refine_factor, refine_factor); + scanner_set_u32!(lance_scanner_set_ef, ef); + ++/// Set both vector-index partition-search bounds to the same value. ++/// ++/// This replaces any values previously configured through ++/// `minimum_nprobes` or `maximum_nprobes`. The value must be greater than zero ++/// and must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_nprobes( ++ scanner: *mut LanceScanner, ++ nprobes: u32, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_nprobes_inner(scanner, nprobes) ++ }) ++} ++ ++unsafe fn scanner_set_nprobes_inner(scanner: *mut LanceScanner, nprobes: u32) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("nprobes")?; ++ let next = NprobesRange::exact(nprobes)?; ++ scanner.nprobes = next; ++ Ok(0) ++} ++ + /// Set the minimum number of vector-index partitions to search. + /// + /// The value must be greater than zero, no greater than a configured +@@ -2283,24 +2386,10 @@ unsafe fn scanner_set_minimum_nprobes_inner( + "scanner is NULL".into(), + )); + } +- if minimum_nprobes == 0 { +- return Err(lance_core::Error::invalid_input_source( +- "minimum_nprobes must be greater than 0, got 0".into(), +- )); +- } + let scanner = unsafe { &mut *scanner }; + scanner.ensure_scan_not_started("minimum_nprobes")?; +- if let Some(maximum_nprobes) = scanner.maximum_nprobes +- && minimum_nprobes > maximum_nprobes +- { +- return Err(lance_core::Error::invalid_input_source( +- format!( +- "minimum_nprobes ({minimum_nprobes}) must not exceed maximum_nprobes ({maximum_nprobes})" +- ) +- .into(), +- )); +- } +- scanner.minimum_nprobes = Some(minimum_nprobes); ++ let next = scanner.nprobes.with_minimum(minimum_nprobes)?; ++ scanner.nprobes = next; + Ok(0) + } + +@@ -2328,24 +2417,10 @@ unsafe fn scanner_set_maximum_nprobes_inner( + "scanner is NULL".into(), + )); + } +- if maximum_nprobes == 0 { +- return Err(lance_core::Error::invalid_input_source( +- "maximum_nprobes must be greater than 0, got 0".into(), +- )); +- } + let scanner = unsafe { &mut *scanner }; + scanner.ensure_scan_not_started("maximum_nprobes")?; +- if let Some(minimum_nprobes) = scanner.minimum_nprobes +- && maximum_nprobes < minimum_nprobes +- { +- return Err(lance_core::Error::invalid_input_source( +- format!( +- "maximum_nprobes ({maximum_nprobes}) must not be less than minimum_nprobes ({minimum_nprobes})" +- ) +- .into(), +- )); +- } +- scanner.maximum_nprobes = Some(maximum_nprobes); ++ let next = scanner.nprobes.with_maximum(maximum_nprobes)?; ++ scanner.nprobes = next; + Ok(0) + } + +@@ -2885,6 +2960,49 @@ mod tests { + ) + } + ++ #[test] ++ fn nprobes_setters_share_one_validated_range() { ++ let (_tmp, uri) = create_test_dataset(); ++ let (dataset, scanner) = open_dataset_and_scanner(&uri); ++ let assert_range = |minimum, maximum| { ++ assert_eq!( ++ unsafe { &*scanner }.nprobes, ++ NprobesRange { minimum, maximum } ++ ); ++ }; ++ ++ assert_range(None, None); ++ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 2) }, 0); ++ assert_range(Some(2), None); ++ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 5) }, 0); ++ assert_range(Some(2), Some(5)); ++ ++ // The combined setter replaces both bounds. ++ assert_eq!(unsafe { lance_scanner_set_nprobes(scanner, 4) }, 0); ++ assert_range(Some(4), Some(4)); ++ ++ // A failed partial update leaves both bounds unchanged. ++ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 5) }, -1); ++ assert_range(Some(4), Some(4)); ++ ++ // Widening the maximum first makes the new minimum valid. ++ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 6) }, 0); ++ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 5) }, 0); ++ assert_range(Some(5), Some(6)); ++ ++ // A later combined call deterministically replaces the widened range. ++ assert_eq!(unsafe { lance_scanner_set_nprobes(scanner, 3) }, 0); ++ assert_range(Some(3), Some(3)); ++ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 2) }, -1); ++ assert_eq!(unsafe { lance_scanner_set_nprobes(scanner, 0) }, -1); ++ assert_range(Some(3), Some(3)); ++ ++ unsafe { ++ lance_scanner_close(scanner); ++ lance_dataset_close(dataset); ++ } ++ } ++ + #[test] + fn prepared_fts_index_only_plan_does_not_scan_indexed_fragment_row_ids() { + let (_tmp, uri) = create_test_dataset(); +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index a550a93..3b3424b 100644 +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -1318,6 +1318,78 @@ fn test_scanner_execution_tuning_options_reject_invalid_values() { + unsafe { lance_dataset_close(ds) }; + } + ++#[test] ++fn test_scanner_strict_batch_size_and_bytes_conflict_is_recoverable() { ++ let (_tmp, uri) = create_test_dataset(); ++ let c_uri = c_str(&uri); ++ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; ++ assert!(!ds.is_null()); ++ ++ let consume = |scanner| { ++ let mut stream = FFI_ArrowArrayStream::empty(); ++ assert_eq!( ++ unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) }, ++ 0 ++ ); ++ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap(); ++ assert_eq!( ++ reader.map(|batch| batch.unwrap().num_rows()).sum::(), ++ 5 ++ ); ++ }; ++ ++ // A byte limit already exists: strict=true is rejected without starting ++ // the scan or replacing the prior strict setting. ++ let bytes_first = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; ++ assert_eq!( ++ unsafe { lance_scanner_set_batch_size_bytes(bytes_first, 1024) }, ++ 0 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_strict_batch_size(bytes_first, true) }, ++ -1 ++ ); ++ assert!( ++ take_last_error_message() ++ .contains("strict_batch_size=true cannot be combined with batch_size_bytes=1024") ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_use_stats(bytes_first, false) }, ++ 0, ++ "the rejected setter must not mark the scan as started" ++ ); ++ consume(bytes_first); ++ ++ // Strict sizing already exists: the byte limit is rejected without ++ // mutation. The caller can disable strict sizing and retry on this handle. ++ let strict_first = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; ++ assert_eq!( ++ unsafe { lance_scanner_set_strict_batch_size(strict_first, true) }, ++ 0 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_batch_size_bytes(strict_first, 1024) }, ++ -1 ++ ); ++ assert!( ++ take_last_error_message() ++ .contains("strict_batch_size=true cannot be combined with batch_size_bytes=1024") ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_strict_batch_size(strict_first, false) }, ++ 0 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_batch_size_bytes(strict_first, 1024) }, ++ 0 ++ ); ++ consume(strict_first); ++ ++ unsafe { lance_scanner_close(bytes_first) }; ++ unsafe { lance_scanner_close(strict_first) }; ++ unsafe { lance_dataset_close(ds) }; ++} ++ + #[test] + fn test_scanner_execution_tuning_options_reject_after_scan_start() { + let (_tmp, uri) = create_test_dataset(); +@@ -1760,6 +1832,7 @@ fn test_null_safety_comprehensive() { + unsafe { lance_scanner_set_include_deleted_rows(ptr::null_mut(), true) }, + -1 + ); ++ assert_eq!(unsafe { lance_scanner_set_nprobes(ptr::null_mut(), 1) }, -1); + assert_eq!( + unsafe { lance_scanner_set_minimum_nprobes(ptr::null_mut(), 1) }, + -1 +@@ -5663,6 +5736,8 @@ fn test_scanner_adaptive_nprobes_and_approx_mode_validation_and_lifecycle() { + let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; + assert!(!scanner.is_null()); + ++ assert_eq!(unsafe { lance_scanner_set_nprobes(scanner, 0) }, -1); ++ assert!(take_last_error_message().contains("nprobes must be greater than 0, got 0")); + assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 0) }, -1); + assert!(take_last_error_message().contains("minimum_nprobes must be greater than 0, got 0")); + assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 0) }, -1); +@@ -5704,6 +5779,8 @@ fn test_scanner_adaptive_nprobes_and_approx_mode_validation_and_lifecycle() { + 0 + ); + ++ assert_eq!(unsafe { lance_scanner_set_nprobes(scanner, 1) }, -1); ++ assert!(take_last_error_message().contains("nprobes must be set before")); + assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 1) }, -1); + assert!(take_last_error_message().contains("minimum_nprobes must be set before")); + assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 1) }, -1); From ad46763de9c4f3577b2dcdf33e8117fec1683da2 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Wed, 9 Sep 2026 15:50:18 +0800 Subject: [PATCH 04/12] enhance scalar index filter --- be/src/format_v2/table/lance_reader.cpp | 42 +- be/test/format_v2/table/lance_reader_test.cpp | 125 ++ .../datasource/lance/LanceFragmentInfo.java | 4 +- .../lance/source/IndexSegmentSplitPlan.java | 8 +- .../lance/source/LancePredicateConverter.java | 2 +- .../lance/source/LanceScalarIndexPlanner.java | 35 +- .../lance/source/LanceScanNode.java | 17 +- .../org/apache/doris/qe/SessionVariable.java | 4 +- .../datasource/LanceThriftContractTest.java | 24 + .../lance/source/LanceScanNodeTest.java | 182 ++ gensrc/thrift/PlanNodes.thrift | 11 +- thirdparty/download-thirdparty.sh | 25 +- thirdparty/patches/lance-c-0.1.9-pr-77.patch | 1866 +++++++++++++++++ thirdparty/patches/lance-c-0.1.9-pr-79.patch | 1782 ++++++++++++++++ 14 files changed, 4080 insertions(+), 47 deletions(-) create mode 100644 thirdparty/patches/lance-c-0.1.9-pr-77.patch create mode 100644 thirdparty/patches/lance-c-0.1.9-pr-79.patch diff --git a/be/src/format_v2/table/lance_reader.cpp b/be/src/format_v2/table/lance_reader.cpp index 88572bea63b0f3..fb1bb182674276 100644 --- a/be/src/format_v2/table/lance_reader.cpp +++ b/be/src/format_v2/table/lance_reader.cpp @@ -173,6 +173,18 @@ Status LanceTableReader::init(TableReadOptions&& options) { {"deltas_searched", ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LanceVectorIndexSegmentsSearched", TUnit::UNIT, LANCE_READER_PROFILE, 1)}, + {"scalar_segments_requested", + ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LanceScalarIndexSegmentsRequested", + TUnit::UNIT, LANCE_READER_PROFILE, 1)}, + {"scalar_segments_searched", + ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LanceScalarIndexSegmentsSearched", + TUnit::UNIT, LANCE_READER_PROFILE, 1)}, + {"scalar_segment_fallbacks", + ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LanceScalarIndexSegmentFallbacks", + TUnit::UNIT, LANCE_READER_PROFILE, 1)}, + {"scalar_segment_candidate_rows", + ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LanceScalarIndexCandidateRows", + TUnit::UNIT, LANCE_READER_PROFILE, 1)}, }; _lance_time_metrics = { // This is wait time reported by the same Lance scan execution node described above, @@ -182,6 +194,12 @@ Status LanceTableReader::init(TableReadOptions&& options) { {"find_partitions_elapsed", ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LanceIVFPartitionRankingTime", LANCE_READER_PROFILE, 1)}, + {"scalar_segment_prepare_time", + ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LanceScalarIndexSegmentPrepareTime", + LANCE_READER_PROFILE, 1)}, + {"scalar_segment_search_time", + ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LanceScalarIndexSegmentSearchTime", + LANCE_READER_PROFILE, 1)}, }; if (_search_kind != SearchKind::NORMAL) { RETURN_IF_ERROR(_validate_external_search_request()); @@ -870,8 +888,28 @@ Status LanceTableReader::_configure_normal_scan(LanceScanner* scanner, lance_scanner_set_fragment_ids(scanner, fragment_ids.data(), fragment_ids.size()) != 0) { return lance_error("set Lance scanner fragment ids"); } - if (lance_params.__isset.index_segment_uuids && !lance_params.index_segment_uuids.empty()) { - return Status::InvalidArgument("normal Lance scan cannot contain index segment UUIDs"); + std::vector segment_uuids; + size_t segment_count = 0; + RETURN_IF_ERROR(parse_index_segment_uuids(lance_params, &segment_uuids, &segment_count)); + if (segment_count > 1) { + return Status::InvalidArgument("normal Lance scan accepts only one scalar index segment"); + } + if (segment_count == 1) { + if (fragment_ids.empty() || !lance_params.__isset.version || lance_params.version <= 0) { + return Status::InvalidArgument( + "Lance scalar index segment requires a fixed version and nonempty fragment " + "ids"); + } + if (lance_params.__isset.use_scalar_index && !lance_params.use_scalar_index) { + return Status::InvalidArgument( + "Lance scalar index segment cannot be combined with use_scalar_index=false"); + } + if (lance_scanner_set_scalar_index_segment(scanner, segment_uuids.data()) != 0) { + return lance_error("set Lance scanner scalar index segment"); + } + } else if (lance_params.__isset.use_scalar_index && + lance_scanner_set_use_scalar_index(scanner, lance_params.use_scalar_index) != 0) { + return lance_error("set Lance scanner scalar index usage"); } // FE sets this only when every predicate has been pushed into Lance. if (lance_params.__isset.limit && lance_params.limit > 0 && diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp index 763ffdb9db6215..9fac579b92d919 100644 --- a/be/test/format_v2/table/lance_reader_test.cpp +++ b/be/test/format_v2/table/lance_reader_test.cpp @@ -1223,6 +1223,131 @@ TEST(LanceTableReaderFilterTest, CombinesStaticSubstraitFilterWithRuntimeFilter) EXPECT_TRUE(combined_reader.close().ok()); } +TEST(LanceTableReaderScalarSegmentTest, FiltersIndexedAndUncoveredDomainsWithoutLosingRows) { + const auto unique_suffix = std::chrono::steady_clock::now().time_since_epoch().count(); + for (const auto index_type : {LANCE_SCALAR_BTREE, LANCE_SCALAR_BITMAP}) { + const auto dataset_uri = std::filesystem::temp_directory_path() / + ("doris_lance_scalar_segment_" + std::to_string(unique_suffix) + + "_" + std::to_string(index_type) + ".lance"); + Defer cleanup {[&] { + std::error_code error; + std::filesystem::remove_all(dataset_uri, error); + }}; + const auto schema = arrow::schema({arrow::field("row_id", arrow::int64(), false)}); + for (int64_t first_id : {1, 3}) { + arrow::Int64Builder values; + ASSERT_TRUE(values.AppendValues({first_id, first_id + 1}).ok()); + auto array = values.Finish(); + ASSERT_TRUE(array.ok()); + auto batch = arrow::RecordBatch::Make(schema, 2, {std::move(array).ValueUnsafe()}); + auto batches = arrow::RecordBatchReader::Make({batch}, schema); + ASSERT_TRUE(batches.ok()); + ArrowArrayStream stream {}; + ASSERT_TRUE( + arrow::ExportRecordBatchReader(std::move(batches).ValueUnsafe(), &stream).ok()); + auto dataset = ::lance::Dataset::write( + dataset_uri.string(), &stream, + first_id == 1 ? ::lance::WriteMode::Create : ::lance::WriteMode::Append); + if (first_id == 1) { + // The second append remains uncovered by this index segment. + dataset.create_scalar_index("row_id", index_type, "row_id_idx"); + } + } + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + ASSERT_EQ(2, fixture.fragment_ids.size()); + std::vector segments; + ASSERT_TRUE(get_index_segment_uuids(dataset_uri, "row_id_idx", &segments).ok()); + ASSERT_EQ(1, segments.size()); + + const auto read_domain = [&](const std::vector& fragments, bool use_segment, + const std::vector& expected, int64_t searched, + int64_t fallbacks) { + TQueryGlobals globals; + RuntimeState state(globals); + RuntimeProfile profile("lance_scalar_segment"); + TFileScanRangeParams scan_params; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + LanceTableReader reader; + auto filter = create_int64_runtime_in_conjunct("row_id", {2, 4}, 41); + ASSERT_TRUE( + init_reader(&reader, columns, &state, &profile, &scan_params, {filter}).ok()); + auto range = make_lance_range(dataset_uri, fixture.version, fragments); + if (use_segment) { + range.table_format_params.lance_params.__set_index_segment_uuids(segments); + } else { + range.table_format_params.lance_params.__set_use_scalar_index(false); + } + ASSERT_TRUE(prepare_range(&reader, std::move(range)).ok()); + Block block; + add_output_columns(&block, columns); + std::vector actual; + bool eos = false; + while (!eos) { + auto status = reader.get_block(&block, &eos); + ASSERT_TRUE(status.ok()) << status.to_string(); + if (!eos) { + const auto& ids = + assert_cast(*block.get_by_position(0).column); + actual.insert(actual.end(), ids.get_data().begin(), ids.get_data().end()); + } + } + std::ranges::sort(actual); + EXPECT_EQ(expected, actual); + EXPECT_EQ(use_segment ? 1 : 0, + profile.get_counter("LanceScalarIndexSegmentsRequested")->value()); + EXPECT_EQ(searched, profile.get_counter("LanceScalarIndexSegmentsSearched")->value()); + EXPECT_EQ(fallbacks, profile.get_counter("LanceScalarIndexSegmentFallbacks")->value()); + if (!use_segment) { + EXPECT_EQ(0, profile.get_counter("LanceIndexComparisons")->value()); + } + EXPECT_TRUE(reader.close().ok()); + }; + read_domain({fixture.fragment_ids[0]}, true, {2}, 1, 0); + read_domain({fixture.fragment_ids[1]}, false, {4}, 0, 0); + // A segment which cannot cover the whole task must filter the entire explicit domain. + read_domain(fixture.fragment_ids, true, {2, 4}, 0, 1); + } +} + +TEST(LanceTableReaderScalarSegmentTest, RejectsInvalidSegmentAssignments) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + const auto check_invalid = [&](TLanceFileDesc params, const std::string& message) { + TQueryGlobals globals; + RuntimeState state(globals); + RuntimeProfile profile("lance_invalid_scalar_segment"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, {projected_column("row_id", TYPE_BIGINT, false)}, &state, + &profile, &scan_params) + .ok()); + auto range = make_lance_range(dataset_uri, fixture.version, fixture.fragment_ids); + range.table_format_params.__set_lance_params(std::move(params)); + auto status = prepare_range(&reader, std::move(range)); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find(message), std::string::npos) << status.to_string(); + EXPECT_TRUE(reader.close().ok()); + }; + auto params = make_lance_range(dataset_uri, fixture.version, fixture.fragment_ids) + .table_format_params.lance_params; + params.__set_index_segment_uuids({"too-short"}); + check_invalid(params, "16 bytes"); + params.__set_index_segment_uuids({std::string(16, 'a'), std::string(16, 'b')}); + check_invalid(params, "only one scalar index segment"); + params.__set_index_segment_uuids({std::string(16, 'a')}); + params.__set_fragment_ids({}); + check_invalid(params, "nonempty fragment ids"); + params.__set_fragment_ids(fixture.fragment_ids); + params.__set_version(0); + check_invalid(params, "fixed version"); + params.__set_version(fixture.version); + params.__set_use_scalar_index(false); + check_invalid(params, "use_scalar_index=false"); +} + TEST(LanceTableReaderFilterTest, PushesRuntimeInFilterIntoLanceScanner) { const std::filesystem::path dataset_uri = "./be/test/format_v2/table/lance/data/all_types.lance"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceFragmentInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceFragmentInfo.java index 213bf4bf12c72e..90b80098497f09 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceFragmentInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceFragmentInfo.java @@ -47,8 +47,8 @@ public long getRowCount() { /** * Returns the number of physical rows stored before deletions. * - *

The BE legacy reader reads and merges physical batches before applying the deletion - * vector, so split scheduling uses this value rather than {@link #getRowCount()}. + *

Reading a fragment can process rows later discarded by deletion vectors, so split + * scheduling uses this value rather than {@link #getRowCount()}. */ public long getPhysicalRows() { return physicalRows; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/IndexSegmentSplitPlan.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/IndexSegmentSplitPlan.java index 7faa276d1579ed..9444ae73eab674 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/IndexSegmentSplitPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/IndexSegmentSplitPlan.java @@ -64,12 +64,6 @@ void addIndexSegmentSplit(UUID indexSegmentUuid, datasetUri, version, indexSegmentUuid, fragmentIds, physicalRows)); } - // Ordinary scans use segment coverage only to group fragments. No UUID is sent to Lance. - void addIndexSegmentFragmentGroup(List fragmentIds, long physicalRows) { - indexSegmentFragmentIds.addAll(fragmentIds); - addFragmentGroup(fragmentIds, physicalRows); - } - // Manifest order and row-based split weights are shared by ordinary scans and fallbacks. // An empty index coverage set groups all fragments; vector fallbacks use a group size of 1. void addUncoveredFragments(Iterable fragments, int fragmentsPerSplit) { @@ -99,7 +93,7 @@ private void addFragmentGroup(List fragmentIds, long physicalRows) { splits.add(LanceSplit.forFragments(datasetUri, version, fragmentIds, physicalRows)); } - /** Subdivides ordinary fragment groups to provide work for the available backends. */ + /** Subdivides fragment-only groups for available backends; assigned segments stay intact. */ List buildFragmentSplits(int numBackends, Map visibleFragments) { int targetSplits = Math.min(numBackends, visibleFragments.size()); if (splits.size() >= targetSplits) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LancePredicateConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LancePredicateConverter.java index 41fc1d855f86c8..bd290b15c2ac93 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LancePredicateConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LancePredicateConverter.java @@ -607,7 +607,7 @@ private byte[] serialize(Expression expression) { schemaBuilder.addNames(field.getName()); structBuilder.addTypes(type.get()); } else { - // Lance 4.x removes user-defined top-level fields before handing the + // Lance removes user-defined top-level fields before handing the // ExtendedExpression to DataFusion and remaps field ordinals. This keeps the // envelope aligned with the full dataset schema when unrelated complex or // otherwise unsupported columns are present. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlanner.java index 1b1e4295577d02..dc7c9355130a77 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlanner.java @@ -17,14 +17,16 @@ package org.apache.doris.datasource.lance.source; +import org.apache.doris.analysis.CompoundPredicate; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.SlotRef; import org.apache.doris.datasource.lance.LanceFragmentInfo; import org.apache.doris.datasource.lance.LanceIndexSegmentInfo; import org.apache.doris.datasource.lance.LanceTableMetadata; +import org.lance.index.IndexType; + import java.util.ArrayList; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -32,7 +34,7 @@ import java.util.TreeMap; import java.util.stream.Collectors; -/** Groups fragments using scalar-index coverage. Index search planning remains entirely in Lance. */ +/** Assigns one BTree/Bitmap/LabelList segment and a disjoint fragment domain to each ordinary scan task. */ final class LanceScalarIndexPlanner { static final class Plan { final String indexName; @@ -62,11 +64,12 @@ static Plan plan(LanceTableMetadata metadata, List pushedConjuncts, TreeMap::new, Collectors.toList())); Plan selected = null; for (List segments : indices.values()) { - // The loader copies logical-index field metadata into every segment. An index - // is a candidate when any of its fields occurs in a pushed filter. - // Non-vector indices include INVERTED (FTS); this selects grouping, not search semantics. + // PR #79 supports one top-level key in BTree/Bitmap/LabelList indices. Lance + // performs the final typed driver selection and falls back within the same domain. LanceIndexSegmentInfo index = segments.get(0); - if (index.isVectorIndex() || Collections.disjoint(index.getFieldIds(), filterFields)) { + if ((index.getIndexType() != IndexType.BTREE && index.getIndexType() != IndexType.BITMAP + && index.getIndexType() != IndexType.LABEL_LIST) + || index.getFieldIds().size() != 1 || !filterFields.contains(index.getFieldIds().get(0))) { continue; } Plan candidate = groupFragments(metadata, segments, visibleFragments); @@ -79,7 +82,7 @@ static Plan plan(LanceTableMetadata metadata, List pushedConjuncts, private static Set collectFilterFields(LanceTableMetadata metadata, List pushedConjuncts) { Set slots = new HashSet<>(); - pushedConjuncts.forEach(expr -> expr.collect(SlotRef.class, slots)); + pushedConjuncts.forEach(expr -> collectDriverSlots(expr, slots)); Set fields = new HashSet<>(); for (SlotRef slot : slots) { metadata.getLanceFieldId(slot.getColumnName()).ifPresent(fields::add); @@ -87,6 +90,18 @@ private static Set collectFilterFields(LanceTableMetadata metadata, Lis return fields; } + private static void collectDriverSlots(Expr expr, Set slots) { + // A predicate below OR or NOT is not a necessary condition of the whole filter. + // Do not select its index and then force every task into a non-indexed fallback. + if (expr instanceof CompoundPredicate) { + if (((CompoundPredicate) expr).getOp() == CompoundPredicate.Operator.AND) { + expr.getChildren().forEach(child -> collectDriverSlots(child, slots)); + } + } else { + expr.collect(SlotRef.class, slots); + } + } + private static Plan groupFragments(LanceTableMetadata metadata, List segments, Map visibleFragments) { IndexSegmentSplitPlan splits = new IndexSegmentSplitPlan( @@ -94,7 +109,9 @@ private static Plan groupFragments(LanceTableMetadata metadata, List coveredFragments = new HashSet<>(); long coveredRows = 0; for (LanceIndexSegmentInfo segment : segments) { - if (!segment.getFragmentIds().isPresent()) { + if (!segment.getFragmentIds().isPresent() + || segment.getIndexType() != segments.get(0).getIndexType() + || !segment.getFieldIds().equals(segments.get(0).getFieldIds())) { return null; } List fragments = new ArrayList<>(); @@ -114,7 +131,7 @@ private static Plan groupFragments(LanceTableMetadata metadata, ListThese modes share dataset metadata, storage properties, and BE scan-range serialization. * Keeping them in one node prevents those common parts from drifting apart. The search request is - * also an explicit mode marker. Ordinary scans group fragments using index coverage or a fixed - * fragment count. Indexed vector searches are - * split by physical index segment, with uncovered fragments retained as flat-search fallbacks. + * also an explicit mode marker. Ordinary scans assign one BTree/Bitmap/LabelList segment per split when + * a pushed filter and known, disjoint coverage allow it; uncovered fragments use non-indexed scans. + * Other ordinary scans use fragment splits. Indexed vector searches are split by physical index + * segment, with uncovered fragments retained as flat-search fallbacks. * Full-text searches are split only by committed inverted-index segments, with coverage governed * by the request's STRICT or INDEX_ONLY mode. Each search split produces local candidates; a Doris * TopN above this scan merges them into the requested snapshot-wide result. @@ -581,9 +582,9 @@ protected void setScanParams(TFileRangeDesc rangeDesc, Split split) { lanceParams.setDatasetUri(lanceSplit.getDatasetUri()); lanceParams.setVersion(lanceSplit.getVersion()); if (lanceSplit.hasFragmentIds()) { - if (searchKind == SearchKind.NORMAL && lanceSplit.hasIndexSegmentUuids()) { + if (searchKind == SearchKind.NORMAL && lanceSplit.getIndexSegmentUuids().size() > 1) { throw new IllegalArgumentException( - "Ordinary Lance scan split must not contain index segment UUIDs"); + "Ordinary Lance scan split can contain only one scalar index segment"); } if (searchKind == SearchKind.FULL_TEXT && !lanceSplit.hasIndexSegmentUuids()) { throw new IllegalArgumentException( @@ -606,6 +607,11 @@ protected void setScanParams(TFileRangeDesc rangeDesc, Split split) { // BE serves the row count from table_level_row_count below, leaving fragment_ids unset. throw new IllegalArgumentException("Lance scan split must contain fragments"); } + if (searchKind == SearchKind.NORMAL && scalarIndexPlan != null && !lanceSplit.hasIndexSegmentUuids()) { + // Uncovered fragments belong to separate tasks. Do not repeat global index + // evaluation on these tasks; the complete filter still applies to their rows. + lanceParams.setUseScalarIndex(false); + } // Push LIMIT into each ordinary split scanner only when it is safe to truncate that // split early. External searches use their own per-split candidate bound. if (searchKind == SearchKind.NORMAL && canPushDownLimit()) { @@ -707,6 +713,7 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { result.append(prefix).append("lanceFragmentGrouping=FRAGMENT\n"); } else { result.append(prefix).append("lanceFragmentGrouping=INDEX_SEGMENT\n"); + result.append(prefix).append("lanceScalarIndexScan=SEGMENT\n"); result.append(prefix).append("lanceGroupingIndex=").append(scalarIndexPlan.indexName).append("\n"); result.append(prefix).append("lanceGroupingIndexSegments=").append(plannedIndexSegments).append("\n"); result.append(prefix).append("lanceGroupingIndexedFragments=").append(plannedIndexFragments).append("\n"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 12073b9ffbff7e..b06f2cc08b2138 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -2581,9 +2581,9 @@ public Map getForceEagerAggHintMap() { flag = VariableMgr.INVISIBLE, fuzzy = false, checker = "checkLanceFragmentsPerSplit", description = { "普通 Lance 扫描的调试参数。默认 0 自动划分；正数强制按指定 fragment 数分组，" - + "跳过索引分组和补足 BE 数量的逻辑。不影响 vector/FTS 查询。", + + "跳过标量索引 segment 扫描和补足 BE 数量的逻辑。不影响 vector/FTS 查询。", "Debug override for ordinary Lance scans. Default 0 uses automatic splitting; " - + "a positive value groups that many fragments per split, bypassing index grouping " + + "a positive value groups that many fragments per split, bypassing scalar segment scans " + "and minimum BE parallelism. Does not affect vector/FTS queries."}) public int lanceFragmentsPerSplit = 0; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/LanceThriftContractTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/LanceThriftContractTest.java index 331872d1552a9a..0e41f718998fa5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/LanceThriftContractTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/LanceThriftContractTest.java @@ -29,12 +29,36 @@ import org.junit.Assert; import org.junit.Test; +import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Map; public class LanceThriftContractTest { + @Test + public void testScalarIndexTaskCompactProtocolRoundTrip() throws Exception { + for (boolean indexed : new boolean[] {true, false}) { + TLanceFileDesc source = new TLanceFileDesc() + .setDatasetUri("s3://warehouse/db/table.lance") + .setVersion(42L).setFragmentIds(Arrays.asList(7L, 11L)); + if (indexed) { + ByteBuffer segmentUuid = ByteBuffer.allocate(16).putLong(1).putLong(2); + segmentUuid.flip(); + source.setIndexSegmentUuids(Collections.singletonList(segmentUuid)); + } else { + source.setUseScalarIndex(false); + } + TLanceFileDesc restored = new TLanceFileDesc(); + new TDeserializer(new TCompactProtocol.Factory()).deserialize(restored, + new TSerializer(new TCompactProtocol.Factory()).serialize(source)); + Assert.assertEquals(source, restored); + Assert.assertEquals(!indexed, restored.isSetUseScalarIndex()); + Assert.assertEquals(indexed, restored.isSetIndexSegmentUuids()); + } + } + @Test public void testLanceDescriptorCompactProtocolRoundTrip() throws Exception { TLanceFileDesc lanceDesc = new TLanceFileDesc() diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java index c144af39f5f683..681482cad5c4c4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java @@ -17,6 +17,12 @@ package org.apache.doris.datasource.lance.source; +import org.apache.doris.analysis.BinaryPredicate; +import org.apache.doris.analysis.CompoundPredicate; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.IntLiteral; +import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.common.UserException; @@ -35,6 +41,7 @@ import org.apache.doris.thrift.TFtsMatchOperator; import org.apache.doris.thrift.TFtsQueryType; import org.apache.doris.thrift.TFullTextSearchParams; +import org.apache.doris.thrift.TLanceFileDesc; import org.apache.doris.thrift.TPushAggOp; import org.apache.doris.thrift.TVectorMetric; import org.apache.doris.thrift.TVectorSearchOptions; @@ -42,6 +49,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; import org.junit.Assert; import org.junit.Test; @@ -52,7 +60,9 @@ import java.security.InvalidParameterException; import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.UUID; public class LanceScanNodeTest { @@ -76,6 +86,178 @@ public void testAdditionalTypesRejectSmoothUpgradeSourceBackend() throws Excepti Assert.assertTrue(exception.getMessage().contains("10001")); } + @Test + public void testLabelListSegmentsForAlreadyPushedArrayFilter() { + UUID first = UUID.fromString("11111111-2222-3333-4444-555555555555"); + UUID second = UUID.fromString("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"); + LanceTableMetadata metadata = LanceTableMetadata.withIndexSegments( + "s3://bucket/labels.lance", 42, + new Schema(Collections.singletonList(new Field("labels", + FieldType.nullable(ArrowType.List.INSTANCE), + Collections.singletonList(Field.nullable("item", new ArrowType.Int(64, true)))))), + Arrays.asList(new LanceFragmentInfo(1, 10, 10), new LanceFragmentInfo(2, 20, 20), + new LanceFragmentInfo(3, 30, 30)), + Collections.singletonMap("labels", 9), + Arrays.asList(scalarSegment(first, IndexType.LABEL_LIST, Collections.singletonList(1L)), + scalarSegment(second, IndexType.LABEL_LIST, Collections.singletonList(2L))), + Collections.emptyMap()); + Map fragments = new LinkedHashMap<>(); + metadata.getFragments().forEach(fragment -> fragments.put(fragment.getId(), fragment)); + // This checks the planner's pushed-predicate contract. Array predicate conversion + // remains a separate prerequisite for ordinary SQL to select a LabelList segment. + Expr filter = new FunctionCallExpr("array_contains", + Arrays.asList(new SlotRef(null, "labels"), new IntLiteral(42))); + LanceScalarIndexPlanner.Plan plan = + LanceScalarIndexPlanner.plan(metadata, Collections.singletonList(filter), fragments); + Assert.assertNotNull(plan); + Assert.assertEquals("key_idx", plan.indexName); + plan.splits.addUncoveredFragments(fragments.values(), 1); + List splits = plan.splits.buildFragmentSplits(20, fragments); + Assert.assertEquals(3, splits.size()); + Assert.assertEquals(Collections.singletonList(first), ((LanceSplit) splits.get(0)).getIndexSegmentUuids()); + Assert.assertEquals(Collections.singletonList(second), ((LanceSplit) splits.get(1)).getIndexSegmentUuids()); + Assert.assertEquals(Collections.singletonList(3L), ((LanceSplit) splits.get(2)).getFragmentIds()); + Assert.assertFalse(((LanceSplit) splits.get(2)).hasIndexSegmentUuids()); + } + + @Test + public void testScalarSegmentsKeepSingleOwnerAndSerializeFallback() throws Exception { + for (IndexType type : Arrays.asList(IndexType.BTREE, IndexType.BITMAP)) { + LanceScanNode node = newNode(); + UUID first = UUID.fromString("11111111-2222-3333-4444-555555555555"); + UUID second = UUID.fromString("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"); + setMetadata(node, scalarMetadata(Arrays.asList( + scalarSegment(first, type, Arrays.asList(1L, 2L)), + scalarSegment(second, type, Collections.singletonList(3L))))); + setPushedConjuncts(node, scalarPredicate()); + node.setLimit(5); + + // More BEs than segments must not cause repeated segment searches. + List splits = node.getSplits(20); + Assert.assertEquals(3, splits.size()); + Assert.assertEquals(Arrays.asList(1L, 2L), ((LanceSplit) splits.get(0)).getFragmentIds()); + Assert.assertEquals(Collections.singletonList(3L), ((LanceSplit) splits.get(1)).getFragmentIds()); + Assert.assertEquals(Collections.singletonList(4L), ((LanceSplit) splits.get(2)).getFragmentIds()); + for (int i = 0; i < splits.size(); i++) { + TFileRangeDesc range = new TFileRangeDesc(); + node.setScanParams(range, splits.get(i)); + TLanceFileDesc params = range.getTableFormatParams().getLanceParams(); + Assert.assertEquals(42L, params.getVersion()); + Assert.assertEquals(5L, params.getLimit()); + if (i < 2) { + ByteBuffer uuid = params.getIndexSegmentUuids().get(0).duplicate(); + Assert.assertEquals(i == 0 ? first : second, new UUID(uuid.getLong(), uuid.getLong())); + Assert.assertFalse(params.isSetUseScalarIndex()); + } else { + Assert.assertFalse(params.isSetIndexSegmentUuids()); + Assert.assertTrue(params.isSetUseScalarIndex()); + Assert.assertFalse(params.isUseScalarIndex()); + } + } + } + } + + @Test + public void testScalarSegmentPlanRejectsUnsafeCoverageAndUnsupportedTypes() throws Exception { + UUID first = UUID.randomUUID(); + UUID second = UUID.randomUUID(); + List> candidates = Arrays.asList( + Collections.singletonList(scalarSegment(first, IndexType.BTREE, null)), + Arrays.asList(scalarSegment(first, IndexType.BTREE, Arrays.asList(1L, 2L)), + scalarSegment(second, IndexType.BTREE, Arrays.asList(2L, 3L))), + Collections.singletonList(scalarSegment(first, IndexType.INVERTED, Arrays.asList(1L, 2L))), + Collections.singletonList(scalarSegment(first, IndexType.VECTOR, Arrays.asList(1L, 2L)))); + for (List segments : candidates) { + LanceScanNode node = newNode(); + setMetadata(node, scalarMetadata(segments)); + setPushedConjuncts(node, scalarPredicate()); + List splits = node.getSplits(20); + Assert.assertEquals(4, splits.size()); + for (Split split : splits) { + TFileRangeDesc range = new TFileRangeDesc(); + node.setScanParams(range, split); + Assert.assertFalse(range.getTableFormatParams().getLanceParams().isSetIndexSegmentUuids()); + Assert.assertFalse(range.getTableFormatParams().getLanceParams().isSetUseScalarIndex()); + } + } + } + + @Test + public void testScalarSegmentSelectionDoesNotDescendThroughOrOrNot() throws Exception { + Expr predicate = scalarPredicate(); + for (Expr filter : Arrays.asList( + new CompoundPredicate(CompoundPredicate.Operator.OR, predicate, predicate), + new CompoundPredicate(CompoundPredicate.Operator.NOT, predicate, null))) { + LanceScanNode node = newNode(); + setMetadata(node, scalarMetadata(Collections.singletonList( + scalarSegment(UUID.randomUUID(), IndexType.BTREE, Arrays.asList(1L, 2L, 3L, 4L))))); + setPushedConjuncts(node, filter); + Assert.assertEquals(4, node.getSplits(20).size()); + setPushedConjuncts(node, new CompoundPredicate(CompoundPredicate.Operator.AND, filter, predicate)); + List splits = node.getSplits(20); + Assert.assertEquals(1, splits.size()); + Assert.assertTrue(((LanceSplit) splits.get(0)).hasIndexSegmentUuids()); + } + } + + @Test + public void testDebugFragmentGroupingBypassesScalarSegments() throws Exception { + SessionVariable session = new SessionVariable(); + session.lanceFragmentsPerSplit = 1; + LanceScanNode node = newNode(session); + setMetadata(node, scalarMetadata(Collections.singletonList( + scalarSegment(UUID.randomUUID(), IndexType.BTREE, Arrays.asList(1L, 2L, 3L, 4L))))); + setPushedConjuncts(node, scalarPredicate()); + List splits = node.getSplits(20); + Assert.assertEquals(4, splits.size()); + for (Split split : splits) { + Assert.assertFalse(((LanceSplit) split).hasIndexSegmentUuids()); + } + } + + @Test + public void testScalarSegmentDoesNotPushLimitPastDorisResidual() throws Exception { + LanceScanNode node = newNode(); + setMetadata(node, scalarMetadata(Collections.singletonList( + scalarSegment(UUID.randomUUID(), IndexType.BTREE, Arrays.asList(1L, 2L, 3L, 4L))))); + node.getConjuncts().add(scalarPredicate()); + node.getConjuncts().add(new BinaryPredicate(BinaryPredicate.Operator.EQ, + new FunctionCallExpr("abs", Collections.singletonList(new SlotRef(null, "key"))), + new IntLiteral(2))); + node.convertPredicate(); + node.setLimit(1); + List splits = node.getSplits(20); + Assert.assertEquals(1, splits.size()); + Assert.assertEquals(1, node.getConjuncts().size()); + TFileRangeDesc range = new TFileRangeDesc(); + node.setScanParams(range, splits.get(0)); + Assert.assertTrue(range.getTableFormatParams().getLanceParams().isSetIndexSegmentUuids()); + Assert.assertFalse(range.getTableFormatParams().getLanceParams().isSetLimit()); + } + + private static Expr scalarPredicate() { + return new BinaryPredicate(BinaryPredicate.Operator.GE, new SlotRef(null, "key"), new IntLiteral(2)); + } + + private static LanceIndexSegmentInfo scalarSegment(UUID uuid, IndexType type, List fragments) { + return new LanceIndexSegmentInfo(uuid, "key_idx", Collections.singletonList(9), fragments, type, null); + } + + private static LanceTableMetadata scalarMetadata(List segments) { + return LanceTableMetadata.withIndexSegments("s3://bucket/scalar.lance", 42, + new Schema(Collections.singletonList(Field.nullable("key", new ArrowType.Int(64, true)))), + Arrays.asList(new LanceFragmentInfo(1, 10, 12), new LanceFragmentInfo(2, 20, 20), + new LanceFragmentInfo(3, 30, 30), new LanceFragmentInfo(4, 40, 40)), + Collections.singletonMap("key", 9), segments, Collections.emptyMap()); + } + + private static void setPushedConjuncts(LanceScanNode node, Expr predicate) { + node.getConjuncts().clear(); + node.getConjuncts().add(predicate); + node.convertPredicate(); + Assert.assertTrue(node.getConjuncts().isEmpty()); + } + @Test public void testGroupedFragmentsPreserveCoverageWeightsAndScanParams() throws Exception { SessionVariable sessionVariable = new SessionVariable(); diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 45b65a0d63d361..5c190e10188a3c 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -570,10 +570,15 @@ struct TLanceFileDesc { // most this many rows; the upper LIMIT operator still enforces the global bound. // Only set for ordinary scans whose predicates are fully pushed into Lance. 4: optional i64 limit - // Physical vector or FTS index segments assigned to this distributed search split. Each value - // is one UUID encoded as 16 bytes in RFC 4122 order. Unset for ordinary and vector - // unindexed-fragment scans. + // Physical index segments assigned to this split. Each UUID is 16 bytes in RFC 4122 order. + // An ordinary scan accepts exactly one scalar segment and requires a fixed version and + // nonempty fragment_ids. Vector/FTS scans interpret these as their own index segments. + // Unset for fragment scans without an assigned segment. 5: optional list index_segment_uuids + // Ordinary scans only. False for uncovered-fragment tasks in a scalar segment plan, + // so these tasks filter their rows without repeating global scalar-index evaluation. + // Unset preserves Lance's default; an explicit scalar segment must not be combined with false. + 6: optional bool use_scalar_index } struct TLanceScanParams { diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index b8f0374a4d4cfc..43648a1abf9b22 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -718,25 +718,18 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " AZURE " ]]; then echo "Finished patching ${AZURE_SOURCE}" fi -# Apply Doris lance-c patches. +# Apply Doris lance-c patches as one chain to the pinned release archive. if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then - if [[ "${LANCE_C_SOURCE}" == "lance-c-0.1.9" ]]; then - cd "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}" - if [[ ! -f "${PATCHED_MARK}" ]]; then - patch --batch --forward --reject-file=- --fuzz=0 --no-backup-if-mismatch -s \ - -p1 <"${TP_PATCH_DIR}/lance-c-0.1.9-pr-73.patch" - patch --batch --forward --reject-file=- --fuzz=0 --no-backup-if-mismatch -s \ - -p1 <"${TP_PATCH_DIR}/lance-c-0.1.9-pr-74.patch" - touch "${PATCHED_MARK}" - fi - # Also update source trees that already have PR #73 and PR #74 applied. - if [[ ! -f "${PATCHED_MARK}_pr_75_pr_78" ]]; then + cd "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}" + if [[ ! -f "${PATCHED_MARK}" ]]; then + # PR #79 requires the Lance v11 APIs introduced by PR #77. + for lance_patch in pr-73 pr-74 pr-75-pr-78 pr-77 pr-79; do patch --batch --forward --reject-file=- --fuzz=0 --no-backup-if-mismatch -s \ - -p1 <"${TP_PATCH_DIR}/lance-c-0.1.9-pr-75-pr-78.patch" - touch "${PATCHED_MARK}_pr_75_pr_78" - fi - cd - + -p1 <"${TP_PATCH_DIR}/${LANCE_C_SOURCE}-${lance_patch}.patch" + done + touch "${PATCHED_MARK}" fi + cd - echo "Finished patching ${LANCE_C_SOURCE}" fi diff --git a/thirdparty/patches/lance-c-0.1.9-pr-77.patch b/thirdparty/patches/lance-c-0.1.9-pr-77.patch new file mode 100644 index 00000000000000..9a8931c6c2afd4 --- /dev/null +++ b/thirdparty/patches/lance-c-0.1.9-pr-77.patch @@ -0,0 +1,1866 @@ +Backport Lance-C PR #77: build: bump lance to v11.0.0. +Upstream: https://github.com/lance-format/lance-c/pull/77 +Commit: eaf06c0374e62de6b519f55efe17de0c58e88c0a +Merged as: 373c2bb53d6d7e2f2a8240ffa4ba1e82daa9fe16 +Apply after PR #73, #74, and #75/#78 on lance-c v0.1.9. + +Cargo.lock is adapted to retain the Foyer dependencies added by PR #73; +the asyncband package is already present, and Foyer's bitflags dependency +is disambiguated. All other upstream changes are retained. + +diff --git a/Cargo.lock b/Cargo.lock +--- a/Cargo.lock ++++ b/Cargo.lock +@@ -222,7 +222,7 @@ + "arrow-schema", + "arrow-select", + "atoi", +- "base64", ++ "base64 0.22.1", + "chrono", + "comfy-table", + "half", +@@ -332,7 +332,7 @@ + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" + dependencies = [ +- "bitflags", ++ "bitflags 2.11.0", + "serde_core", + "serde_json", + ] +@@ -514,7 +514,6 @@ + checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" + dependencies = [ + "aws-lc-sys", +- "untrusted 0.7.1", + "zeroize", + ] + +@@ -641,7 +640,7 @@ + "bytes", + "form_urlencoded", + "hex", +- "hmac", ++ "hmac 0.12.1", + "http 0.2.12", + "http 1.4.0", + "percent-encoding", +@@ -840,6 +839,12 @@ + checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + + [[package]] ++name = "base64" ++version = "0.23.1" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" ++ ++[[package]] + name = "base64-simd" + version = "0.8.0" + source = "registry+https://github.com/rust-lang/crates.io-index" +@@ -870,6 +875,12 @@ + + [[package]] + name = "bitflags" ++version = "1.3.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" ++ ++[[package]] ++name = "bitflags" + version = "2.11.0" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +@@ -897,16 +908,15 @@ + + [[package]] + name = "blake3" +-version = "1.8.3" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" +-dependencies = [ +- "arrayref", ++version = "1.8.7" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" ++dependencies = [ + "arrayvec", + "cc", + "cfg-if 1.0.4", + "constant_time_eq", +- "cpufeatures 0.2.17", ++ "cpufeatures 0.3.0", + ] + + [[package]] +@@ -1138,6 +1148,12 @@ + ] + + [[package]] ++name = "cmov" ++version = "0.5.4" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" ++ ++[[package]] + name = "colorchoice" + version = "1.0.5" + source = "registry+https://github.com/rust-lang/crates.io-index" +@@ -1316,12 +1332,13 @@ + ] + + [[package]] +-name = "crc32c" +-version = "0.6.8" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +-dependencies = [ +- "rustc_version", ++name = "crc-fast" ++version = "1.10.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" ++dependencies = [ ++ "digest 0.10.7", ++ "spin 0.10.1", + ] + + [[package]] +@@ -1457,6 +1474,15 @@ + version = "0.0.7" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1" ++ ++[[package]] ++name = "ctutils" ++version = "0.4.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" ++dependencies = [ ++ "cmov", ++] + + [[package]] + name = "darling" +@@ -1806,7 +1832,7 @@ + dependencies = [ + "arrow", + "arrow-buffer", +- "base64", ++ "base64 0.22.1", + "blake2", + "blake3", + "chrono", +@@ -2140,6 +2166,37 @@ + checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d" + + [[package]] ++name = "defmt" ++version = "1.1.1" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" ++dependencies = [ ++ "bitflags 1.3.2", ++ "defmt-macros", ++] ++ ++[[package]] ++name = "defmt-macros" ++version = "1.1.1" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" ++dependencies = [ ++ "defmt-parser", ++ "proc-macro2", ++ "quote", ++ "syn 2.0.117", ++] ++ ++[[package]] ++name = "defmt-parser" ++version = "1.0.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" ++dependencies = [ ++ "thiserror 2.0.18", ++] ++ ++[[package]] + name = "der" + version = "0.7.10" + source = "registry+https://github.com/rust-lang/crates.io-index" +@@ -2181,6 +2238,7 @@ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", ++ "ctutils", + ] + + [[package]] +@@ -2359,7 +2417,7 @@ + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" + dependencies = [ +- "bitflags", ++ "bitflags 2.11.0", + "rustc_version", + ] + +@@ -2459,7 +2517,7 @@ + dependencies = [ + "anyhow", + "asyncband", +- "bitflags", ++ "bitflags 2.11.0", + "datasketches", + "equivalent", + "foyer-common", +@@ -2518,6 +2576,12 @@ + ] + + [[package]] ++name = "frostem" ++version = "1.20260821.5" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "36a80a7406da302e04bfd2ca987907590d3a1f3c69958947c43890abd7426b2f" ++ ++[[package]] + name = "fs4" + version = "0.13.1" + source = "registry+https://github.com/rust-lang/crates.io-index" +@@ -2535,8 +2599,8 @@ + + [[package]] + name = "fsst" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow-array", + "rand 0.9.2", +@@ -2885,14 +2949,23 @@ + + [[package]] + name = "goosefs-sdk" +-version = "0.1.5" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f" +-dependencies = [ ++version = "0.1.9" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "e1ea4eee6dcbc31b25ab4fd577adc55b677d2bed3aa3016c44c58fbe1b2298a5" ++dependencies = [ ++ "arc-swap", + "async-trait", + "bytes", + "dashmap", ++ "fastrand", ++ "futures", + "hostname", ++ "io-uring", ++ "itoa", ++ "libc", ++ "lru", ++ "memmap2", ++ "moka", + "prost", + "prost-types", + "rand 0.9.2", +@@ -2905,6 +2978,7 @@ + "tonic-prost", + "tracing", + "uuid", ++ "xxhash-rust", + ] + + [[package]] +@@ -3056,6 +3130,15 @@ + checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" + dependencies = [ + "digest 0.10.7", ++] ++ ++[[package]] ++name = "hmac" ++version = "0.13.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" ++dependencies = [ ++ "digest 0.11.3", + ] + + [[package]] +@@ -3211,7 +3294,7 @@ + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" + dependencies = [ +- "base64", ++ "base64 0.22.1", + "bytes", + "futures-channel", + "futures-util", +@@ -3505,7 +3588,7 @@ + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62" + dependencies = [ +- "bitflags", ++ "bitflags 2.11.0", + "cfg-if 1.0.4", + "libc", + ] +@@ -3567,10 +3650,12 @@ + + [[package]] + name = "jiff" +-version = "0.2.23" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" +-dependencies = [ ++version = "0.2.35" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" ++dependencies = [ ++ "defmt", ++ "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "js-sys", +@@ -3579,15 +3664,25 @@ + "portable-atomic-util", + "serde_core", + "wasm-bindgen", +- "windows-sys 0.61.2", ++ "windows-link", ++] ++ ++[[package]] ++name = "jiff-core" ++version = "0.1.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" ++dependencies = [ ++ "defmt", + ] + + [[package]] + name = "jiff-static" +-version = "0.2.23" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +-dependencies = [ ++version = "0.2.35" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" ++dependencies = [ ++ "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.117", +@@ -3698,24 +3793,6 @@ + ] + + [[package]] +-name = "jsonwebtoken" +-version = "10.4.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" +-dependencies = [ +- "aws-lc-rs", +- "base64", +- "getrandom 0.2.17", +- "js-sys", +- "pem", +- "serde", +- "serde_json", +- "signature", +- "simple_asn1", +- "zeroize", +-] +- +-[[package]] + name = "konst" + version = "0.4.3" + source = "registry+https://github.com/rust-lang/crates.io-index" +@@ -3734,8 +3811,8 @@ + + [[package]] + name = "lance" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arc-swap", + "arrow", +@@ -3751,7 +3828,6 @@ + "async-recursion", + "async-trait", + "async_cell", +- "aws-credential-types", + "byteorder", + "bytes", + "chrono", +@@ -3766,7 +3842,6 @@ + "either", + "fst", + "futures", +- "half", + "humantime", + "itertools 0.14.0", + "lance-arrow", +@@ -3808,8 +3883,8 @@ + + [[package]] + name = "lance-arrow" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow-array", + "arrow-buffer", +@@ -3831,7 +3906,7 @@ + [[package]] + name = "lance-arrow-scalar" + version = "58.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow-array", + "arrow-buffer", +@@ -3845,7 +3920,7 @@ + [[package]] + name = "lance-arrow-stats" + version = "58.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow-array", + "arrow-schema", +@@ -3854,8 +3929,8 @@ + + [[package]] + name = "lance-bitpacking" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrayref", + "crunchy", +@@ -3899,20 +3974,19 @@ + + [[package]] + name = "lance-core" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "async-trait", +- "byteorder", ++ "blake3", + "bytes", + "datafusion-common", + "datafusion-sql", + "futures", +- "itertools 0.14.0", + "lance-arrow", + "lance-derive", + "libc", +@@ -3923,13 +3997,13 @@ + "object_store", + "pin-project", + "prost", ++ "quick_cache", + "rand 0.9.2", + "roaring", + "serde_json", + "snafu", + "tempfile", + "tokio", +- "tokio-stream", + "tokio-util", + "tracing", + "twox-hash", +@@ -3938,8 +4012,8 @@ + + [[package]] + name = "lance-datafusion" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow", + "arrow-array", +@@ -3959,7 +4033,6 @@ + "jsonb", + "lance-arrow", + "lance-core", +- "lance-datagen", + "lance-geo", + "log", + "pin-project", +@@ -3971,8 +4044,8 @@ + + [[package]] + name = "lance-datagen" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow", + "arrow-array", +@@ -3989,8 +4062,8 @@ + + [[package]] + name = "lance-derive" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "proc-macro2", + "quote", +@@ -3999,8 +4072,8 @@ + + [[package]] + name = "lance-encoding" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow-arith", + "arrow-array", +@@ -4025,8 +4098,6 @@ + "num-traits", + "prost", + "prost-build", +- "rand 0.9.2", +- "strum", + "tokio", + "tracing", + "xxhash-rust", +@@ -4035,12 +4106,13 @@ + + [[package]] + name = "lance-file" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", ++ "arrow-cast", + "arrow-data", + "arrow-schema", + "arrow-select", +@@ -4066,8 +4138,8 @@ + + [[package]] + name = "lance-geo" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "datafusion", + "geo-traits", +@@ -4081,13 +4153,14 @@ + + [[package]] + name = "lance-index" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arc-swap", + "arrow", + "arrow-arith", + "arrow-array", ++ "arrow-ipc", + "arrow-ord", + "arrow-schema", + "arrow-select", +@@ -4096,7 +4169,6 @@ + "async-trait", + "bitvec", + "bytes", +- "chrono", + "crossbeam-queue", + "datafusion", + "datafusion-common", +@@ -4116,7 +4188,6 @@ + "lance-bitpacking", + "lance-core", + "lance-datafusion", +- "lance-datagen", + "lance-encoding", + "lance-file", + "lance-geo", +@@ -4146,13 +4217,12 @@ + "tempfile", + "tokio", + "tracing", +- "uuid", + ] + + [[package]] + name = "lance-index-core" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow-array", + "arrow-schema", +@@ -4174,18 +4244,12 @@ + + [[package]] + name = "lance-io" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow", +- "arrow-arith", + "arrow-array", +- "arrow-buffer", +- "arrow-cast", +- "arrow-data", + "arrow-schema", +- "arrow-select", +- "async-recursion", + "async-trait", + "aws-config", + "aws-credential-types", +@@ -4193,10 +4257,8 @@ + "bytes", + "chrono", + "futures", +- "goosefs-sdk", + "http 1.4.0", + "io-uring", +- "lance-arrow", + "lance-core", + "lance-namespace", + "log", +@@ -4208,34 +4270,37 @@ + "pin-project", + "prost", + "rand 0.9.2", ++ "reqsign-core", ++ "reqsign-file-read-tokio", ++ "reqsign-google", + "serde", ++ "serde_json", + "tempfile", + "tokio", + "tracing", + "url", ++ "uuid", + ] + + [[package]] + name = "lance-linalg" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow-array", +- "arrow-buffer", + "arrow-schema", + "cc", + "half", + "lance-arrow", + "lance-core", + "num-traits", +- "rand 0.9.2", + "rayon", + ] + + [[package]] + name = "lance-namespace" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow", + "async-trait", +@@ -4247,9 +4312,9 @@ + + [[package]] + name = "lance-namespace-reqwest-client" +-version = "0.8.6" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "ba3f0a235e3ed5f8805205649ccc7d7d0f3df23ce1294242c9265ad488d7f19d" ++version = "0.11.1" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "1d06b1fbb5d41f93bc652b61e2872af92e8a6c5f6b4ce8839a8ecfa05365d359" + dependencies = [ + "reqwest 0.12.28", + "serde", +@@ -4261,14 +4326,13 @@ + + [[package]] + name = "lance-select" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "byteorder", +- "bytes", + "itertools 0.14.0", + "lance-core", + "roaring", +@@ -4277,8 +4341,8 @@ + + [[package]] + name = "lance-table" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow", + "arrow-array", +@@ -4286,6 +4350,7 @@ + "arrow-ipc", + "arrow-schema", + "async-trait", ++ "blake3", + "byteorder", + "bytes", + "chrono", +@@ -4315,11 +4380,11 @@ + + [[package]] + name = "lance-tokenizer" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" +-dependencies = [ ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" ++dependencies = [ ++ "frostem", + "icu_segmenter", +- "rust-stemmers", + "serde", + "stop-words", + "unicode-normalization", +@@ -4331,7 +4396,7 @@ + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + dependencies = [ +- "spin", ++ "spin 0.9.8", + ] + + [[package]] +@@ -4462,9 +4527,9 @@ + + [[package]] + name = "log" +-version = "0.4.29" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" ++version = "0.4.34" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + + [[package]] + name = "loom" +@@ -4480,6 +4545,15 @@ + ] + + [[package]] ++name = "lru" ++version = "0.18.4" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" ++dependencies = [ ++ "hashbrown 0.17.1", ++] ++ ++[[package]] + name = "lru-slab" + version = "0.1.2" + source = "registry+https://github.com/rust-lang/crates.io-index" +@@ -4569,6 +4643,15 @@ + version = "2.8.0" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" ++ ++[[package]] ++name = "memmap2" ++version = "0.9.11" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" ++dependencies = [ ++ "libc", ++] + + [[package]] + name = "memoffset" +@@ -4809,7 +4892,7 @@ + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" + dependencies = [ +- "bitflags", ++ "bitflags 2.11.0", + ] + + [[package]] +@@ -4838,7 +4921,7 @@ + checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" + dependencies = [ + "async-trait", +- "base64", ++ "base64 0.22.1", + "bytes", + "chrono", + "form_urlencoded", +@@ -4854,7 +4937,7 @@ + "md-5 0.10.6", + "parking_lot", + "percent-encoding", +- "quick-xml", ++ "quick-xml 0.39.4", + "rand 0.10.1", + "reqwest 0.12.28", + "ring", +@@ -4873,9 +4956,9 @@ + + [[package]] + name = "object_store_opendal" +-version = "0.57.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8" ++version = "0.58.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "88f165780495c17aa3ce86846600504198c3fffd99073521552751c2430fa6ac" + dependencies = [ + "async-trait", + "bytes", +@@ -4908,12 +4991,13 @@ + + [[package]] + name = "opendal" +-version = "0.57.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" ++version = "0.58.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "33dbff14cc9bb085224256d6a81289d2f3202e85b06f408d42534b42162a4231" + dependencies = [ + "ctor 1.0.13", + "opendal-core", ++ "opendal-http-transport-reqwest", + "opendal-layer-concurrent-limit", + "opendal-layer-logging", + "opendal-layer-retry", +@@ -4931,24 +5015,22 @@ + + [[package]] + name = "opendal-core" +-version = "0.57.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309" ++version = "0.58.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "48dbcef97d3eb7591db2c18d5cae95c836bcce07359b98d98dd6f4e861eb77b7" + dependencies = [ + "anyhow", +- "base64", ++ "asyncband", ++ "base64 0.23.1", + "bytes", + "futures", + "http 1.4.0", +- "http-body 1.0.1", + "jiff", + "log", + "md-5 0.11.0", +- "mea", + "percent-encoding", +- "quick-xml", ++ "quick-xml 0.41.0", + "reqsign-core", +- "reqwest 0.13.3", + "serde", + "serde_json", + "tokio", +@@ -4958,22 +5040,36 @@ + ] + + [[package]] +-name = "opendal-layer-concurrent-limit" +-version = "0.57.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f" +-dependencies = [ ++name = "opendal-http-transport-reqwest" ++version = "0.58.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "85663452ea32bbc17e8f79ab29788c846d116ec7de31451be9c787e462dcb36c" ++dependencies = [ ++ "bytes", + "futures", + "http 1.4.0", +- "mea", ++ "http-body 1.0.1", + "opendal-core", ++ "reqwest 0.13.4", ++] ++ ++[[package]] ++name = "opendal-layer-concurrent-limit" ++version = "0.58.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "03f9e144b5228d741c3763ade8711d9b72e5fb6d998e779f2d7a09da0b5a3eba" ++dependencies = [ ++ "asyncband", ++ "futures", ++ "http 1.4.0", ++ "opendal-core", + ] + + [[package]] + name = "opendal-layer-logging" +-version = "0.57.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2" ++version = "0.58.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "c2de17c61cd32e9d8d7d8efb91795e714dbccbafbc3c4e219e1542f4d6324161" + dependencies = [ + "log", + "opendal-core", +@@ -4981,9 +5077,9 @@ + + [[package]] + name = "opendal-layer-retry" +-version = "0.57.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7" ++version = "0.58.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "e94db301964a25366090484d61e6da16d5979cc8faf02c3e12210dc74fafed38" + dependencies = [ + "backon", + "log", +@@ -4992,9 +5088,9 @@ + + [[package]] + name = "opendal-layer-timeout" +-version = "0.57.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3" ++version = "0.58.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "08956ddda07465449bfd48825f4f0f25e0351278ac974eaa659895d9d74f2c80" + dependencies = [ + "opendal-core", + "tokio", +@@ -5002,17 +5098,17 @@ + + [[package]] + name = "opendal-service-azblob" +-version = "0.57.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a" +-dependencies = [ +- "base64", ++version = "0.58.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "6d278d2fb57661947d1c9fb44047e432b2782c48dc085b7abc99fe6e18c26cf8" ++dependencies = [ ++ "base64 0.23.1", + "bytes", + "http 1.4.0", + "log", + "opendal-core", + "opendal-service-azure-common", +- "quick-xml", ++ "quick-xml 0.41.0", + "reqsign-azure-storage", + "reqsign-core", + "reqsign-file-read-tokio", +@@ -5023,17 +5119,18 @@ + + [[package]] + name = "opendal-service-azdls" +-version = "0.57.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361" +-dependencies = [ +- "base64", ++version = "0.58.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "2d564484a8f7d091827e825cfc91ed45bd48e64d262451ee041fe843db81bd8a" ++dependencies = [ ++ "asyncband", ++ "base64 0.23.1", + "bytes", + "http 1.4.0", + "log", + "opendal-core", + "opendal-service-azure-common", +- "quick-xml", ++ "quick-xml 0.41.0", + "reqsign-azure-storage", + "reqsign-core", + "reqsign-file-read-tokio", +@@ -5043,9 +5140,9 @@ + + [[package]] + name = "opendal-service-azure-common" +-version = "0.57.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" ++version = "0.58.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "cfcc1bfdac4f54d9018c462dd32ad9e2f68fdf584f825c811550c8ffc87894cd" + dependencies = [ + "http 1.4.0", + "opendal-core", +@@ -5053,15 +5150,15 @@ + + [[package]] + name = "opendal-service-cos" +-version = "0.57.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901" ++version = "0.58.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "bb021c128ebde42e6f3e719d4cfed27e994017aa503a7a1e5818bdb61fd67dc9" + dependencies = [ + "bytes", + "http 1.4.0", + "log", + "opendal-core", +- "quick-xml", ++ "quick-xml 0.41.0", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-tencent-cos", +@@ -5070,9 +5167,9 @@ + + [[package]] + name = "opendal-service-gcs" +-version = "0.57.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47" ++version = "0.58.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "da1f2a8c975fd22fea01f0409bcb8774f1ac02ad79863a3490098203121555d3" + dependencies = [ + "async-trait", + "bytes", +@@ -5080,7 +5177,7 @@ + "log", + "opendal-core", + "percent-encoding", +- "quick-xml", ++ "quick-xml 0.41.0", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-google", +@@ -5091,9 +5188,9 @@ + + [[package]] + name = "opendal-service-goosefs" +-version = "0.57.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4" ++version = "0.58.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "89fd71b80078f2983bd363e322fbebfa6a46d5fc56f17e4f23d76d5edb31226b" + dependencies = [ + "bytes", + "goosefs-sdk", +@@ -5105,9 +5202,9 @@ + + [[package]] + name = "opendal-service-hf" +-version = "0.57.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848" ++version = "0.58.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "c8a3c8ec0c2918fa23f258fa28822f222ec8c1e3a501ded665b78bc65a9d7734" + dependencies = [ + "bytes", + "hf-xet", +@@ -5115,22 +5212,21 @@ + "log", + "opendal-core", + "percent-encoding", +- "reqwest 0.13.3", + "serde", + "serde_json", + ] + + [[package]] + name = "opendal-service-oss" +-version = "0.57.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace" ++version = "0.58.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "8e3ce7a2ceb925e0f28f545b169eb57d04cec6116aae94b8584d2bdbe7d456c6" + dependencies = [ + "bytes", + "http 1.4.0", + "log", + "opendal-core", +- "quick-xml", ++ "quick-xml 0.41.0", + "reqsign-aliyun-oss", + "reqsign-core", + "reqsign-file-read-tokio", +@@ -5139,18 +5235,18 @@ + + [[package]] + name = "opendal-service-s3" +-version = "0.57.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb" +-dependencies = [ +- "base64", ++version = "0.58.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "c64335f9f24ccb62ac36f1d976342b48611a75ba61979813a4f78a4ebd94de42" ++dependencies = [ ++ "base64 0.23.1", + "bytes", +- "crc32c", ++ "crc-fast", + "http 1.4.0", + "log", + "md-5 0.11.0", + "opendal-core", +- "quick-xml", ++ "quick-xml 0.41.0", + "reqsign-aws-v4", + "reqsign-core", + "reqsign-file-read-tokio", +@@ -5160,14 +5256,14 @@ + + [[package]] + name = "opendal-service-tos" +-version = "0.57.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "6f2f7a4c32e5202eb4ac72e76c4b5e30c86ab60762811172f4111103b9d673a1" ++version = "0.58.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "70c3c507c3a565b2feb4c7b5f653436acd34ca59ffc842a7671b5498b13b76bf" + dependencies = [ + "bytes", + "http 1.4.0", + "opendal-core", +- "quick-xml", ++ "quick-xml 0.41.0", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-volcengine-tos", +@@ -5274,7 +5370,7 @@ + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "898bac3fa00d0ba57a4e8289837e965baa2dee8c3749f3b11d45a64b4223d9c3" + dependencies = [ +- "base64", ++ "base64 0.22.1", + "serde", + ] + +@@ -5312,16 +5408,16 @@ + checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" + dependencies = [ + "digest 0.10.7", +- "hmac", ++ "hmac 0.12.1", + ] + + [[package]] + name = "pem" +-version = "3.0.6" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +-dependencies = [ +- "base64", ++version = "4.0.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" ++dependencies = [ ++ "base64 0.23.1", + "serde_core", + ] + +@@ -5580,6 +5676,28 @@ + dependencies = [ + "memchr", + "serde", ++] ++ ++[[package]] ++name = "quick-xml" ++version = "0.41.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" ++dependencies = [ ++ "memchr", ++ "serde", ++] ++ ++[[package]] ++name = "quick_cache" ++version = "0.6.24" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9" ++dependencies = [ ++ "ahash", ++ "equivalent", ++ "hashbrown 0.16.1", ++ "parking_lot", + ] + + [[package]] +@@ -5806,7 +5924,7 @@ + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" + dependencies = [ +- "bitflags", ++ "bitflags 2.11.0", + ] + + [[package]] +@@ -5887,9 +6005,9 @@ + + [[package]] + name = "reqsign-aliyun-oss" +-version = "3.0.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "57ac2757f3140aa2e213b554148ae0b52733e624fc6723f0cc6bb3d440176c95" ++version = "3.1.4" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "68d24d281f734a463093b7b93aae8b16f5f8496a54fbeec4d2d10b0488295296" + dependencies = [ + "anyhow", + "form_urlencoded", +@@ -5903,18 +6021,18 @@ + ] + + [[package]] +-name = "reqsign-aws-v4" +-version = "3.0.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "44eaca382e94505a49f1a4849658d153aebf79d9c1a58e5dd3b10361511e9f43" +-dependencies = [ +- "anyhow", ++name = "reqsign-aws-core" ++version = "3.1.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "4d63b56638bb3cc7bd376a7cdce1ba3089777a08f47e4097888f2d784cc3f46c" ++dependencies = [ + "bytes", + "form_urlencoded", ++ "hex", + "http 1.4.0", + "log", + "percent-encoding", +- "quick-xml", ++ "quick-xml 0.41.0", + "reqsign-core", + "rust-ini", + "serde", +@@ -5924,17 +6042,31 @@ + ] + + [[package]] ++name = "reqsign-aws-v4" ++version = "3.2.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "4a0c499f4ed12d04c3d4c78fe4cb01aee22c9dae22848c14db2c6313d9df9f43" ++dependencies = [ ++ "bytes", ++ "http 1.4.0", ++ "log", ++ "quick-xml 0.41.0", ++ "reqsign-aws-core", ++ "reqsign-core", ++ "serde", ++] ++ ++[[package]] + name = "reqsign-azure-storage" +-version = "3.0.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "7a321980405d596bd34aaf95c4722a3de4128a67fd19e74a81a83aa3fdf082e6" ++version = "3.2.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "e8177b4f08620ab7f2e9cab7d7ccb9da1b61c66b889fed46cc0880b0fe75eb6b" + dependencies = [ + "anyhow", +- "base64", ++ "base64 0.23.1", + "bytes", + "form_urlencoded", + "http 1.4.0", +- "jsonwebtoken", + "log", + "pem", + "percent-encoding", +@@ -5947,31 +6079,33 @@ + + [[package]] + name = "reqsign-core" +-version = "3.0.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "b10302cf0a7d7e7352ba211fc92c3c5bebf1286153e49cc5aa87348078a8e102" ++version = "3.3.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "f4ac1510872d9481205975d264deb39c109797e5068cc882ed9064270eaae5fa" + dependencies = [ + "anyhow", +- "base64", ++ "base64 0.23.1", + "bytes", +- "form_urlencoded", + "futures", + "hex", +- "hmac", ++ "hmac 0.13.0", + "http 1.4.0", + "jiff", + "log", + "percent-encoding", ++ "rsa", ++ "serde", ++ "serde_json", + "sha1", +- "sha2 0.10.9", ++ "sha2 0.11.0", + "windows-sys 0.61.2", + ] + + [[package]] + name = "reqsign-file-read-tokio" +-version = "3.0.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "e2d89295b3d17abea31851cc8de55d843d89c52132c864963c38d41920613dc5" ++version = "3.0.5" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "95c3371bfc7e5c7f9627a04133af3583fd6c28715e7c83f79db38f3b384f535f" + dependencies = [ + "anyhow", + "reqsign-core", +@@ -5980,13 +6114,13 @@ + + [[package]] + name = "reqsign-google" +-version = "3.0.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "35cc609b49c69e76ecaceb775a03f792d1ed3e7755ab3548d4534fd801e3242e" +-dependencies = [ ++version = "3.1.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "f81a9d38870892443489c0abb5332edfa81d5a14c437af9caef7c194897c92c1" ++dependencies = [ ++ "bytes", + "form_urlencoded", + "http 1.4.0", +- "jsonwebtoken", + "log", + "percent-encoding", + "reqsign-aws-v4", +@@ -5994,15 +6128,14 @@ + "rsa", + "serde", + "serde_json", +- "sha2 0.10.9", + "tokio", + ] + + [[package]] + name = "reqsign-tencent-cos" +-version = "3.0.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "e128f19525861dbded59e1e7c17653a8ed63d573ca04aed708d552dbef5bb32a" ++version = "3.0.5" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "b15c5a4df7c3f16823ae242675c5ebfb52d640cc9a50d1fcf943263247fa1730" + dependencies = [ + "anyhow", + "http 1.4.0", +@@ -6015,9 +6148,9 @@ + + [[package]] + name = "reqsign-volcengine-tos" +-version = "3.0.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "f9d757602a7ef2b6025c0da77e6d2e23fbdef35930fa466b15ffbf0a3f13acf7" ++version = "3.1.1" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "173387eb5ae4cf6a0a7098665ebcc6729862819dc3d95a81aee840767803e3d9" + dependencies = [ + "anyhow", + "http 1.4.0", +@@ -6032,7 +6165,7 @@ + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" + dependencies = [ +- "base64", ++ "base64 0.22.1", + "bytes", + "encoding_rs", + "futures-core", +@@ -6074,11 +6207,11 @@ + + [[package]] + name = "reqwest" +-version = "0.13.3" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" +-dependencies = [ +- "base64", ++version = "0.13.4" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" ++dependencies = [ ++ "base64 0.22.1", + "bytes", + "futures-core", + "futures-util", +@@ -6121,7 +6254,7 @@ + "anyhow", + "async-trait", + "http 1.4.0", +- "reqwest 0.13.3", ++ "reqwest 0.13.4", + "thiserror 2.0.18", + "tower-service", + ] +@@ -6136,7 +6269,7 @@ + "cfg-if 1.0.4", + "getrandom 0.2.17", + "libc", +- "untrusted 0.9.0", ++ "untrusted", + "windows-sys 0.52.0", + ] + +@@ -6199,16 +6332,6 @@ + ] + + [[package]] +-name = "rust-stemmers" +-version = "1.2.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" +-dependencies = [ +- "serde", +- "serde_derive", +-] +- +-[[package]] + name = "rustc-hash" + version = "2.1.1" + source = "registry+https://github.com/rust-lang/crates.io-index" +@@ -6229,7 +6352,7 @@ + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" + dependencies = [ +- "bitflags", ++ "bitflags 2.11.0", + "errno", + "libc", + "linux-raw-sys", +@@ -6309,7 +6432,7 @@ + "aws-lc-rs", + "ring", + "rustls-pki-types", +- "untrusted 0.9.0", ++ "untrusted", + ] + + [[package]] +@@ -6434,7 +6557,7 @@ + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" + dependencies = [ +- "bitflags", ++ "bitflags 2.11.0", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", +@@ -6563,7 +6686,7 @@ + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" + dependencies = [ +- "base64", ++ "base64 0.22.1", + "bs58", + "chrono", + "hex", +@@ -6604,13 +6727,13 @@ + + [[package]] + name = "sha1" +-version = "0.10.6" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" ++version = "0.11.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" + dependencies = [ + "cfg-if 1.0.4", +- "cpufeatures 0.2.17", +- "digest 0.10.7", ++ "cpufeatures 0.3.0", ++ "digest 0.11.3", + ] + + [[package]] +@@ -6714,18 +6837,6 @@ + checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + + [[package]] +-name = "simple_asn1" +-version = "0.6.4" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" +-dependencies = [ +- "num-bigint", +- "num-traits", +- "thiserror 2.0.18", +- "time", +-] +- +-[[package]] + name = "siphasher" + version = "1.0.2" + source = "registry+https://github.com/rust-lang/crates.io-index" +@@ -6799,6 +6910,12 @@ + checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + + [[package]] ++name = "spin" ++version = "0.10.1" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" ++ ++[[package]] + name = "spki" + version = "0.7.3" + source = "registry+https://github.com/rust-lang/crates.io-index" +@@ -6877,28 +6994,6 @@ + version = "0.11.1" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +- +-[[package]] +-name = "strum" +-version = "0.26.3" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +-dependencies = [ +- "strum_macros", +-] +- +-[[package]] +-name = "strum_macros" +-version = "0.26.4" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +-dependencies = [ +- "heck", +- "proc-macro2", +- "quote", +- "rustversion", +- "syn 2.0.117", +-] + + [[package]] + name = "substrait" +@@ -7000,7 +7095,7 @@ + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" + dependencies = [ +- "bitflags", ++ "bitflags 2.11.0", + "core-foundation 0.9.4", + "system-configuration-sys", + ] +@@ -7274,7 +7369,7 @@ + checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" + dependencies = [ + "async-trait", +- "base64", ++ "base64 0.22.1", + "bytes", + "h2", + "http 1.4.0", +@@ -7332,7 +7427,7 @@ + checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" + dependencies = [ + "async-compression", +- "bitflags", ++ "bitflags 2.11.0", + "bytes", + "futures-core", + "futures-util", +@@ -7568,12 +7663,6 @@ + + [[package]] + name = "untrusted" +-version = "0.7.1" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +- +-[[package]] +-name = "untrusted" + version = "0.9.0" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +@@ -7818,7 +7907,7 @@ + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" + dependencies = [ +- "bitflags", ++ "bitflags 2.11.0", + "hashbrown 0.15.5", + "indexmap 2.14.0", + "semver", +@@ -8259,7 +8348,7 @@ + checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" + dependencies = [ + "anyhow", +- "bitflags", ++ "bitflags 2.11.0", + "indexmap 2.14.0", + "log", + "serde", +@@ -8337,7 +8426,7 @@ + dependencies = [ + "anyhow", + "async-trait", +- "base64", ++ "base64 0.22.1", + "bytes", + "clap", + "crc32fast", +@@ -8348,7 +8437,7 @@ + "more-asserts", + "rand 0.10.1", + "redb", +- "reqwest 0.13.3", ++ "reqwest 0.13.4", + "reqwest-middleware", + "serde", + "serde_json", +@@ -8374,7 +8463,7 @@ + checksum = "cb838aa8eb67d730af301584cf003caad407487606058292a6750711b603fbee" + dependencies = [ + "async-trait", +- "base64", ++ "base64 0.22.1", + "blake3", + "bytemuck", + "bytes", +@@ -8461,7 +8550,7 @@ + "oneshot", + "pin-project", + "rand 0.10.1", +- "reqwest 0.13.3", ++ "reqwest 0.13.4", + "serde", + "serde_json", + "shellexpand", +@@ -8557,20 +8646,6 @@ + version = "1.8.2" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +-dependencies = [ +- "zeroize_derive", +-] +- +-[[package]] +-name = "zeroize_derive" +-version = "1.4.3" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +-dependencies = [ +- "proc-macro2", +- "quote", +- "syn 2.0.117", +-] + + [[package]] + name = "zerotrie" +diff --git a/Cargo.toml b/Cargo.toml +--- a/Cargo.toml ++++ b/Cargo.toml +@@ -18,14 +18,14 @@ + crate-type = ["cdylib", "staticlib", "rlib"] + + [dependencies] +-lance = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c", features = ["substrait"] } +-lance-core = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" } +-lance-file = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" } +-lance-index = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" } +-lance-io = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" } +-lance-linalg = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" } +-lance-table = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" } +-lance-datafusion = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c", features = ["substrait"] } ++lance = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe", features = ["substrait"] } ++lance-core = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } ++lance-file = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } ++lance-index = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } ++lance-io = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } ++lance-linalg = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } ++lance-table = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } ++lance-datafusion = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe", features = ["substrait"] } + datafusion = { version = "54.0.0", default-features = false } + arrow = { version = "58.0.0", features = ["prettyprint", "ffi"] } + arrow-array = "58.0.0" +@@ -49,9 +49,9 @@ + uuid = { version = "1", features = ["v4"] } + + [dev-dependencies] +-lance = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c", features = ["substrait"] } +-lance-datagen = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" } +-lance-file = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" } ++lance = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe", features = ["substrait"] } ++lance-datagen = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } ++lance-file = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } + tokio = { version = "1", features = ["rt-multi-thread", "macros"] } + arrow-array = "58.0.0" + arrow-schema = "58.0.0" +diff --git a/src/fts_query.rs b/src/fts_query.rs +--- a/src/fts_query.rs ++++ b/src/fts_query.rs +@@ -246,7 +246,7 @@ + .with_max_expansions(match_query.max_expansions) + .with_prefix_length(match_query.prefix_length); + PreparedFtsQuery::Match(Arc::new( +- build_global_bm25_scorer(&indices, &query_tokens, ¶ms).await?, ++ build_global_bm25_scorer(&indices, &query_tokens, ¶ms, None).await?, + )) + } + FtsQuery::Phrase(phrase_query) => { +@@ -260,7 +260,7 @@ + let query_tokens = collect_query_tokens(&phrase_query.terms, &mut tokenizer); + let params = query.params().with_phrase_slop(Some(phrase_query.slop)); + PreparedFtsQuery::Phrase(Arc::new( +- build_global_bm25_scorer(&indices, &query_tokens, ¶ms).await?, ++ build_global_bm25_scorer(&indices, &query_tokens, ¶ms, None).await?, + )) + } + _ => { +diff --git a/src/index_segment.rs b/src/index_segment.rs +--- a/src/index_segment.rs ++++ b/src/index_segment.rs +@@ -889,7 +889,7 @@ + // TODO(upstream-lance): Remove this fail-fast once Lance's distributed + // vector-index path reconstructs a supplied PQ codebook with an L2 + // ProductQuantizer, matching the ordinary full-dataset path. Pinned Lance +- // revision e934cc2c rewraps supplied codebooks with DistanceType::Dot in ++ // revision ab6b5bbe rewraps supplied codebooks with DistanceType::Dot in + // `make_global_pq`, which silently switches PQ code assignment away from + // the L2 contract shared by full-dataset builds and index readers. + if matches!( +@@ -910,7 +910,7 @@ + let selected_fragment_ids: HashSet = fragment_ids.iter().copied().collect(); + if selected_fragment_ids != all_fragment_ids { + return Err(invalid_input(format!( +- "pq_codebook is supplied for metric=DOT, index_type={:?}, mode={:?}, and an effective strict fragment subset ({} of {} fragments): pinned Lance revision e934cc2c reconstructs the supplied codebook with a DOT ProductQuantizer in the distributed build path (make_global_pq), silently breaking the L2 PQ-assignment contract; cover the full dataset in one segment (pass NULL fragment_ids or list every fragment) or wait for upstream Lance DOT support", ++ "pq_codebook is supplied for metric=DOT, index_type={:?}, mode={:?}, and an effective strict fragment subset ({} of {} fragments): pinned Lance revision ab6b5bbe reconstructs the supplied codebook with a DOT ProductQuantizer in the distributed build path (make_global_pq), silently breaking the L2 PQ-assignment contract; cover the full dataset in one segment (pass NULL fragment_ids or list every fragment) or wait for upstream Lance DOT support", + params.index_type, + parsed.mode, + selected_fragment_ids.len(), +diff --git a/src/scanner.rs b/src/scanner.rs +--- a/src/scanner.rs ++++ b/src/scanner.rs +@@ -661,7 +661,7 @@ + exec.params().clone(), + exec.prefilter_source().clone(), + segments.to_vec(), +- ) ++ )? + .with_base_scorer(Arc::clone(scorer)); + return Ok((Arc::new(replacement), rewritten)); + } +@@ -681,7 +681,7 @@ + exec.params().clone(), + exec.prefilter_source().clone(), + segments.to_vec(), +- ) ++ )? + .with_base_scorer(Arc::clone(scorer)); + return Ok((Arc::new(replacement), rewritten)); + } +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -2845,8 +2845,7 @@ + format!("data/{}", filename), + field_ids, + column_indices, +- meta.major_version as u32, +- meta.minor_version as u32, ++ meta.version, + None, // file_size_bytes + None, // base_id + ); +@@ -3919,6 +3918,7 @@ + created_at: Some(u64::MAX), + base_id: None, + files: Vec::new(), ++ covering_fields: Vec::new(), + } + .encode_to_vec(); + assert_eq!( +@@ -3943,6 +3943,7 @@ + created_at: None, + base_id: None, + files: Vec::new(), ++ covering_fields: Vec::new(), + } + .encode_to_vec(); + assert_eq!( +@@ -3970,6 +3971,7 @@ + created_at: None, + base_id: None, + files: Vec::new(), ++ covering_fields: Vec::new(), + } + .encode_to_vec(); + assert_eq!( +@@ -4547,7 +4549,7 @@ + let message = take_last_error_message(); + assert!(message.contains("metric=DOT"), "{message}"); + assert!(message.contains("strict fragment subset"), "{message}"); +- assert!(message.contains("e934cc2c"), "{message}"); ++ assert!(message.contains("ab6b5bbe"), "{message}"); + assert!(message.contains("1 of 2 fragments"), "{message}"); + assert!(!centroids.is_released()); + assert!(!codebook.is_released()); diff --git a/thirdparty/patches/lance-c-0.1.9-pr-79.patch b/thirdparty/patches/lance-c-0.1.9-pr-79.patch new file mode 100644 index 00000000000000..ad32a48065f09d --- /dev/null +++ b/thirdparty/patches/lance-c-0.1.9-pr-79.patch @@ -0,0 +1,1782 @@ +From d819fbdfa52031d84d1fa01f2d06c51a6712c40b Mon Sep 17 00:00:00 2001 +From: zhangstar333 +Date: Tue, 8 Sep 2026 22:30:48 +0800 +Subject: [PATCH 1/5] scalar index segment + +--- + docs/scalar-segment-scans.md | 78 +++++++++++ + include/lance/lance.h | 24 ++++ + include/lance/lance.hpp | 14 ++ + src/lib.rs | 1 + + src/scalar_segment.rs | 224 +++++++++++++++++++++++++++++++ + src/scanner.rs | 61 +++++++++ + tests/c_api_test.rs | 249 +++++++++++++++++++++++++++++++++++ + 7 files changed, 651 insertions(+) + create mode 100644 docs/scalar-segment-scans.md + create mode 100644 src/scalar_segment.rs + +diff --git a/docs/scalar-segment-scans.md b/docs/scalar-segment-scans.md +new file mode 100644 +index 0000000..dc7c141 +--- /dev/null ++++ b/docs/scalar-segment-scans.md +@@ -0,0 +1,78 @@ ++# Scalar index segment scans ++ ++An ordinary scanner can use one physical BTree/Bitmap segment to generate ++candidates, then read those candidates with the complete scanner filter. This ++does not run a global search of the other segments of the logical index. It does ++not subdivide a physical segment or make its own index search incremental. ++ ++## Configuring a task ++ ++Open a fixed dataset version. Select the physical index UUID from that version's ++metadata and pass the task's complete fragment domain explicitly: ++ ++```c ++LanceScanner *scanner = lance_scanner_new(dataset, columns, full_filter_sql); ++/* Check every return value in production. */ ++lance_scanner_set_fragment_ids(scanner, fragment_ids, fragment_count); ++lance_scanner_set_scalar_index_segment(scanner, segment_uuid_16_bytes); ++lance_scanner_set_limit(scanner, 20000); ++/* The scanner-owning thread calls lance_scanner_next as usual. */ ++``` ++ ++SQL, Substrait and additional SQL filters keep their existing precedence and AND ++composition. The caller does not supply a separate driver predicate: Lance-C ++uses the typed filter planner and selects a necessary indexed leaf belonging to ++the requested logical index. It only descends through AND, never through OR or ++NOT. It then searches the selected UUID and applies the complete filter while ++reading candidates with automatic scalar-index planning disabled. ++ ++Each task's fragment IDs define its result domain, including on fallback. A ++distributed planner must assign disjoint domains whose union covers the intended ++scan. Unindexed fragments need their own tasks, or an explicit domain including ++them (which causes that task to use fallback). Merely listing indexed segments ++does not include appended, unindexed data automatically. ++ ++An unknown UUID, absent fragment or invalid option combination is an error. A ++known segment with incomplete/unknown coverage, no suitable driver, unsupported ++index type, nested key, overlays, fragment reuse, non-exact results or unsupported ++row-ID domain falls back to a non-indexed scan of the entire explicit domain. ++I/O and corruption errors are propagated, not converted to empty results or ++successful fallback. ++ ++The first implementation supports live-row ordinary scans and cannot be combined ++with vector/FTS queries. Physical row-address ++results on stable-row-ID datasets currently fall back; results already expressed ++in the correct row-ID domain use the candidate path. Deletes and all remaining ++predicates are handled by the ordinary reader. No candidate-count limit is ++applied: LIMIT/OFFSET remain after the scanner's complete filter. ++ ++If the host has additional predicates outside Lance, do not set a local limit ++before those predicates. Never divide the global limit by the number of tasks. ++Global OFFSET belongs to the coordinator, not independently to each task. ++ ++## Stopping after the host limit ++ ++This mode uses the existing scanner/stream lifecycle. Once the host has enough ++rows, it stops requesting further batches and closes the scanner after any active ++call has returned. Do not call `lance_scanner_close` concurrently with `next`. ++Exported Arrow streams remain owned by the caller and must also be released after ++their active consumers have finished. ++ ++A host stop flag does not interrupt an in-progress `lance_scanner_next`: current ++index evaluation or I/O may finish before the host observes stop and closes the ++stream. No separate cancellation signal or thread is introduced. The host remains ++responsible for enforcing the global LIMIT across concurrent tasks. ++ ++## Memory and statistics ++ ++Candidate masks stay in Rust, and record batches are streamed. Each active task ++can still hold a complete segment's candidate set; scanner I/O buffer size does ++not cap that allocation. Control task concurrency and physical segment size. ++ ++Successful exhaustion merges segment-search metrics into the existing statistics ++callback exactly once. New metrics include `scalar_segments_requested`, ++`scalar_segments_searched`, `scalar_segment_candidate_rows`, ++`scalar_segment_prepare_time`, `scalar_segment_search_time`, and ++`scalar_segment_fallback_*` reasons. `prepare_time` includes search time. Early ++release, cancellation and errors retain the existing callback contract: final ++statistics are not guaranteed. Metrics do not establish global task concurrency. +diff --git a/include/lance/lance.h b/include/lance/lance.h +index 8173ae5..cbd8330 100644 +--- a/include/lance/lance.h ++++ b/include/lance/lance.h +@@ -1858,6 +1858,30 @@ int32_t lance_scanner_set_index_segments( + size_t len + ); + ++/** ++ * Accelerate an ordinary scalar-filtered scan with one physical index segment. ++ * segment_uuid points to 16 UUID bytes in RFC 4122 order; NULL clears the setting. ++ * Must be configured before scanning. Requires explicit nonempty fragment_ids, ++ * which define BOTH the read and fallback domain, independently of the segment. ++ * Missing snapshot UUIDs / fragment IDs are errors. Extra segment coverage is ++ * excluded by fragment_ids; incomplete coverage falls back to a full filtered ++ * scan of those fragment_ids. Callers distributing work must assign disjoint ++ * fragment domains and separately include any unindexed data they wish to read. ++ * ++ * BTree/Bitmap searches use a necessary AND-conjunct of the full scanner filter ++ * on the selected logical index. All predicates are reapplied during candidate ++ * reads; other scalar indices are disabled. OR/NOT-only filters, overlays, ++ * fragment reuse, unsupported index types / result domains ++ * and missing coverage use the same domain without an index. No filter also ++ * falls back. LIMIT/OFFSET apply after the complete scanner filter, never to the ++ * unfiltered candidate set. Vector/FTS queries are rejected. ++ * ++ * UUID bytes are copied. Metadata and final option compatibility are validated ++ * when creating the stream. Index corruption or I/O failures remain errors. ++ */ ++int32_t lance_scanner_set_scalar_index_segment( ++ LanceScanner* scanner, const uint8_t* segment_uuid); ++ + /* ─── Full-text search (Phase 2) ─── */ + + /** +diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp +index c12c0c6..8e1c6cf 100644 +--- a/include/lance/lance.hpp ++++ b/include/lance/lance.hpp +@@ -1311,6 +1311,20 @@ class Scanner { + return *this; + } + ++ /// Restrict scalar candidate generation to one segment; fragment_ids is ++ /// required and defines the complete read/fallback domain. See lance.h. ++ Scanner& scalar_index_segment(const std::array& segment_uuid) { ++ if (lance_scanner_set_scalar_index_segment(handle_.get(), segment_uuid.data()) != 0) ++ check_error(); ++ return *this; ++ } ++ ++ Scanner& clear_scalar_index_segment() { ++ if (lance_scanner_set_scalar_index_segment(handle_.get(), nullptr) != 0) ++ check_error(); ++ return *this; ++ } ++ + /// Restrict scan to specific fragment IDs. + Scanner& fragment_ids(const uint64_t* ids, size_t len) { + if (lance_scanner_set_fragment_ids(handle_.get(), ids, len) != 0) +diff --git a/src/lib.rs b/src/lib.rs +index 8b212f5..c7ca4cf 100644 +--- a/src/lib.rs ++++ b/src/lib.rs +@@ -39,6 +39,7 @@ mod index_segment; + mod merge_insert; + mod restore; + pub mod runtime; ++mod scalar_segment; + mod scanner; + mod session; + pub mod stream_guard; +diff --git a/src/scalar_segment.rs b/src/scalar_segment.rs +new file mode 100644 +index 0000000..2ab1297 +--- /dev/null ++++ b/src/scalar_segment.rs +@@ -0,0 +1,224 @@ ++// SPDX-License-Identifier: Apache-2.0 ++// SPDX-FileCopyrightText: Copyright The Lance Authors ++ ++//! Segment-scoped candidate generation for ordinary scans. The explicit fragment ++//! list is the read domain, including on fallback; a segment is only an accelerator. ++ ++use std::collections::HashSet; ++use std::sync::Arc; ++use std::time::Instant; ++ ++use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; ++use lance::Dataset; ++use lance::dataset::scanner::{ ++ ExecutionStatsCallback, ExecutionSummaryCounts, RowAddrMask, Scanner, ++}; ++use lance::index::{DatasetIndexExt, DatasetIndexInternalExt}; ++use lance::io::exec::utils::IndexMetrics; ++use lance_core::{Error, Result}; ++use lance_datafusion::planner::Planner; ++use lance_datafusion::utils::MetricsExt; ++use lance_index::IndexType; ++use lance_index::scalar::SearchResult; ++use lance_index::scalar::expression::{PlannerIndexExt, ScalarIndexExpr, ScalarIndexSearch}; ++use uuid::Uuid; ++ ++pub(crate) struct PreparedScalarSegment { ++ pub dataset: Arc, ++ pub segment_uuid: Uuid, ++ pub fragment_ids: Vec, ++ pub callback: Option, ++} ++ ++fn invalid(message: impl Into) -> Error { ++ Error::invalid_input_source(message.into().into()) ++} ++ ++// Only descend through AND: a leaf below OR or NOT need not contain all matches ++// of the full expression. The original expression is always reapplied by reader. ++fn driver<'a>(expr: &'a ScalarIndexExpr, index_name: &str) -> Option<&'a ScalarIndexSearch> { ++ match expr { ++ ScalarIndexExpr::Query(search) if search.index_name == index_name => Some(search), ++ ScalarIndexExpr::And(lhs, rhs) => { ++ driver(lhs, index_name).or_else(|| driver(rhs, index_name)) ++ } ++ _ => None, ++ } ++} ++ ++impl PreparedScalarSegment { ++ pub async fn configure(self, mut reader: Scanner) -> Result { ++ // Never let either candidate reads or fallback re-enter a global index search. ++ reader.use_scalar_index(false); ++ let mut stats = ExecutionSummaryCounts::default(); ++ stats ++ .all_counts ++ .insert("scalar_segments_requested".into(), 1); ++ let plan_metrics = ExecutionPlanMetricsSet::new(); ++ let metrics = IndexMetrics::new(&plan_metrics, 0); ++ let started = Instant::now(); ++ let reason = self ++ .configure_candidates(&mut reader, &metrics, &mut stats) ++ .await?; ++ metrics.flush_io(); ++ stats.all_times.insert( ++ "scalar_segment_prepare_time".into(), ++ started.elapsed().as_nanos().min(usize::MAX as u128) as usize, ++ ); ++ if let Some(reason) = reason { ++ stats ++ .all_counts ++ .insert("scalar_segment_fallbacks".into(), 1); ++ stats ++ .all_counts ++ .insert(format!("scalar_segment_fallback_{reason}"), 1); ++ } ++ for (name, count) in plan_metrics.clone_inner().iter_counts() { ++ let name = name.as_ref(); ++ match name { ++ "iops" => stats.iops += count.value(), ++ "requests" => stats.requests += count.value(), ++ "bytes_read" => stats.bytes_read += count.value(), ++ "indices_loaded" => stats.indices_loaded += count.value(), ++ "parts_loaded" => stats.parts_loaded += count.value(), ++ "index_comparisons" => stats.index_comparisons += count.value(), ++ _ => *stats.all_counts.entry(name.to_string()).or_default() += count.value(), ++ } ++ } ++ if let Some(callback) = self.callback { ++ // Preserve the callback's once-per-successfully-exhausted-stream contract. ++ // Candidate work is not part of the underlying reader's plan metrics. ++ reader.scan_stats_callback(Arc::new(move |read| { ++ let mut combined = read.clone(); ++ combined.iops += stats.iops; ++ combined.requests += stats.requests; ++ combined.bytes_read += stats.bytes_read; ++ combined.indices_loaded += stats.indices_loaded; ++ combined.parts_loaded += stats.parts_loaded; ++ combined.index_comparisons += stats.index_comparisons; ++ for (name, value) in &stats.all_counts { ++ *combined.all_counts.entry(name.clone()).or_default() += value; ++ } ++ for (name, value) in &stats.all_times { ++ *combined.all_times.entry(name.clone()).or_default() += value; ++ } ++ callback(&combined); ++ })); ++ } ++ Ok(reader) ++ } ++ ++ async fn configure_candidates( ++ &self, ++ reader: &mut Scanner, ++ metrics: &IndexMetrics, ++ stats: &mut ExecutionSummaryCounts, ++ ) -> Result> { ++ let fragments = self.dataset.get_fragments(); ++ let visible: HashSet = fragments.iter().map(|f| f.id() as u64).collect(); ++ if self.fragment_ids.iter().any(|id| !visible.contains(id)) { ++ return Err(invalid( ++ "scalar segment fragment_ids contains a fragment absent from the dataset snapshot", ++ )); ++ } ++ let indices = self.dataset.load_indices().await?; ++ let index_meta = indices ++ .iter() ++ .find(|i| i.uuid == self.segment_uuid) ++ .ok_or_else(|| { ++ invalid(format!( ++ "scalar index segment {} is absent from the dataset snapshot", ++ self.segment_uuid ++ )) ++ })?; ++ let field_id = index_meta ++ .keyed_field() ++ .ok_or_else(|| invalid("scalar segment must index a single key field"))?; ++ let field = ++ self.dataset.schema().field_by_id(field_id).ok_or_else(|| { ++ invalid("scalar segment key field is absent from the dataset schema") ++ })?; ++ // Keep V1 to flat scalar fields. A dotted name is not sufficient to prove ++ // the field path of an evolved or nested schema. ++ if !self ++ .dataset ++ .schema() ++ .fields ++ .iter() ++ .any(|f| f.id == field.id) ++ { ++ return Ok(Some("nested_field")); ++ } ++ let scope: HashSet = self.fragment_ids.iter().copied().collect(); ++ let Some(coverage) = index_meta.fragment_bitmap.as_ref() else { ++ return Ok(Some("unknown_coverage")); ++ }; ++ if self ++ .fragment_ids ++ .iter() ++ .any(|id| u32::try_from(*id).map_or(true, |id| !coverage.contains(id))) ++ { ++ // Scan the ENTIRE explicit read domain, not just the covered part. ++ return Ok(Some("partial_coverage")); ++ } ++ if fragments ++ .iter() ++ .filter(|f| scope.contains(&(f.id() as u64))) ++ .any(|f| !f.metadata().overlays.is_empty() || f.metadata().physical_rows.is_none()) ++ { ++ return Ok(Some("fragment_state")); ++ } ++ // Fragment reuse can change the domain of an old segment. Until its ++ // coverage mapping is handled here, preserve correctness with a scoped scan. ++ if self.dataset.frag_reuse_index_uuid().await.is_some() { ++ return Ok(Some("fragment_reuse")); ++ } ++ let Some(filter) = reader.get_expr_filter()? else { ++ return Ok(Some("no_filter")); ++ }; ++ let planner = Planner::new(Arc::new(self.dataset.schema().into())); ++ let index_info = self.dataset.scalar_index_info().await?; ++ let filter_plan = planner.create_filter_plan(filter, &index_info, true)?; ++ let Some(search) = filter_plan ++ .index_query ++ .as_ref() ++ .and_then(|expr| driver(expr, &index_meta.name)) ++ else { ++ return Ok(Some("no_driver")); ++ }; ++ if search.column != field.name { ++ return Ok(Some("field_path")); ++ } ++ let index = self ++ .dataset ++ .open_scalar_index(&search.column, &self.segment_uuid, metrics) ++ .await?; ++ if !matches!(index.index_type(), IndexType::BTree | IndexType::Bitmap) { ++ return Ok(Some("index_type")); ++ } ++ // External masks use _rowid, not necessarily physical row addresses. ++ if index.results_are_row_addresses() && self.dataset.manifest.uses_stable_row_ids() { ++ return Ok(Some("row_id_domain")); ++ } ++ let started = Instant::now(); ++ let result = index.search(search.query.as_ref(), metrics).await?; ++ stats.all_times.insert( ++ "scalar_segment_search_time".into(), ++ started.elapsed().as_nanos().min(usize::MAX as u128) as usize, ++ ); ++ stats ++ .all_counts ++ .insert("scalar_segments_searched".into(), 1); ++ let SearchResult::Exact(rows) = result else { ++ return Ok(Some("inexact_result")); ++ }; ++ stats.all_counts.insert( ++ "scalar_segment_candidate_rows".into(), ++ rows.len().unwrap_or(0) as usize, ++ ); ++ // Do not truncate candidates at LIMIT. The reader evaluates the complete ++ // filter before applying its existing limit/offset operators. ++ reader.with_row_addr_prefilter(RowAddrMask::from_allowed(rows.selected_rows().clone())); ++ Ok(None) ++ } ++} +diff --git a/src/scanner.rs b/src/scanner.rs +index 4ceeb0e..ac3cff6 100644 +--- a/src/scanner.rs ++++ b/src/scanner.rs +@@ -39,6 +39,7 @@ use crate::fts_query::{ + }; + use crate::helpers; + use crate::runtime::{RT, block_on}; ++use crate::scalar_segment::PreparedScalarSegment; + use crate::stream_guard::GuardedReader; + + /// Data type tag for query vectors, mirroring the C enum `LanceDataType`. +@@ -108,6 +109,7 @@ pub struct LanceScanner { + include_deleted_rows: bool, + fragment_ids: Option>, + index_segments: Option>, ++ scalar_index_segment: Option, + nearest: Option, + nprobes: NprobesRange, + approx_mode: Option, +@@ -256,6 +258,7 @@ impl LanceScanner { + include_deleted_rows: false, + fragment_ids: None, + index_segments: None, ++ scalar_index_segment: None, + nearest: None, + nprobes: NprobesRange::default(), + approx_mode: None, +@@ -470,12 +473,37 @@ impl LanceScanner { + None + }; + self.apply_filter(&mut scanner)?; ++ let scalar_segment = if let Some(segment_uuid) = self.scalar_index_segment { ++ if self.nearest.is_some() ++ || self.fts_query.is_some() ++ || self.fts_context.is_some() ++ || self.index_segments.is_some() ++ || self.fts_index_segments.is_some() ++ { ++ return Err(lance_core::Error::invalid_input_source( ++ "scalar_index_segment requires an ordinary scan of live rows".into(), ++ )); ++ } ++ let fragment_ids = self.fragment_ids.as_ref().filter(|ids| !ids.is_empty()) ++ .ok_or_else(|| lance_core::Error::invalid_input_source( ++ "scalar_index_segment requires explicit nonempty fragment_ids for its read and fallback domain".into(), ++ ))?; ++ Some(PreparedScalarSegment { ++ dataset: Arc::clone(&self.dataset), ++ segment_uuid, ++ fragment_ids: fragment_ids.clone(), ++ callback: self.scan_statistics_callback.clone(), ++ }) ++ } else { ++ None ++ }; + if let Some(callback) = &self.scan_statistics_callback { + scanner.scan_stats_callback(callback.clone()); + } + Ok(PreparedScanner { + scanner, + distributed_fts, ++ scalar_segment, + }) + } + } +@@ -490,10 +518,18 @@ struct PreparedFtsExecution { + struct PreparedScanner { + scanner: lance::dataset::scanner::Scanner, + distributed_fts: Option, ++ scalar_segment: Option, + } + + impl PreparedScanner { + async fn try_into_stream(self) -> Result { ++ if let Some(scalar_segment) = self.scalar_segment { ++ return scalar_segment ++ .configure(self.scanner) ++ .await? ++ .try_into_stream() ++ .await; ++ } + let Some(distributed_fts) = self.distributed_fts else { + return self.scanner.try_into_stream().await; + }; +@@ -858,6 +894,31 @@ macro_rules! scanner_ffi_try { + }}; + } + ++/// Select one physical scalar index segment. NULL clears the selection. ++/// Requires explicit fragment_ids and an ordinary live-row scan. See the C header. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_scalar_index_segment( ++ scanner: *mut LanceScanner, ++ segment_uuid: *const u8, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, { ++ let scanner = unsafe { scanner.as_mut() } ++ .ok_or_else(|| lance_core::Error::invalid_input_source("scanner is NULL".into()))?; ++ scanner.ensure_scan_not_started("scalar_index_segment")?; ++ let segment = if segment_uuid.is_null() { ++ None ++ } else { ++ Some( ++ Uuid::from_slice(unsafe { std::slice::from_raw_parts(segment_uuid, 16) }) ++ .map_err(|e| lance_core::Error::invalid_input_source(e.into()))?, ++ ) ++ }; ++ scanner.scalar_index_segment = segment; ++ Ok(0) ++ }) ++} ++ + // --------------------------------------------------------------------------- + // Scanner lifecycle + builder + // --------------------------------------------------------------------------- +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index 3b3424b..a8a7eab 100644 +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -12513,3 +12513,252 @@ fn test_add_columns_stream_null_dataset_consumes_stream() { + assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); + assert_stream_consumed(&stream, &drop_count); + } ++ ++// Segment scans deliberately use an unprojected nullable key and a residual ++// predicate so a candidate LIMIT or loss of filter columns changes the answer. ++fn create_scalar_segment_fixture( ++ kind: lance_index::IndexType, ++ stable: bool, ++) -> (tempfile::TempDir, String, Vec<[u8; 16]>) { ++ use lance::dataset::WriteParams; ++ use lance::index::DatasetIndexExt; ++ use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams}; ++ let tmp = tempfile::tempdir().unwrap(); ++ let uri = tmp.path().join("segments").to_str().unwrap().to_owned(); ++ let uuids = lance_c::runtime::block_on(async { ++ let schema = Arc::new(Schema::new(vec![ ++ Field::new("id", DataType::Int32, false), ++ Field::new("key", DataType::Int32, true), ++ ])); ++ let batch = RecordBatch::try_new( ++ schema.clone(), ++ vec![ ++ Arc::new(Int32Array::from_iter_values(0..12)), ++ Arc::new(Int32Array::from( ++ (0..12) ++ .map(|id| if id % 4 == 0 { None } else { Some(id % 3) }) ++ .collect::>(), ++ )), ++ ], ++ ) ++ .unwrap(); ++ let mut ds = Dataset::write( ++ arrow::record_batch::RecordBatchIterator::new(vec![Ok(batch)], schema), ++ &uri, ++ Some(WriteParams { ++ max_rows_per_file: 4, ++ enable_stable_row_ids: stable, ++ ..Default::default() ++ }), ++ ) ++ .await ++ .unwrap(); ++ let params = ScalarIndexParams::for_builtin(if kind == lance_index::IndexType::Bitmap { ++ BuiltinIndexType::Bitmap ++ } else { ++ BuiltinIndexType::BTree ++ }); ++ let fragments = ds.get_fragments(); ++ assert_eq!(fragments.len(), 3); ++ let mut segments = Vec::new(); ++ for fragment in fragments.iter().take(2) { ++ segments.push( ++ ds.create_index_builder(&["key"], kind, ¶ms) ++ .name("key_idx".into()) ++ .fragments(vec![fragment.id() as u32]) ++ .execute_uncommitted() ++ .await ++ .unwrap(), ++ ); ++ } ++ let uuids = segments.iter().map(|s| *s.uuid.as_bytes()).collect(); ++ ds.commit_existing_index_segments("key_idx", "key", segments) ++ .await ++ .unwrap(); ++ uuids ++ }); ++ (tmp, uri, uuids) ++} ++ ++fn scalar_segment_ids( ++ uri: &str, ++ uuid: &[u8; 16], ++ fragments: &[u64], ++ filter: &str, ++ limit: Option, ++ offset: i64, ++) -> (Vec, CapturedScanStatistics) { ++ let uri = c_str(uri); ++ let filter = c_str(filter); ++ let id = c_str("id"); ++ let columns = [id.as_ptr(), ptr::null()]; ++ let mut captured = CapturedScanStatistics::default(); ++ let mut ids = Vec::new(); ++ unsafe { ++ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0); ++ assert!(!ds.is_null()); ++ let scanner = lance_scanner_new(ds, columns.as_ptr(), filter.as_ptr()); ++ assert!(!scanner.is_null()); ++ assert_eq!( ++ lance_scanner_set_fragment_ids(scanner, fragments.as_ptr(), fragments.len()), ++ 0 ++ ); ++ assert_eq!( ++ lance_scanner_set_scalar_index_segment(scanner, uuid.as_ptr()), ++ 0 ++ ); ++ if let Some(limit) = limit { ++ assert_eq!(lance_scanner_set_limit(scanner, limit), 0); ++ } ++ assert_eq!(lance_scanner_set_offset(scanner, offset), 0); ++ assert_eq!( ++ lance_scanner_set_statistics_callback( ++ scanner, ++ Some(capture_scan_statistics), ++ (&mut captured as *mut CapturedScanStatistics).cast() ++ ), ++ 0 ++ ); ++ let mut stream = FFI_ArrowArrayStream::empty(); ++ let rc = lance_scanner_to_arrow_stream(scanner, &mut stream); ++ assert_eq!( ++ rc, ++ 0, ++ "{}", ++ if rc != 0 { ++ take_last_error_message() ++ } else { ++ String::new() ++ } ++ ); ++ assert_eq!( ++ lance_scanner_set_scalar_index_segment(scanner, ptr::null()), ++ -1 ++ ); ++ { ++ let reader = ArrowArrayStreamReader::from_raw(&mut stream).unwrap(); ++ for batch in reader { ++ let batch = batch.unwrap(); ++ assert_eq!(batch.num_columns(), 1); ++ ids.extend( ++ batch ++ .column(0) ++ .as_any() ++ .downcast_ref::() ++ .unwrap() ++ .values() ++ .iter() ++ .copied(), ++ ); ++ } ++ } ++ lance_scanner_close(scanner); ++ lance_dataset_close(ds); ++ } ++ (ids, captured) ++} ++ ++#[test] ++fn test_scalar_segment_scope_residual_limit_and_unindexed_fallback() { ++ for kind in [ ++ lance_index::IndexType::BTree, ++ lance_index::IndexType::Bitmap, ++ ] { ++ let (_tmp, uri, uuids) = create_scalar_segment_fixture(kind, false); ++ let (ids, stats) = ++ scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", None, 0); ++ assert_eq!(ids, vec![2, 3]); ++ assert_eq!(stats.calls, 1); ++ assert!( ++ stats ++ .metrics ++ .iter() ++ .any(|(name, _, value)| name == "scalar_segments_searched" && *value == 1) ++ ); ++ let (ids, _) = ++ scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", Some(1), 1); ++ assert_eq!( ++ ids, ++ vec![3], ++ "offset and limit must apply after residual filtering" ++ ); ++ let (ids, _) = scalar_segment_ids(&uri, &uuids[1], &[1], "key >= 0 AND id >= 2", None, 0); ++ assert_eq!(ids, vec![5, 6, 7]); ++ let (ids, stats) = ++ scalar_segment_ids(&uri, &uuids[0], &[0, 2], "key >= 0 AND id >= 2", None, 0); ++ assert_eq!( ++ ids, ++ vec![2, 3, 9, 10, 11], ++ "partial coverage must not omit unindexed rows" ++ ); ++ assert!( ++ stats ++ .metrics ++ .iter() ++ .any(|(name, _, _)| name == "scalar_segment_fallback_partial_coverage") ++ ); ++ let (ids, stats) = scalar_segment_ids(&uri, &uuids[0], &[0], "key = 99 OR id = 0", None, 0); ++ assert_eq!( ++ ids, ++ vec![0], ++ "OR must not use just one branch as candidates" ++ ); ++ assert!( ++ stats ++ .metrics ++ .iter() ++ .any(|(name, _, _)| name == "scalar_segment_fallback_no_driver") ++ ); ++ let (ids, _) = scalar_segment_ids(&uri, &uuids[0], &[0], "key = 99", None, 0); ++ assert!(ids.is_empty()); ++ } ++} ++ ++#[test] ++fn test_scalar_segment_stable_row_ids_and_deletes() { ++ use lance::index::DatasetIndexExt; ++ let (_tmp, uri, uuids) = create_scalar_segment_fixture(lance_index::IndexType::BTree, true); ++ lance_c::runtime::block_on(async { ++ let mut ds = Dataset::open(&uri).await.unwrap(); ++ ds.delete("id = 2").await.unwrap(); ++ assert_eq!(ds.load_indices().await.unwrap().len(), 2); ++ }); ++ let (ids, _) = scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", None, 0); ++ assert_eq!(ids, vec![3]); ++} ++ ++#[test] ++fn test_scalar_segment_requires_explicit_domain_and_checks_uuid() { ++ let (_tmp, uri, uuids) = create_scalar_segment_fixture(lance_index::IndexType::BTree, false); ++ let uri = c_str(&uri); ++ let filter = c_str("key >= 0"); ++ unsafe { ++ assert_eq!( ++ lance_scanner_set_scalar_index_segment(ptr::null_mut(), ptr::null()), ++ -1 ++ ); ++ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0); ++ let scanner = lance_scanner_new(ds, ptr::null(), filter.as_ptr()); ++ assert_eq!( ++ lance_scanner_set_scalar_index_segment(scanner, uuids[0].as_ptr()), ++ 0 ++ ); ++ let mut batch = ptr::null_mut(); ++ assert_eq!(lance_scanner_next(scanner, &mut batch), -1); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ lance_scanner_close(scanner); ++ let scanner = lance_scanner_new(ds, ptr::null(), filter.as_ptr()); ++ assert_eq!( ++ lance_scanner_set_fragment_ids(scanner, [0u64].as_ptr(), 1), ++ 0 ++ ); ++ assert_eq!( ++ lance_scanner_set_scalar_index_segment(scanner, [0u8; 16].as_ptr()), ++ 0 ++ ); ++ assert_eq!(lance_scanner_next(scanner, &mut batch), -1); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ lance_scanner_close(scanner); ++ lance_dataset_close(ds); ++ } ++} + +From 17240674d739293a494c07dfe5aec92a97185363 Mon Sep 17 00:00:00 2001 +From: zhangstar333 +Date: Tue, 8 Sep 2026 23:17:23 +0800 +Subject: [PATCH 2/5] update + +--- + docs/scalar-segment-scans.md | 3 ++ + include/lance/lance.h | 4 +-- + src/scalar_segment.rs | 7 +++++ + tests/c_api_test.rs | 60 ++++++++++++++++++++++++++++++++++-- + 4 files changed, 70 insertions(+), 4 deletions(-) + +diff --git a/docs/scalar-segment-scans.md b/docs/scalar-segment-scans.md +index dc7c141..f72dbfc 100644 +--- a/docs/scalar-segment-scans.md ++++ b/docs/scalar-segment-scans.md +@@ -36,6 +36,9 @@ An unknown UUID, absent fragment or invalid option combination is an error. A + known segment with incomplete/unknown coverage, no suitable driver, unsupported + index type, nested key, overlays, fragment reuse, non-exact results or unsupported + row-ID domain falls back to a non-indexed scan of the entire explicit domain. ++Legacy (v1) storage also takes this fallback because ordinary scans cannot consume ++external row masks; it reports `scalar_segment_fallback_legacy_storage` without ++searching the index. + I/O and corruption errors are propagated, not converted to empty results or + successful fallback. + +diff --git a/include/lance/lance.h b/include/lance/lance.h +index cbd8330..ab15247 100644 +--- a/include/lance/lance.h ++++ b/include/lance/lance.h +@@ -1870,8 +1870,8 @@ int32_t lance_scanner_set_index_segments( + * + * BTree/Bitmap searches use a necessary AND-conjunct of the full scanner filter + * on the selected logical index. All predicates are reapplied during candidate +- * reads; other scalar indices are disabled. OR/NOT-only filters, overlays, +- * fragment reuse, unsupported index types / result domains ++ * reads; other scalar indices are disabled. Legacy storage, OR/NOT-only filters, ++ * overlays, fragment reuse, unsupported index types / result domains + * and missing coverage use the same domain without an index. No filter also + * falls back. LIMIT/OFFSET apply after the complete scanner filter, never to the + * unfiltered candidate set. Vector/FTS queries are rejected. +diff --git a/src/scalar_segment.rs b/src/scalar_segment.rs +index 2ab1297..5e121c9 100644 +--- a/src/scalar_segment.rs ++++ b/src/scalar_segment.rs +@@ -138,6 +138,13 @@ impl PreparedScalarSegment { + self.dataset.schema().field_by_id(field_id).ok_or_else(|| { + invalid("scalar segment key field is absent from the dataset schema") + })?; ++ // Match Lance's plain-scan external-mask restriction. Keep the scoped, ++ // full-filtered reader intact and avoid index work on legacy storage. ++ if self.dataset.manifest().data_storage_format.lance_file_format() ++ == lance_file::version::ConcreteFileVersion::V1 ++ { ++ return Ok(Some("legacy_storage")); ++ } + // Keep V1 to flat scalar fields. A dotted name is not sufficient to prove + // the field path of an evolved or nested schema. + if !self +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index a8a7eab..0cb998b 100644 +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -12519,6 +12519,15 @@ fn test_add_columns_stream_null_dataset_consumes_stream() { + fn create_scalar_segment_fixture( + kind: lance_index::IndexType, + stable: bool, ++) -> (tempfile::TempDir, String, Vec<[u8; 16]>) { ++ create_scalar_segment_fixture_with_options(kind, stable, None, &[&[0], &[1]]) ++} ++ ++fn create_scalar_segment_fixture_with_options( ++ kind: lance_index::IndexType, ++ stable: bool, ++ storage_version: Option, ++ segment_fragments: &[&[u32]], + ) -> (tempfile::TempDir, String, Vec<[u8; 16]>) { + use lance::dataset::WriteParams; + use lance::index::DatasetIndexExt; +@@ -12548,6 +12557,7 @@ fn create_scalar_segment_fixture( + Some(WriteParams { + max_rows_per_file: 4, + enable_stable_row_ids: stable, ++ data_storage_version: storage_version, + ..Default::default() + }), + ) +@@ -12561,11 +12571,11 @@ fn create_scalar_segment_fixture( + let fragments = ds.get_fragments(); + assert_eq!(fragments.len(), 3); + let mut segments = Vec::new(); +- for fragment in fragments.iter().take(2) { ++ for fragment_ids in segment_fragments { + segments.push( + ds.create_index_builder(&["key"], kind, ¶ms) + .name("key_idx".into()) +- .fragments(vec![fragment.id() as u32]) ++ .fragments(fragment_ids.to_vec()) + .execute_uncommitted() + .await + .unwrap(), +@@ -12714,6 +12724,52 @@ fn test_scalar_segment_scope_residual_limit_and_unindexed_fallback() { + } + } + ++#[test] ++fn test_scalar_segment_legacy_storage_falls_back() { ++ use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; ++ ++ // Three fragments, with one segment covering 0 and 1. Reading only fragment ++ // 0 must retain the full predicate and must not leak rows from fragment 1. ++ let (_tmp, uri, uuids) = create_scalar_segment_fixture_with_options( ++ lance_index::IndexType::BTree, ++ false, ++ Some(LanceFileVersion::Legacy), ++ &[&[0, 1]], ++ ); ++ assert_eq!(uuids.len(), 1); ++ lance_c::runtime::block_on(async { ++ let ds = Dataset::open(&uri).await.unwrap(); ++ assert_eq!( ++ ds.manifest().data_storage_format.lance_file_format(), ++ ConcreteFileVersion::V1 ++ ); ++ }); ++ ++ let (ids, stats) = ++ scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", None, 0); ++ assert_eq!(ids, vec![2, 3]); ++ assert_eq!(stats.calls, 1); ++ assert_eq!(stats.indices_loaded, 0); ++ assert_eq!(stats.index_comparisons, 0); ++ assert!(stats.metrics.iter().any(|(name, _, value)| { ++ name == "scalar_segment_fallback_legacy_storage" && *value == 1 ++ })); ++ assert!( ++ !stats ++ .metrics ++ .iter() ++ .any(|(name, _, value)| name == "scalar_segments_searched" && *value != 0) ++ ); ++ ++ let (ids, _) = ++ scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", Some(1), 1); ++ assert_eq!( ++ ids, ++ vec![3], ++ "fallback must retain LIMIT/OFFSET after filtering" ++ ); ++} ++ + #[test] + fn test_scalar_segment_stable_row_ids_and_deletes() { + use lance::index::DatasetIndexExt; + +From f9ce263e4e32540c26afa6ccd18021ca81c090f1 Mon Sep 17 00:00:00 2001 +From: zhangstar333 +Date: Tue, 8 Sep 2026 23:21:02 +0800 +Subject: [PATCH 3/5] update + +--- + src/scalar_segment.rs | 6 +++++- + tests/c_api_test.rs | 6 ++---- + 2 files changed, 7 insertions(+), 5 deletions(-) + +diff --git a/src/scalar_segment.rs b/src/scalar_segment.rs +index 5e121c9..01df58b 100644 +--- a/src/scalar_segment.rs ++++ b/src/scalar_segment.rs +@@ -140,7 +140,11 @@ impl PreparedScalarSegment { + })?; + // Match Lance's plain-scan external-mask restriction. Keep the scoped, + // full-filtered reader intact and avoid index work on legacy storage. +- if self.dataset.manifest().data_storage_format.lance_file_format() ++ if self ++ .dataset ++ .manifest() ++ .data_storage_format ++ .lance_file_format() + == lance_file::version::ConcreteFileVersion::V1 + { + return Ok(Some("legacy_storage")); +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index 0cb998b..efd4284 100644 +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -12745,8 +12745,7 @@ fn test_scalar_segment_legacy_storage_falls_back() { + ); + }); + +- let (ids, stats) = +- scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", None, 0); ++ let (ids, stats) = scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", None, 0); + assert_eq!(ids, vec![2, 3]); + assert_eq!(stats.calls, 1); + assert_eq!(stats.indices_loaded, 0); +@@ -12761,8 +12760,7 @@ fn test_scalar_segment_legacy_storage_falls_back() { + .any(|(name, _, value)| name == "scalar_segments_searched" && *value != 0) + ); + +- let (ids, _) = +- scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", Some(1), 1); ++ let (ids, _) = scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", Some(1), 1); + assert_eq!( + ids, + vec![3], + +From 221d8800f56790ab8b48b12f3aaaf9da06359380 Mon Sep 17 00:00:00 2001 +From: zhangstar333 +Date: Wed, 9 Sep 2026 12:59:08 +0800 +Subject: [PATCH 4/5] add LabelList + +--- + docs/scalar-segment-scans.md | 12 ++- + include/lance/lance.h | 8 +- + include/lance/lance.hpp | 4 +- + src/scalar_segment.rs | 7 +- + tests/c_api_test.rs | 190 ++++++++++++++++++++++++++++++++--- + 5 files changed, 200 insertions(+), 21 deletions(-) + +diff --git a/docs/scalar-segment-scans.md b/docs/scalar-segment-scans.md +index f72dbfc..72f6188 100644 +--- a/docs/scalar-segment-scans.md ++++ b/docs/scalar-segment-scans.md +@@ -1,10 +1,18 @@ + # Scalar index segment scans + +-An ordinary scanner can use one physical BTree/Bitmap segment to generate +-candidates, then read those candidates with the complete scanner filter. This ++An ordinary scanner can use one physical BTree, Bitmap, or LabelList segment to ++generate candidates, then read them with the complete scanner filter. This + does not run a global search of the other segments of the logical index. It does + not subdivide a physical segment or make its own index search incremental. + ++LabelList supports indexed array membership predicates. Every candidate search ++must return `SearchResult::Exact`. ++LabelList query values should match the array element type, for example ++`array_contains(int32_labels, CAST(42 AS INT))`; a cast on the indexed column ++can prevent the planner from finding an index driver and cause fallback. ++`AtMost` and `AtLeast` results still fall back; this mode does not enable FMIndex, ++NGram, BloomFilter, ZoneMap, or Inverted indices. ++ + ## Configuring a task + + Open a fixed dataset version. Select the physical index UUID from that version's +diff --git a/include/lance/lance.h b/include/lance/lance.h +index ab15247..b5c6902 100644 +--- a/include/lance/lance.h ++++ b/include/lance/lance.h +@@ -1868,9 +1868,11 @@ int32_t lance_scanner_set_index_segments( + * scan of those fragment_ids. Callers distributing work must assign disjoint + * fragment domains and separately include any unindexed data they wish to read. + * +- * BTree/Bitmap searches use a necessary AND-conjunct of the full scanner filter +- * on the selected logical index. All predicates are reapplied during candidate +- * reads; other scalar indices are disabled. Legacy storage, OR/NOT-only filters, ++ * BTree/Bitmap/LabelList searches use a necessary AND-conjunct of the ++ * full scanner filter on the selected logical index and require an Exact result. ++ * AtMost/AtLeast results fall back to a full filtered scan of fragment_ids. ++ * All predicates are reapplied during candidate reads; other scalar indices ++ * are disabled. Legacy storage, OR/NOT-only filters, + * overlays, fragment reuse, unsupported index types / result domains + * and missing coverage use the same domain without an index. No filter also + * falls back. LIMIT/OFFSET apply after the complete scanner filter, never to the +diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp +index 8e1c6cf..ebb8141 100644 +--- a/include/lance/lance.hpp ++++ b/include/lance/lance.hpp +@@ -1311,8 +1311,8 @@ class Scanner { + return *this; + } + +- /// Restrict scalar candidate generation to one segment; fragment_ids is +- /// required and defines the complete read/fallback domain. See lance.h. ++ /// Generate exact candidates from one BTree/Bitmap/LabelList segment. ++ /// fragment_ids is required and defines the complete read/fallback domain. See lance.h. + Scanner& scalar_index_segment(const std::array& segment_uuid) { + if (lance_scanner_set_scalar_index_segment(handle_.get(), segment_uuid.data()) != 0) + check_error(); +diff --git a/src/scalar_segment.rs b/src/scalar_segment.rs +index 01df58b..1faf500 100644 +--- a/src/scalar_segment.rs ++++ b/src/scalar_segment.rs +@@ -204,7 +204,12 @@ impl PreparedScalarSegment { + .dataset + .open_scalar_index(&search.column, &self.segment_uuid, metrics) + .await?; +- if !matches!(index.index_type(), IndexType::BTree | IndexType::Bitmap) { ++ // These implementations can return exact candidates. Keep the runtime ++ // Exact check below: a type alone is not a guarantee for every query. ++ if !matches!( ++ index.index_type(), ++ IndexType::BTree | IndexType::Bitmap | IndexType::LabelList ++ ) { + return Ok(Some("index_type")); + } + // External masks use _rowid, not necessarily physical row addresses. +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index efd4284..3322d42 100644 +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -12528,6 +12528,21 @@ fn create_scalar_segment_fixture_with_options( + stable: bool, + storage_version: Option, + segment_fragments: &[&[u32]], ++) -> (tempfile::TempDir, String, Vec<[u8; 16]>) { ++ let key = Arc::new(Int32Array::from( ++ (0..12) ++ .map(|id| if id % 4 == 0 { None } else { Some(id % 3) }) ++ .collect::>(), ++ )); ++ create_scalar_segment_fixture_from_key(kind, stable, storage_version, segment_fragments, key) ++} ++ ++fn create_scalar_segment_fixture_from_key( ++ kind: lance_index::IndexType, ++ stable: bool, ++ storage_version: Option, ++ segment_fragments: &[&[u32]], ++ key: arrow_array::ArrayRef, + ) -> (tempfile::TempDir, String, Vec<[u8; 16]>) { + use lance::dataset::WriteParams; + use lance::index::DatasetIndexExt; +@@ -12537,17 +12552,14 @@ fn create_scalar_segment_fixture_with_options( + let uuids = lance_c::runtime::block_on(async { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), +- Field::new("key", DataType::Int32, true), ++ Field::new("key", key.data_type().clone(), true), + ])); ++ let row_count = key.len(); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ +- Arc::new(Int32Array::from_iter_values(0..12)), +- Arc::new(Int32Array::from( +- (0..12) +- .map(|id| if id % 4 == 0 { None } else { Some(id % 3) }) +- .collect::>(), +- )), ++ Arc::new(Int32Array::from_iter_values(0..row_count as i32)), ++ key, + ], + ) + .unwrap(); +@@ -12563,13 +12575,9 @@ fn create_scalar_segment_fixture_with_options( + ) + .await + .unwrap(); +- let params = ScalarIndexParams::for_builtin(if kind == lance_index::IndexType::Bitmap { +- BuiltinIndexType::Bitmap +- } else { +- BuiltinIndexType::BTree +- }); ++ let params = ScalarIndexParams::for_builtin(BuiltinIndexType::try_from(kind).unwrap()); + let fragments = ds.get_fragments(); +- assert_eq!(fragments.len(), 3); ++ assert_eq!(fragments.len(), row_count.div_ceil(4)); + let mut segments = Vec::new(); + for fragment_ids in segment_fragments { + segments.push( +@@ -12724,6 +12732,162 @@ fn test_scalar_segment_scope_residual_limit_and_unindexed_fallback() { + } + } + ++#[test] ++fn test_scalar_segment_label_list_exact_candidates() { ++ use arrow_array::builder::{Int32Builder, ListBuilder}; ++ use lance::index::DatasetIndexExt; ++ use lance_index::IndexType; ++ ++ for stable in [false, true] { ++ let mut lists = ListBuilder::new(Int32Builder::new()); ++ for row in 0..16 { ++ match row { ++ 0 | 9 | 13 => lists.append(false), ++ 1 | 10 | 14 => lists.append(true), ++ 4 => { ++ lists.values().append_value(7); ++ lists.append(true); ++ } ++ _ => { ++ lists.values().append_value(42); ++ if row == 3 || row == 11 || row == 15 { ++ lists.values().append_value(7); ++ } ++ if row == 6 { ++ lists.values().append_null(); ++ } ++ lists.append(true); ++ } ++ } ++ } ++ // S0 covers fragments 0 and 1, S1 covers 2, and 3 is unindexed. ++ let (_tmp, uri, uuids) = create_scalar_segment_fixture_from_key( ++ IndexType::LabelList, ++ stable, ++ None, ++ &[&[0, 1], &[2]], ++ Arc::new(lists.finish()), ++ ); ++ let predicate = "array_contains(key, CAST(42 AS INT))"; ++ let filter = format!("{predicate} AND id >= 3"); ++ for (fragments, expected) in [(vec![0, 1], vec![3, 5, 6, 7]), (vec![0], vec![3])] { ++ let (ids, stats) = scalar_segment_ids(&uri, &uuids[0], &fragments, &filter, None, 0); ++ assert_eq!(ids, expected, "stable={stable}"); ++ assert_eq!(stats.calls, 1); ++ assert!( ++ stats ++ .metrics ++ .iter() ++ .any(|(name, _, value)| { name == "scalar_segments_searched" && *value == 1 }), ++ "stable={stable}, metrics={:?}", ++ stats.metrics ++ ); ++ assert!( ++ !stats ++ .metrics ++ .iter() ++ .any(|(name, _, value)| { name == "scalar_segment_fallbacks" && *value != 0 }) ++ ); ++ } ++ let (ids, _) = scalar_segment_ids(&uri, &uuids[0], &[0, 1], &filter, Some(1), 1); ++ assert_eq!(ids, vec![5], "limit/offset must follow the residual filter"); ++ let (ids, _) = scalar_segment_ids(&uri, &uuids[1], &[2], &filter, None, 0); ++ assert_eq!(ids, vec![8, 11]); ++ let (ids, stats) = scalar_segment_ids(&uri, &uuids[0], &[0, 3], &filter, None, 0); ++ assert_eq!(ids, vec![3, 12, 15]); ++ assert!(stats.metrics.iter().any(|(name, _, value)| { ++ name == "scalar_segment_fallback_partial_coverage" && *value == 1 ++ })); ++ let (ids, stats) = scalar_segment_ids( ++ &uri, ++ &uuids[0], ++ &[0], ++ &format!("{predicate} OR id = 0"), ++ None, ++ 0, ++ ); ++ assert_eq!(ids, vec![0, 2, 3]); ++ assert!(stats.metrics.iter().any(|(name, _, value)| { ++ name == "scalar_segment_fallback_no_driver" && *value == 1 ++ })); ++ ++ for (predicate, expected) in [ ++ ( ++ "array_has_all(key, [CAST(42 AS INT), CAST(7 AS INT)])", ++ vec![3], ++ ), ++ ( ++ "array_has_any(key, [CAST(42 AS INT), CAST(99 AS INT)])", ++ vec![2, 3, 5, 6, 7], ++ ), ++ ("array_contains(key, CAST(99 AS INT))", vec![]), ++ ("array_contains(key, CAST(NULL AS INT))", vec![]), ++ ("array_has_any(key, [])", vec![]), ++ ] { ++ let (ids, _) = scalar_segment_ids(&uri, &uuids[0], &[0, 1], predicate, None, 0); ++ assert_eq!(ids, expected, "{predicate}, stable={stable}"); ++ } ++ // An untyped integer literal casts this Int32 list to Int64. Such ++ // a column expression must retain the scan fallback. ++ let (ids, stats) = ++ scalar_segment_ids(&uri, &uuids[0], &[0, 1], "array_contains(key, 42)", None, 0); ++ assert_eq!(ids, vec![2, 3, 5, 6, 7]); ++ assert!(stats.metrics.iter().any(|(name, _, value)| { ++ name == "scalar_segment_fallback_no_driver" && *value == 1 ++ })); ++ ++ lance_c::runtime::block_on(async { ++ let mut ds = Dataset::open(&uri).await.unwrap(); ++ ds.delete("id = 3").await.unwrap(); ++ assert_eq!(ds.load_indices().await.unwrap().len(), 2); ++ }); ++ let (ids, _) = scalar_segment_ids(&uri, &uuids[0], &[0, 1], &filter, None, 0); ++ assert_eq!(ids, vec![5, 6, 7]); ++ } ++} ++ ++#[test] ++fn test_scalar_segment_text_indices_still_fall_back() { ++ for kind in [lance_index::IndexType::Fm, lance_index::IndexType::NGram] { ++ for stable in [false, true] { ++ let key = Arc::new(StringArray::from(vec![ ++ Some("needle"), ++ None, ++ Some(""), ++ Some("other"), ++ Some("needle"), ++ Some("other"), ++ Some(""), ++ None, ++ ])); ++ let (_tmp, uri, uuids) = ++ create_scalar_segment_fixture_from_key(kind, stable, None, &[&[0, 1]], key); ++ for (predicate, expected) in [ ++ ("contains(key, 'needle')", vec![0]), ++ ("contains(key, '')", vec![0, 2, 3]), ++ ] { ++ let (ids, stats) = scalar_segment_ids(&uri, &uuids[0], &[0], predicate, None, 0); ++ assert_eq!(ids, expected, "{kind:?}, stable={stable}, {predicate}"); ++ assert!( ++ stats.metrics.iter().any(|(name, _, value)| { ++ name == "scalar_segment_fallbacks" && *value == 1 ++ }) ++ ); ++ if predicate == "contains(key, 'needle')" { ++ assert!(stats.metrics.iter().any(|(name, _, value)| { ++ name == "scalar_segment_fallback_index_type" && *value == 1 ++ })); ++ } ++ assert!( ++ !stats.metrics.iter().any(|(name, _, value)| { ++ name == "scalar_segments_searched" && *value != 0 ++ }) ++ ); ++ } ++ } ++ } ++} ++ + #[test] + fn test_scalar_segment_legacy_storage_falls_back() { + use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; + +From 30dda06a1cc6bad37d04ed207115409ccef91715 Mon Sep 17 00:00:00 2001 +From: zhangstar333 +Date: Wed, 9 Sep 2026 13:27:54 +0800 +Subject: [PATCH 5/5] update + +--- + docs/scalar-segment-scans.md | 55 +++++++++-- + include/lance/lance.h | 16 ++- + include/lance/lance.hpp | 5 + + src/scalar_segment.rs | 12 ++- + src/scanner.rs | 13 ++- + tests/c_api_test.rs | 187 +++++++++++++++++++++++++++++++++++ + 6 files changed, 272 insertions(+), 16 deletions(-) + +diff --git a/docs/scalar-segment-scans.md b/docs/scalar-segment-scans.md +index 72f6188..efd0498 100644 +--- a/docs/scalar-segment-scans.md ++++ b/docs/scalar-segment-scans.md +@@ -27,11 +27,19 @@ lance_scanner_set_limit(scanner, 20000); + /* The scanner-owning thread calls lance_scanner_next as usual. */ + ``` + ++The segment setter copies the UUID; passing NULL clears it. Final option ++compatibility and snapshot metadata are checked when preparing the stream, not ++by the setter. Configure all options before the first `next`, Arrow stream ++export, or asynchronous scan. A failed preparation also freezes the options; ++create a new scanner to retry with different settings. ++ + SQL, Substrait and additional SQL filters keep their existing precedence and AND + composition. The caller does not supply a separate driver predicate: Lance-C + uses the typed filter planner and selects a necessary indexed leaf belonging to + the requested logical index. It only descends through AND, never through OR or +-NOT. It then searches the selected UUID and applies the complete filter while ++NOT, and chooses the first matching leaf in the planner's expression tree; ++this is not a selectivity-based choice or a guarantee of SQL text order. ++It then searches the selected UUID and applies the complete filter while + reading candidates with automatic scalar-index planning disabled. + + Each task's fragment IDs define its result domain, including on fallback. A +@@ -40,9 +48,16 @@ scan. Unindexed fragments need their own tasks, or an explicit domain including + them (which causes that task to use fallback). Merely listing indexed segments + does not include appended, unindexed data automatically. + +-An unknown UUID, absent fragment or invalid option combination is an error. A +-known segment with incomplete/unknown coverage, no suitable driver, unsupported +-index type, nested key, overlays, fragment reuse, non-exact results or unsupported ++`use_scalar_index=false` disables segment search regardless of setter order. ++The scanner validates the selected snapshot UUID and fragment domain, then scans ++that domain with the full filter and LIMIT/OFFSET without opening the index or ++generating candidates. It reports `scalar_segment_fallback_disabled`. ++ ++An unknown UUID, absent fragment, invalid option combination or segment metadata ++without one valid schema key field is an error. A known segment with ++incomplete/unknown coverage, no suitable driver, unsupported ++index type, nested key, overlays, unknown physical row counts, fragment reuse, ++non-exact results or unsupported + row-ID domain falls back to a non-indexed scan of the entire explicit domain. + Legacy (v1) storage also takes this fallback because ordinary scans cannot consume + external row masks; it reports `scalar_segment_fallback_legacy_storage` without +@@ -50,8 +65,15 @@ searching the index. + I/O and corruption errors are propagated, not converted to empty results or + successful fallback. + +-The first implementation supports live-row ordinary scans and cannot be combined +-with vector/FTS queries. Physical row-address ++The first implementation supports live-row ordinary scans and rejects vector/FTS ++queries and `include_deleted_rows=true` at stream creation, even when ++`use_scalar_index=false`. An index built after a delete does not contain the ++tombstoned rows, so even exact segment candidates ++cannot satisfy a scan that includes deleted rows. To read those rows, clear the ++segment setting and use an ordinary scan with `with_row_id=true`, ++`include_deleted_rows=true`, and `use_scalar_index=false`. ++Fragments removed from the current snapshot are not scanned by this option. ++Physical row-address + results on stable-row-ID datasets currently fall back; results already expressed + in the correct row-ID domain use the candidate path. Deletes and all remaining + predicates are handled by the ordinary reader. No candidate-count limit is +@@ -84,6 +106,23 @@ Successful exhaustion merges segment-search metrics into the existing statistics + callback exactly once. New metrics include `scalar_segments_requested`, + `scalar_segments_searched`, `scalar_segment_candidate_rows`, + `scalar_segment_prepare_time`, `scalar_segment_search_time`, and +-`scalar_segment_fallback_*` reasons. `prepare_time` includes search time. Early +-release, cancellation and errors retain the existing callback contract: final ++`scalar_segment_fallbacks` plus `scalar_segment_fallback_*` reasons. ++`scalar_segment_prepare_time` includes search time. Metrics describe each ++successfully exhausted stream, including separately exported streams: ++ ++- `scalar_segments_requested` is 1 even on fallback. ++- `scalar_segments_searched` is 1 after a completed index search, including one ++ whose inexact result causes fallback. A fallback can therefore include index work. ++- `scalar_segment_candidate_rows` counts TRUE rows in the exact segment result before fragment ++ restriction, residual filtering, deletion handling and LIMIT/OFFSET. It is not ++ the output or physical-read row count; a result without a known cardinality is ++ reported as 0. The reader's mask may also include NULL candidates that the full ++ filter subsequently discards. ++- A fallback records one reason, the first eligibility check that fails. Disabled ++ scalar indices and legacy storage bypass index opening and search after snapshot ++ validation. Other fallback reasons may be found after opening or searching an index. ++- Search and candidate metrics may be absent when their stage did not execute; ++ consumers should treat absent counts as 0. ++ ++Early release, cancellation and errors retain the existing callback contract: final + statistics are not guaranteed. Metrics do not establish global task concurrency. +diff --git a/include/lance/lance.h b/include/lance/lance.h +index b5c6902..ed5bb6c 100644 +--- a/include/lance/lance.h ++++ b/include/lance/lance.h +@@ -1015,7 +1015,9 @@ int32_t lance_scanner_set_scan_in_order(LanceScanner* scanner, bool scan_in_orde + * Configure whether scalar indices may be used to optimize filters. + * + * Scalar indices are enabled by default. Disable this to force filter +- * evaluation without scalar indices. This setting is independent of ++ * evaluation without scalar indices, including an explicitly selected scalar ++ * segment (which falls back to a scan of its explicit fragment_ids). ++ * This setting is independent of + * `lance_scanner_set_use_index`, which controls vector ANN index usage. + * Must be set before scanning starts. + */ +@@ -1051,7 +1053,11 @@ int32_t lance_scanner_with_row_address(LanceScanner* scanner, bool enable); + + /** + * Configure whether deleted rows still present in storage are returned. +- * Deleted rows have a NULL `_rowid`; callers should also enable row IDs. ++ * Requires with_row_id=true; deleted rows have a NULL `_rowid`. ++ * For filtered scans, also set use_scalar_index=false: indices built after a ++ * deletion may omit tombstoned rows. Incompatible with scalar_index_segment, ++ * even when scalar indices are disabled. ++ * Fragments removed from the current snapshot are not scanned. + * Must be set before scanning starts. + */ + int32_t lance_scanner_set_include_deleted_rows( +@@ -1867,16 +1873,20 @@ int32_t lance_scanner_set_index_segments( + * excluded by fragment_ids; incomplete coverage falls back to a full filtered + * scan of those fragment_ids. Callers distributing work must assign disjoint + * fragment domains and separately include any unindexed data they wish to read. ++ * The segment metadata must identify one key field present in the schema. + * + * BTree/Bitmap/LabelList searches use a necessary AND-conjunct of the + * full scanner filter on the selected logical index and require an Exact result. ++ * use_scalar_index=false skips segment search and uses the scoped fallback; ++ * snapshot UUID and fragment validation still applies. + * AtMost/AtLeast results fall back to a full filtered scan of fragment_ids. + * All predicates are reapplied during candidate reads; other scalar indices + * are disabled. Legacy storage, OR/NOT-only filters, + * overlays, fragment reuse, unsupported index types / result domains + * and missing coverage use the same domain without an index. No filter also + * falls back. LIMIT/OFFSET apply after the complete scanner filter, never to the +- * unfiltered candidate set. Vector/FTS queries are rejected. ++ * unfiltered candidate set. Vector/FTS queries and include_deleted_rows=true ++ * are rejected even when use_scalar_index=false; segment mode is live-row-only. + * + * UUID bytes are copied. Metadata and final option compatibility are validated + * when creating the stream. Index corruption or I/O failures remain errors. +diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp +index ebb8141..d96cf9f 100644 +--- a/include/lance/lance.hpp ++++ b/include/lance/lance.hpp +@@ -1270,6 +1270,7 @@ class Scanner { + } + + /// Configure whether scalar indices may be used to optimize filters. ++ /// False also disables explicit scalar segment search, retaining its fragment domain. + Scanner& use_scalar_index(bool enable = true) { + if (lance_scanner_set_use_scalar_index(handle_.get(), enable) != 0) + check_error(); +@@ -1305,6 +1306,8 @@ class Scanner { + } + + /// Configure whether deleted rows still present in storage are returned. ++ /// Requires with_row_id(true); use_scalar_index(false) is needed for filtered scans. ++ /// Incompatible with scalar_index_segment. See lance.h. + Scanner& include_deleted_rows(bool include_deleted_rows = true) { + if (lance_scanner_set_include_deleted_rows(handle_.get(), include_deleted_rows) != 0) + check_error(); +@@ -1313,6 +1316,8 @@ class Scanner { + + /// Generate exact candidates from one BTree/Bitmap/LabelList segment. + /// fragment_ids is required and defines the complete read/fallback domain. See lance.h. ++ /// Requires live rows only: include_deleted_rows(true) is rejected at stream creation. ++ /// use_scalar_index(false) selects the scoped fallback without searching the segment. + Scanner& scalar_index_segment(const std::array& segment_uuid) { + if (lance_scanner_set_scalar_index_segment(handle_.get(), segment_uuid.data()) != 0) + check_error(); +diff --git a/src/scalar_segment.rs b/src/scalar_segment.rs +index 1faf500..747dac6 100644 +--- a/src/scalar_segment.rs ++++ b/src/scalar_segment.rs +@@ -27,6 +27,7 @@ pub(crate) struct PreparedScalarSegment { + pub dataset: Arc, + pub segment_uuid: Uuid, + pub fragment_ids: Vec, ++ pub use_scalar_index: bool, + pub callback: Option, + } + +@@ -138,6 +139,11 @@ impl PreparedScalarSegment { + self.dataset.schema().field_by_id(field_id).ok_or_else(|| { + invalid("scalar segment key field is absent from the dataset schema") + })?; ++ // Explicitly disabling scalar indices also disables this accelerator. ++ // Keep snapshot validation above, but do not plan, open or search an index. ++ if !self.use_scalar_index { ++ return Ok(Some("disabled")); ++ } + // Match Lance's plain-scan external-mask restriction. Keep the scoped, + // full-filtered reader intact and avoid index work on legacy storage. + if self +@@ -149,8 +155,8 @@ impl PreparedScalarSegment { + { + return Ok(Some("legacy_storage")); + } +- // Keep V1 to flat scalar fields. A dotted name is not sufficient to prove +- // the field path of an evolved or nested schema. ++ // Keep this implementation to flat scalar fields. A dotted name cannot ++ // prove the field path of an evolved or nested schema. + if !self + .dataset + .schema() +@@ -234,6 +240,8 @@ impl PreparedScalarSegment { + ); + // Do not truncate candidates at LIMIT. The reader evaluates the complete + // filter before applying its existing limit/offset operators. ++ // The raw selected bitmap can overlap NULL rows; the full filter removes ++ // those as well. The metric above counts semantic TRUE rows, not mask size. + reader.with_row_addr_prefilter(RowAddrMask::from_allowed(rows.selected_rows().clone())); + Ok(None) + } +diff --git a/src/scanner.rs b/src/scanner.rs +index ac3cff6..bda554d 100644 +--- a/src/scanner.rs ++++ b/src/scanner.rs +@@ -479,9 +479,10 @@ impl LanceScanner { + || self.fts_context.is_some() + || self.index_segments.is_some() + || self.fts_index_segments.is_some() ++ || self.include_deleted_rows + { + return Err(lance_core::Error::invalid_input_source( +- "scalar_index_segment requires an ordinary scan of live rows".into(), ++ "scalar_index_segment requires an ordinary scan of live rows; vector/FTS queries and include_deleted_rows=true are unsupported".into(), + )); + } + let fragment_ids = self.fragment_ids.as_ref().filter(|ids| !ids.is_empty()) +@@ -492,6 +493,7 @@ impl LanceScanner { + dataset: Arc::clone(&self.dataset), + segment_uuid, + fragment_ids: fragment_ids.clone(), ++ use_scalar_index: self.use_scalar_index.unwrap_or(true), + callback: self.scan_statistics_callback.clone(), + }) + } else { +@@ -896,6 +898,8 @@ macro_rules! scanner_ffi_try { + + /// Select one physical scalar index segment. NULL clears the selection. + /// Requires explicit fragment_ids and an ordinary live-row scan. See the C header. ++/// include_deleted_rows=true is rejected when preparing the scan, even if ++/// use_scalar_index=false selects the scoped non-indexed fallback. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn lance_scanner_set_scalar_index_segment( + scanner: *mut LanceScanner, +@@ -1239,7 +1243,8 @@ unsafe fn scanner_set_scan_in_order_inner( + /// Configure whether scalar indices may be used to optimize filters. + /// + /// Scalar indices are enabled by default in Lance. Must be set before the scan +-/// starts. ++/// starts. False also disables explicit scalar segment search while preserving ++/// the configured fragment domain and snapshot validation. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn lance_scanner_set_use_scalar_index( + scanner: *mut LanceScanner, +@@ -1381,7 +1386,9 @@ unsafe fn scanner_with_row_address_inner(scanner: *mut LanceScanner, enable: boo + + /// Configure whether deleted rows still present in storage are returned. + /// +-/// Deleted rows have a NULL `_rowid`, so callers should also enable row IDs. ++/// Requires with_row_id=true; deleted rows have a NULL `_rowid`. ++/// Filtered scans also need use_scalar_index=false because indices may omit ++/// tombstoned rows. Incompatible with scalar_index_segment. + /// Must be set before the scan starts. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn lance_scanner_set_include_deleted_rows( +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index 3322d42..513f8d0 100644 +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -12945,6 +12945,193 @@ fn test_scalar_segment_stable_row_ids_and_deletes() { + assert_eq!(ids, vec![3]); + } + ++#[test] ++fn test_scalar_segment_honors_use_scalar_index_false() { ++ let (_tmp, uri, uuids) = create_scalar_segment_fixture(lance_index::IndexType::BTree, false); ++ let (ids, stats) = scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0", None, 0); ++ assert_eq!(ids, vec![1, 2, 3]); ++ assert!( ++ stats ++ .metrics ++ .iter() ++ .any(|(name, _, value)| name == "scalar_segments_searched" && *value == 1) ++ ); ++ ++ let uri = c_str(&uri); ++ unsafe { ++ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0); ++ assert!(!ds.is_null()); ++ for disable_first in [false, true] { ++ for (fragments, filter, limit, offset, expected) in [ ++ (vec![0u64], "key >= 0", None, 0, vec![1, 2, 3]), ++ (vec![0, 2], "key >= 0 AND id >= 2", Some(2), 1, vec![3, 9]), ++ ] { ++ let filter = c_str(filter); ++ let scanner = lance_scanner_new(ds, ptr::null(), filter.as_ptr()); ++ assert!(!scanner.is_null()); ++ assert_eq!( ++ lance_scanner_set_fragment_ids(scanner, fragments.as_ptr(), fragments.len()), ++ 0 ++ ); ++ if disable_first { ++ assert_eq!(lance_scanner_set_use_scalar_index(scanner, false), 0); ++ } ++ assert_eq!( ++ lance_scanner_set_scalar_index_segment(scanner, uuids[0].as_ptr()), ++ 0 ++ ); ++ if !disable_first { ++ assert_eq!(lance_scanner_set_use_scalar_index(scanner, false), 0); ++ } ++ if let Some(limit) = limit { ++ assert_eq!(lance_scanner_set_limit(scanner, limit), 0); ++ } ++ assert_eq!(lance_scanner_set_offset(scanner, offset), 0); ++ let mut captured = CapturedScanStatistics::default(); ++ assert_eq!( ++ lance_scanner_set_statistics_callback( ++ scanner, ++ Some(capture_scan_statistics), ++ (&mut captured as *mut CapturedScanStatistics).cast(), ++ ), ++ 0 ++ ); ++ let mut stream = FFI_ArrowArrayStream::empty(); ++ assert_eq!(lance_scanner_to_arrow_stream(scanner, &mut stream), 0); ++ let mut ids = Vec::new(); ++ for batch in ArrowArrayStreamReader::from_raw(&mut stream).unwrap() { ++ let batch = batch.unwrap(); ++ ids.extend_from_slice( ++ batch ++ .column_by_name("id") ++ .unwrap() ++ .as_any() ++ .downcast_ref::() ++ .unwrap() ++ .values(), ++ ); ++ } ++ assert_eq!(ids, expected); ++ assert_eq!(captured.calls, 1); ++ for metric in ["scalar_segments_searched", "scalar_segment_candidate_rows"] { ++ assert_eq!( ++ captured ++ .metrics ++ .iter() ++ .filter(|(name, _, _)| name == metric) ++ .map(|(_, _, value)| *value) ++ .sum::(), ++ 0, ++ "{metric}" ++ ); ++ } ++ assert_eq!(captured.indices_loaded, 0); ++ assert_eq!(captured.index_comparisons, 0); ++ assert!(captured.metrics.iter().any(|(name, _, value)| name ++ == "scalar_segment_fallback_disabled" ++ && *value == 1)); ++ lance_scanner_close(scanner); ++ } ++ } ++ lance_dataset_close(ds); ++ } ++} ++ ++#[test] ++fn test_scalar_segment_rejects_include_deleted_rows_after_index_rebuild() { ++ use lance::index::DatasetIndexExt; ++ use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams}; ++ ++ let (_tmp, uri, _) = create_scalar_segment_fixture(lance_index::IndexType::BTree, false); ++ let uuid = lance_c::runtime::block_on(async { ++ let mut ds = Dataset::open(&uri).await.unwrap(); ++ ds.delete("id = 2").await.unwrap(); ++ ds.drop_index("key_idx").await.unwrap(); ++ // A segment built after the delete cannot return the tombstoned row, ++ // even though its search result is Exact for the indexed live rows. ++ let params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); ++ let segment = ds ++ .create_index_builder(&["key"], lance_index::IndexType::BTree, ¶ms) ++ .name("key_idx".into()) ++ .fragments(vec![0]) ++ .execute_uncommitted() ++ .await ++ .unwrap(); ++ let uuid = *segment.uuid.as_bytes(); ++ ds.commit_existing_index_segments("key_idx", "key", vec![segment]) ++ .await ++ .unwrap(); ++ uuid ++ }); ++ ++ let (ids, _) = scalar_segment_ids(&uri, &uuid, &[0], "key >= 0 AND id >= 2", None, 0); ++ assert_eq!(ids, vec![3], "live-row segment scans remain supported"); ++ ++ let uri = c_str(&uri); ++ let filter = c_str("key >= 0 AND id >= 2"); ++ unsafe { ++ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0); ++ assert!(!ds.is_null()); ++ // Check both setter orders: compatibility is validated at stream creation. ++ for segment_first in [None, Some(false), Some(true)] { ++ let scanner = lance_scanner_new(ds, ptr::null(), filter.as_ptr()); ++ assert!(!scanner.is_null()); ++ assert_eq!( ++ lance_scanner_set_fragment_ids(scanner, [0u64].as_ptr(), 1), ++ 0 ++ ); ++ assert_eq!(lance_scanner_with_row_id(scanner, true), 0); ++ if segment_first.is_none() { ++ assert_eq!(lance_scanner_set_use_scalar_index(scanner, false), 0); ++ } ++ if segment_first == Some(true) { ++ assert_eq!( ++ lance_scanner_set_scalar_index_segment(scanner, uuid.as_ptr()), ++ 0 ++ ); ++ } ++ assert_eq!(lance_scanner_set_include_deleted_rows(scanner, true), 0); ++ if segment_first == Some(false) { ++ assert_eq!( ++ lance_scanner_set_scalar_index_segment(scanner, uuid.as_ptr()), ++ 0 ++ ); ++ } ++ let mut stream = FFI_ArrowArrayStream::empty(); ++ let rc = lance_scanner_to_arrow_stream(scanner, &mut stream); ++ if segment_first.is_some() { ++ assert_eq!(rc, -1, "segment scans must not silently omit deleted rows"); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ assert!(take_last_error_message().contains("include_deleted_rows=true")); ++ } else { ++ assert_eq!(rc, 0); ++ let reader = ArrowArrayStreamReader::from_raw(&mut stream).unwrap(); ++ let mut ids = Vec::new(); ++ for batch in reader { ++ let batch = batch.unwrap(); ++ ids.extend_from_slice( ++ batch ++ .column_by_name("id") ++ .unwrap() ++ .as_any() ++ .downcast_ref::() ++ .unwrap() ++ .values(), ++ ); ++ } ++ ids.sort_unstable(); ++ assert_eq!( ++ ids, ++ vec![2, 3], ++ "ordinary scans can still read tombstoned rows" ++ ); ++ } ++ lance_scanner_close(scanner); ++ } ++ lance_dataset_close(ds); ++ } ++} ++ + #[test] + fn test_scalar_segment_requires_explicit_domain_and_checks_uuid() { + let (_tmp, uri, uuids) = create_scalar_segment_fixture(lance_index::IndexType::BTree, false); From 1577041e44cbf5998725805931eb9005dc094069 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Wed, 9 Sep 2026 19:24:54 +0800 Subject: [PATCH 05/12] update thirdparty --- thirdparty/download-thirdparty.sh | 5 +- thirdparty/patches/lance-c-0.1.9-pr-73.patch | 133 ++-- .../patches/lance-c-0.1.9-pr-75-pr-78.patch | 2 +- thirdparty/patches/lance-c-0.1.9-pr-77.patch | 717 +++++++++--------- 4 files changed, 442 insertions(+), 415 deletions(-) diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index 43648a1abf9b22..6239ab8cc3b4f3 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -722,8 +722,9 @@ fi if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then cd "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}" if [[ ! -f "${PATCHED_MARK}" ]]; then - # PR #79 requires the Lance v11 APIs introduced by PR #77. - for lance_patch in pr-73 pr-74 pr-75-pr-78 pr-77 pr-79; do + # Apply the merged PRs first; the latest PR #73 and #79 both require Lance v11. + # This order keeps the upstream patches unchanged, including Cargo.lock. + for lance_patch in pr-74 pr-75-pr-78 pr-77 pr-73 pr-79; do patch --batch --forward --reject-file=- --fuzz=0 --no-backup-if-mismatch -s \ -p1 <"${TP_PATCH_DIR}/${LANCE_C_SOURCE}-${lance_patch}.patch" done diff --git a/thirdparty/patches/lance-c-0.1.9-pr-73.patch b/thirdparty/patches/lance-c-0.1.9-pr-73.patch index 7d5eb45ad6ce7d..33a8985b25aecc 100644 --- a/thirdparty/patches/lance-c-0.1.9-pr-73.patch +++ b/thirdparty/patches/lance-c-0.1.9-pr-73.patch @@ -1,10 +1,10 @@ -From a4c71309ddb76ad79808e3e8f7797bcd9bfc174a Mon Sep 17 00:00:00 2001 +From cc373477a6485cb24ed7ea88ef506e93caacedd0 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Thu, 3 Sep 2026 13:00:43 +0800 Subject: [PATCH] foyer --- - Cargo.lock | 205 ++++++ + Cargo.lock | 207 +++++- Cargo.toml | 4 + README.md | 22 + include/lance/lance.h | 63 ++ @@ -19,32 +19,28 @@ Subject: [PATCH] foyer tests/c_api_test.rs | 221 +++++++ tests/cpp/test_c_api.c | 32 + tests/cpp/test_cpp_api.cpp | 23 + - 15 files changed, 1918 insertions(+), 1 deletion(-) + 15 files changed, 1917 insertions(+), 4 deletions(-) create mode 100644 src/data_cache.rs create mode 100644 src/foyer_data_cache.rs diff --git a/Cargo.lock b/Cargo.lock -index 60c1caf..85536f5 100644 +index bc37cb9..199f997 100644 --- a/Cargo.lock +++ b/Cargo.lock -@@ -434,6 +434,16 @@ dependencies = [ - "loom", +@@ -444,6 +444,12 @@ dependencies = [ + "slab", ] +[[package]] +name = "asyncband" -+version = "0.6.7" ++version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "94a214ba60d6231afd0e805e3c27c45a1626d9debaa5a5061c45a1ea1b2f1ed0" -+dependencies = [ -+ "hashbrown 0.17.1", -+ "slab", -+] ++checksum = "2e52766975a4f080528a898235c51e82e65df9db713419067b5368040eeb5659" + [[package]] name = "atoi" version = "2.0.0" -@@ -1267,6 +1277,17 @@ version = "0.8.7" +@@ -1293,6 +1299,17 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" @@ -62,7 +58,7 @@ index 60c1caf..85536f5 100644 [[package]] name = "countio" version = "0.3.0" -@@ -2112,6 +2133,12 @@ dependencies = [ +@@ -2148,6 +2165,12 @@ dependencies = [ "url", ] @@ -73,9 +69,9 @@ index 60c1caf..85536f5 100644 +checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d" + [[package]] - name = "der" - version = "0.7.10" -@@ -2298,6 +2325,16 @@ version = "0.2.3" + name = "defmt" + version = "1.1.1" +@@ -2366,6 +2389,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" @@ -92,18 +88,18 @@ index 60c1caf..85536f5 100644 [[package]] name = "fastrand" version = "2.3.0" -@@ -2369,6 +2406,127 @@ dependencies = [ +@@ -2437,12 +2470,133 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "foyer" -+version = "0.22.4" ++version = "0.22.5" +source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "cab5c4bac30455a0dbc4c858436fb440d51bc6543daafbdf35c5e6ca94c0afd7" ++checksum = "f911e6f0b4909f23d65a95c5d27bcf2f92855b8a0f629b47b743d53d14f2828b" +dependencies = [ + "anyhow", -+ "asyncband", ++ "asyncband 0.7.1", + "equivalent", + "foyer-common", + "foyer-memory", @@ -118,9 +114,9 @@ index 60c1caf..85536f5 100644 + +[[package]] +name = "foyer-common" -+version = "0.22.4" ++version = "0.22.5" +source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "e3634c6f3da978b6cae98d54367a2342041d3a4786b20c0384164cc7fce94787" ++checksum = "05cdcae6cedec72c28e97ada0b453e33b1df57fbc209708091107be2a283d577" +dependencies = [ + "anyhow", + "bytes", @@ -143,13 +139,13 @@ index 60c1caf..85536f5 100644 + +[[package]] +name = "foyer-memory" -+version = "0.22.4" ++version = "0.22.5" +source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "5349a61af676b3275bfef7a5fceceeadf6b0a7dd9ec14ba0edd78e1ff771d101" ++checksum = "8a51c8ce8e1e323a1e087ac45bf629d953676fc5e0037d14a799fadd272b42db" +dependencies = [ + "anyhow", -+ "asyncband", -+ "bitflags", ++ "asyncband 0.7.1", ++ "bitflags 2.11.0", + "datasketches", + "equivalent", + "foyer-common", @@ -168,13 +164,13 @@ index 60c1caf..85536f5 100644 + +[[package]] +name = "foyer-storage" -+version = "0.22.4" ++version = "0.22.5" +source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "49cfcce3c10a1f2ac65bfcf0bd85501462d454c594c8a16a4a27cce773599381" ++checksum = "127a64057f63e361123cf62b3fd50a36147783687d4cd36e7087c27f9909265b" +dependencies = [ + "allocator-api2", + "anyhow", -+ "asyncband", ++ "asyncband 0.7.1", + "bytes", + "core_affinity", + "equivalent", @@ -200,13 +196,19 @@ index 60c1caf..85536f5 100644 + +[[package]] +name = "foyer-tokio" -+version = "0.22.4" ++version = "0.22.5" +source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "6b7315103199f3415a010befd6dd5a1c8e7082fbc49c0194931e65629995d9d1" ++checksum = "cbdbb9f39443cb348a069baa1a0ec73bcea848a4a383eb2da1a5ea7a0ca05941" +dependencies = [ + "tokio", +] + + [[package]] + name = "frostem" + version = "1.20260821.5" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "36a80a7406da302e04bfd2ca987907590d3a1f3c69958947c43890abd7426b2f" + +[[package]] +name = "fs4" +version = "0.13.1" @@ -220,7 +222,7 @@ index 60c1caf..85536f5 100644 [[package]] name = "fs_extra" version = "1.3.0" -@@ -3392,6 +3550,15 @@ dependencies = [ +@@ -3485,6 +3639,15 @@ dependencies = [ "either", ] @@ -236,7 +238,7 @@ index 60c1caf..85536f5 100644 [[package]] name = "itoa" version = "1.0.18" -@@ -3703,8 +3870,11 @@ dependencies = [ +@@ -3788,8 +3951,11 @@ dependencies = [ "arrow", "arrow-array", "arrow-schema", @@ -248,7 +250,7 @@ index 60c1caf..85536f5 100644 "futures", "half", "lance", -@@ -3718,6 +3888,7 @@ dependencies = [ +@@ -3803,6 +3969,7 @@ dependencies = [ "lance-table", "libc", "log", @@ -256,9 +258,9 @@ index 60c1caf..85536f5 100644 "pin-project", "prost", "snafu", -@@ -4399,6 +4570,15 @@ version = "2.8.0" - source = "registry+https://github.com/rust-lang/crates.io-index" - checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +@@ -4492,6 +4659,15 @@ dependencies = [ + "libc", + ] +[[package]] +name = "memoffset" @@ -272,7 +274,7 @@ index 60c1caf..85536f5 100644 [[package]] name = "mime" version = "0.3.17" -@@ -4436,6 +4616,16 @@ dependencies = [ +@@ -4529,6 +4705,16 @@ dependencies = [ "windows-sys 0.61.2", ] @@ -289,7 +291,34 @@ index 60c1caf..85536f5 100644 [[package]] name = "moka" version = "0.12.15" -@@ -6547,6 +6737,12 @@ version = "0.4.12" +@@ -4840,7 +5026,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "48dbcef97d3eb7591db2c18d5cae95c836bcce07359b98d98dd6f4e861eb77b7" + dependencies = [ + "anyhow", +- "asyncband", ++ "asyncband 0.6.7", + "base64 0.23.1", + "bytes", + "futures", +@@ -4879,7 +5065,7 @@ version = "0.58.2" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "03f9e144b5228d741c3763ade8711d9b72e5fb6d998e779f2d7a09da0b5a3eba" + dependencies = [ +- "asyncband", ++ "asyncband 0.6.7", + "futures", + "http 1.4.0", + "opendal-core", +@@ -4943,7 +5129,7 @@ version = "0.58.2" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "2d564484a8f7d091827e825cfc91ed45bd48e64d262451ee041fe843db81bd8a" + dependencies = [ +- "asyncband", ++ "asyncband 0.6.7", + "base64 0.23.1", + "bytes", + "http 1.4.0", +@@ -6668,6 +6854,12 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" @@ -302,7 +331,7 @@ index 60c1caf..85536f5 100644 [[package]] name = "smallvec" version = "1.15.1" -@@ -7831,6 +8027,15 @@ dependencies = [ +@@ -7930,6 +8122,15 @@ dependencies = [ "windows-targets 0.52.6", ] @@ -319,7 +348,7 @@ index 60c1caf..85536f5 100644 name = "windows-sys" version = "0.60.2" diff --git a/Cargo.toml b/Cargo.toml -index d072a5d..342928c 100644 +index 3920a65..356c9dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,8 @@ datafusion = { version = "54.0.0", default-features = false } @@ -335,7 +364,7 @@ index d072a5d..342928c 100644 half = "2" tokio = { version = "1", features = ["rt-multi-thread", "sync"] } futures = "0.3" -+foyer = "=0.22.4" ++foyer = "=0.22.5" log = "0.4" libc = "0.2" +object_store = "0.13.2" @@ -383,7 +412,7 @@ index 2056671..d9a5f2b 100644 `lance_dataset_open` takes a `version` argument — `0` means the latest, any diff --git a/include/lance/lance.h b/include/lance/lance.h -index 3bf291f..e4c593e 100644 +index 31213da..152351b 100644 --- a/include/lance/lance.h +++ b/include/lance/lance.h @@ -206,6 +206,36 @@ typedef struct LanceSessionCacheStats { @@ -471,10 +500,10 @@ index 3bf291f..e4c593e 100644 void lance_dataset_close(LanceDataset* dataset); diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp -index 6cf245f..c1102e4 100644 +index 404d2df..3fe9dbc 100644 --- a/include/lance/lance.hpp +++ b/include/lance/lance.hpp -@@ -171,6 +171,13 @@ struct SqlColumn { +@@ -176,6 +176,13 @@ struct SqlColumn { // ─── Shared Session ────────────────────────────────────────────────────────── @@ -488,7 +517,7 @@ index 6cf245f..c1102e4 100644 class Session { Handle handle_; -@@ -180,6 +187,21 @@ class Session { +@@ -185,6 +192,21 @@ class Session { if (!handle_) check_error(); } @@ -510,7 +539,7 @@ index 6cf245f..c1102e4 100644 LanceSessionCacheStats cache_stats() const { LanceSessionCacheStats stats{}; if (lance_session_get_cache_stats(handle_.get(), &stats) != 0) -@@ -260,6 +282,13 @@ class Dataset { +@@ -265,6 +287,13 @@ class Dataset { return Dataset(ds); } @@ -1975,7 +2004,7 @@ index 1971510..ba51c87 100644 // SAFETY: `out_dataset` is non-NULL (checked above) and the caller // guarantees it points to caller-owned, writable storage of size diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs -index 8805764..b4313f4 100644 +index bde742d..a4ea8f8 100644 --- a/tests/c_api_test.rs +++ b/tests/c_api_test.rs @@ -96,10 +96,83 @@ fn create_large_dataset(num_rows: i32) -> (tempfile::TempDir, String) { @@ -2183,7 +2212,7 @@ index 8805764..b4313f4 100644 #[test] fn test_open_nonexistent() { let c_uri = c_str("memory://nonexistent_dataset_xyz"); -@@ -2787,6 +2974,40 @@ fn test_dataset_restore_to_prior_version() { +@@ -2977,6 +3164,40 @@ fn test_dataset_restore_to_prior_version() { unsafe { lance_dataset_close(ds) }; } @@ -2275,7 +2304,7 @@ index c49ecfa..dd674eb 100644 test_scan_with_limit(uri); test_versions(uri); diff --git a/tests/cpp/test_cpp_api.cpp b/tests/cpp/test_cpp_api.cpp -index 17b1ab6..e1aadbd 100644 +index 28762b8..5f34bde 100644 --- a/tests/cpp/test_cpp_api.cpp +++ b/tests/cpp/test_cpp_api.cpp @@ -99,6 +99,28 @@ static void test_shared_session(const std::string& uri) { @@ -2307,7 +2336,7 @@ index 17b1ab6..e1aadbd 100644 static void test_dataset_schema(const std::string& uri) { TEST(test_dataset_schema); -@@ -922,6 +944,7 @@ int main(int argc, char** argv) { +@@ -929,6 +951,7 @@ int main(int argc, char** argv) { test_dataset_open(uri); test_shared_session(uri); diff --git a/thirdparty/patches/lance-c-0.1.9-pr-75-pr-78.patch b/thirdparty/patches/lance-c-0.1.9-pr-75-pr-78.patch index 3e2240a1e6f46b..08687627e7ea21 100644 --- a/thirdparty/patches/lance-c-0.1.9-pr-75-pr-78.patch +++ b/thirdparty/patches/lance-c-0.1.9-pr-75-pr-78.patch @@ -1,5 +1,5 @@ Lance-C v0.1.9 scanner options: PR #75 followed by PR #78. -Apply after lance-c-0.1.9-pr-73.patch and lance-c-0.1.9-pr-74.patch. +Apply after lance-c-0.1.9-pr-74.patch, before lance-c-0.1.9-pr-77.patch. The upstream mail patches below are concatenated without modification. PR #75: https://github.com/lance-format/lance-c/pull/75 diff --git a/thirdparty/patches/lance-c-0.1.9-pr-77.patch b/thirdparty/patches/lance-c-0.1.9-pr-77.patch index 9a8931c6c2afd4..341a1c692dcebd 100644 --- a/thirdparty/patches/lance-c-0.1.9-pr-77.patch +++ b/thirdparty/patches/lance-c-0.1.9-pr-77.patch @@ -1,17 +1,49 @@ -Backport Lance-C PR #77: build: bump lance to v11.0.0. -Upstream: https://github.com/lance-format/lance-c/pull/77 -Commit: eaf06c0374e62de6b519f55efe17de0c58e88c0a -Merged as: 373c2bb53d6d7e2f2a8240ffa4ba1e82daa9fe16 -Apply after PR #73, #74, and #75/#78 on lance-c v0.1.9. +From eaf06c0374e62de6b519f55efe17de0c58e88c0a Mon Sep 17 00:00:00 2001 +From: "jianjian.xie" +Date: Fri, 4 Sep 2026 22:04:55 -0700 +Subject: [PATCH] build: bump lance to v11.0.0 -Cargo.lock is adapted to retain the Foyer dependencies added by PR #73; -the asyncband package is already present, and Foyer's bitflags dependency -is disambiguated. All other upstream changes are retained. +Summary: +Intent: +- Move the lance git pins from e934cc2c to ab6b5bbe (lance v11.0.0 release tag) + so lance-c tracks a released upstream version instead of an arbitrary commit. +- Pick up the v11 blob APIs (read_blob_ranges, Option-based take_blobs results) + needed to answer #76 without a second pin bump. + +Changes: +- Point all lance, lance-core, lance-file, lance-index, lance-io, lance-linalg, + lance-table, lance-datafusion, and lance-datagen dependencies at ab6b5bbe. +- Re-resolve Cargo.lock; blake3, jiff, and reqwest 0.13 were unlocked explicitly + because v11 raised their minimum versions, the rest follows from lance v11 + (opendal 0.58, lance-namespace-reqwest-client 0.11, etc.). +- Adapt to upstream signature changes: build_global_bm25_scorer takes an optional + metrics collector, MatchQueryExec/PhraseQueryExec::new_with_segments are now + fallible, DataFile::new takes a ConcreteFileVersion, and pb::IndexMetadata + gained covering_fields. +- Keep the DOT PQ strict-subset guard; upstream make_global_pq is unchanged at + v11, so only the referenced revision in the comment and error text moved. + +Test Plan: +- cargo fmt, cargo check --all-targets, cargo clippy --all-targets -D warnings. +- cargo test: 367 passed, 0 failed, 2 ignored. +- cargo test --test compile_and_run_test -- --ignored: 2 passed (C and C++ + compile-and-run against the rebuilt library). + +Co-Authored-By: Claude Fable 5.1 +--- + Cargo.lock | 653 ++++++++++++++++++++++++------------------- + Cargo.toml | 22 +- + src/fts_query.rs | 4 +- + src/index_segment.rs | 4 +- + src/scanner.rs | 4 +- + tests/c_api_test.rs | 8 +- + 6 files changed, 391 insertions(+), 304 deletions(-) diff --git a/Cargo.lock b/Cargo.lock +index 60c1caf..bc37cb9 100644 --- a/Cargo.lock +++ b/Cargo.lock -@@ -222,7 +222,7 @@ +@@ -222,7 +222,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", @@ -20,7 +52,7 @@ diff --git a/Cargo.lock b/Cargo.lock "chrono", "comfy-table", "half", -@@ -332,7 +332,7 @@ +@@ -332,7 +332,7 @@ version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" dependencies = [ @@ -29,7 +61,24 @@ diff --git a/Cargo.lock b/Cargo.lock "serde_core", "serde_json", ] -@@ -514,7 +514,6 @@ +@@ -434,6 +434,16 @@ dependencies = [ + "loom", + ] + ++[[package]] ++name = "asyncband" ++version = "0.6.7" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "94a214ba60d6231afd0e805e3c27c45a1626d9debaa5a5061c45a1ea1b2f1ed0" ++dependencies = [ ++ "hashbrown 0.17.1", ++ "slab", ++] ++ + [[package]] + name = "atoi" + version = "2.0.0" +@@ -504,7 +514,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" dependencies = [ "aws-lc-sys", @@ -37,7 +86,7 @@ diff --git a/Cargo.lock b/Cargo.lock "zeroize", ] -@@ -641,7 +640,7 @@ +@@ -631,7 +640,7 @@ dependencies = [ "bytes", "form_urlencoded", "hex", @@ -46,45 +95,43 @@ diff --git a/Cargo.lock b/Cargo.lock "http 0.2.12", "http 1.4.0", "percent-encoding", -@@ -840,6 +839,12 @@ +@@ -829,6 +838,12 @@ version = "0.22.1" + source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - [[package]] ++[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + -+[[package]] + [[package]] name = "base64-simd" version = "0.8.0" - source = "registry+https://github.com/rust-lang/crates.io-index" -@@ -870,6 +875,12 @@ +@@ -858,6 +873,12 @@ dependencies = [ + "num-traits", + ] - [[package]] - name = "bitflags" ++[[package]] ++name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + -+[[package]] -+name = "bitflags" + [[package]] + name = "bitflags" version = "2.11.0" - source = "registry+https://github.com/rust-lang/crates.io-index" - checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" -@@ -897,16 +908,15 @@ +@@ -887,16 +908,15 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.3" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" --dependencies = [ -- "arrayref", +version = "1.8.7" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" -+dependencies = [ + dependencies = [ +- "arrayref", "arrayvec", "cc", "cfg-if 1.0.4", @@ -94,44 +141,41 @@ diff --git a/Cargo.lock b/Cargo.lock ] [[package]] -@@ -1138,6 +1148,12 @@ +@@ -1127,6 +1147,12 @@ dependencies = [ + "cc", ] - [[package]] ++[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + -+[[package]] + [[package]] name = "colorchoice" version = "1.0.5" - source = "registry+https://github.com/rust-lang/crates.io-index" -@@ -1316,12 +1332,13 @@ +@@ -1295,12 +1321,13 @@ dependencies = [ ] [[package]] -name = "crc32c" -version = "0.6.8" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" --dependencies = [ -- "rustc_version", +name = "crc-fast" +version = "1.10.0" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" -+dependencies = [ + dependencies = [ +- "rustc_version", + "digest 0.10.7", + "spin 0.10.1", ] [[package]] -@@ -1457,6 +1474,15 @@ - version = "0.0.7" +@@ -1437,6 +1464,15 @@ version = "0.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1" -+ + +[[package]] +name = "ctutils" +version = "0.4.2" @@ -140,10 +184,11 @@ diff --git a/Cargo.lock b/Cargo.lock +dependencies = [ + "cmov", +] - ++ [[package]] name = "darling" -@@ -1806,7 +1832,7 @@ + version = "0.23.0" +@@ -1785,7 +1821,7 @@ checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" dependencies = [ "arrow", "arrow-buffer", @@ -152,10 +197,11 @@ diff --git a/Cargo.lock b/Cargo.lock "blake2", "blake3", "chrono", -@@ -2140,6 +2166,37 @@ - checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d" +@@ -2112,6 +2148,37 @@ dependencies = [ + "url", + ] - [[package]] ++[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" @@ -186,11 +232,10 @@ diff --git a/Cargo.lock b/Cargo.lock + "thiserror 2.0.18", +] + -+[[package]] + [[package]] name = "der" version = "0.7.10" - source = "registry+https://github.com/rust-lang/crates.io-index" -@@ -2181,6 +2238,7 @@ +@@ -2154,6 +2221,7 @@ dependencies = [ "block-buffer 0.12.1", "const-oid 0.10.2", "crypto-common 0.2.2", @@ -198,7 +243,7 @@ diff --git a/Cargo.lock b/Cargo.lock ] [[package]] -@@ -2359,7 +2417,7 @@ +@@ -2322,7 +2390,7 @@ version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ @@ -207,29 +252,20 @@ diff --git a/Cargo.lock b/Cargo.lock "rustc_version", ] -@@ -2459,7 +2517,7 @@ - dependencies = [ - "anyhow", - "asyncband", -- "bitflags", -+ "bitflags 2.11.0", - "datasketches", - "equivalent", - "foyer-common", -@@ -2518,6 +2576,12 @@ +@@ -2369,6 +2437,12 @@ dependencies = [ + "percent-encoding", ] - [[package]] ++[[package]] +name = "frostem" +version = "1.20260821.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36a80a7406da302e04bfd2ca987907590d3a1f3c69958947c43890abd7426b2f" + -+[[package]] - name = "fs4" - version = "0.13.1" - source = "registry+https://github.com/rust-lang/crates.io-index" -@@ -2535,8 +2599,8 @@ + [[package]] + name = "fs_extra" + version = "1.3.0" +@@ -2377,8 +2451,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" @@ -240,18 +276,16 @@ diff --git a/Cargo.lock b/Cargo.lock dependencies = [ "arrow-array", "rand 0.9.2", -@@ -2885,14 +2949,23 @@ +@@ -2727,14 +2801,23 @@ dependencies = [ [[package]] name = "goosefs-sdk" -version = "0.1.5" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f" --dependencies = [ +version = "0.1.9" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f" +checksum = "e1ea4eee6dcbc31b25ab4fd577adc55b677d2bed3aa3016c44c58fbe1b2298a5" -+dependencies = [ + dependencies = [ + "arc-swap", "async-trait", "bytes", @@ -268,7 +302,7 @@ diff --git a/Cargo.lock b/Cargo.lock "prost", "prost-types", "rand 0.9.2", -@@ -2905,6 +2978,7 @@ +@@ -2747,6 +2830,7 @@ dependencies = [ "tonic-prost", "tracing", "uuid", @@ -276,12 +310,10 @@ diff --git a/Cargo.lock b/Cargo.lock ] [[package]] -@@ -3056,6 +3130,15 @@ - checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" - dependencies = [ +@@ -2900,6 +2984,15 @@ dependencies = [ "digest 0.10.7", -+] -+ + ] + +[[package]] +name = "hmac" +version = "0.13.0" @@ -289,10 +321,12 @@ diff --git a/Cargo.lock b/Cargo.lock +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", - ] - ++] ++ [[package]] -@@ -3211,7 +3294,7 @@ + name = "hostname" + version = "0.4.2" +@@ -3053,7 +3146,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ @@ -301,7 +335,7 @@ diff --git a/Cargo.lock b/Cargo.lock "bytes", "futures-channel", "futures-util", -@@ -3505,7 +3588,7 @@ +@@ -3347,7 +3440,7 @@ version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62" dependencies = [ @@ -310,24 +344,22 @@ diff --git a/Cargo.lock b/Cargo.lock "cfg-if 1.0.4", "libc", ] -@@ -3567,10 +3650,12 @@ +@@ -3400,10 +3493,12 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.23" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" --dependencies = [ +version = "0.2.35" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" -+dependencies = [ + dependencies = [ + "defmt", + "jiff-core", "jiff-static", "jiff-tzdb-platform", "js-sys", -@@ -3579,15 +3664,25 @@ +@@ -3412,15 +3507,25 @@ dependencies = [ "portable-atomic-util", "serde_core", "wasm-bindgen", @@ -347,21 +379,20 @@ diff --git a/Cargo.lock b/Cargo.lock [[package]] name = "jiff-static" -version = "0.2.23" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" --dependencies = [ +version = "0.2.35" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" -+dependencies = [ + dependencies = [ + "jiff-core", "proc-macro2", "quote", "syn 2.0.117", -@@ -3698,24 +3793,6 @@ +@@ -3530,24 +3635,6 @@ dependencies = [ + "serde_json", ] - [[package]] +-[[package]] -name = "jsonwebtoken" -version = "10.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" @@ -379,11 +410,10 @@ diff --git a/Cargo.lock b/Cargo.lock - "zeroize", -] - --[[package]] + [[package]] name = "konst" version = "0.4.3" - source = "registry+https://github.com/rust-lang/crates.io-index" -@@ -3734,8 +3811,8 @@ +@@ -3567,8 +3654,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" @@ -394,7 +424,7 @@ diff --git a/Cargo.lock b/Cargo.lock dependencies = [ "arc-swap", "arrow", -@@ -3751,7 +3828,6 @@ +@@ -3584,7 +3671,6 @@ dependencies = [ "async-recursion", "async-trait", "async_cell", @@ -402,7 +432,7 @@ diff --git a/Cargo.lock b/Cargo.lock "byteorder", "bytes", "chrono", -@@ -3766,7 +3842,6 @@ +@@ -3599,7 +3685,6 @@ dependencies = [ "either", "fst", "futures", @@ -410,7 +440,7 @@ diff --git a/Cargo.lock b/Cargo.lock "humantime", "itertools 0.14.0", "lance-arrow", -@@ -3808,8 +3883,8 @@ +@@ -3641,8 +3726,8 @@ dependencies = [ [[package]] name = "lance-arrow" @@ -421,7 +451,7 @@ diff --git a/Cargo.lock b/Cargo.lock dependencies = [ "arrow-array", "arrow-buffer", -@@ -3831,7 +3906,7 @@ +@@ -3664,7 +3749,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" @@ -430,7 +460,7 @@ diff --git a/Cargo.lock b/Cargo.lock dependencies = [ "arrow-array", "arrow-buffer", -@@ -3845,7 +3920,7 @@ +@@ -3678,7 +3763,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" @@ -439,7 +469,7 @@ diff --git a/Cargo.lock b/Cargo.lock dependencies = [ "arrow-array", "arrow-schema", -@@ -3854,8 +3929,8 @@ +@@ -3687,8 +3772,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" @@ -450,7 +480,7 @@ diff --git a/Cargo.lock b/Cargo.lock dependencies = [ "arrayref", "crunchy", -@@ -3899,20 +3974,19 @@ +@@ -3728,20 +3813,19 @@ dependencies = [ [[package]] name = "lance-core" @@ -474,7 +504,7 @@ diff --git a/Cargo.lock b/Cargo.lock "lance-arrow", "lance-derive", "libc", -@@ -3923,13 +3997,13 @@ +@@ -3752,13 +3836,13 @@ dependencies = [ "object_store", "pin-project", "prost", @@ -489,7 +519,7 @@ diff --git a/Cargo.lock b/Cargo.lock "tokio-util", "tracing", "twox-hash", -@@ -3938,8 +4012,8 @@ +@@ -3767,8 +3851,8 @@ dependencies = [ [[package]] name = "lance-datafusion" @@ -500,7 +530,7 @@ diff --git a/Cargo.lock b/Cargo.lock dependencies = [ "arrow", "arrow-array", -@@ -3959,7 +4033,6 @@ +@@ -3788,7 +3872,6 @@ dependencies = [ "jsonb", "lance-arrow", "lance-core", @@ -508,7 +538,7 @@ diff --git a/Cargo.lock b/Cargo.lock "lance-geo", "log", "pin-project", -@@ -3971,8 +4044,8 @@ +@@ -3800,8 +3883,8 @@ dependencies = [ [[package]] name = "lance-datagen" @@ -519,7 +549,7 @@ diff --git a/Cargo.lock b/Cargo.lock dependencies = [ "arrow", "arrow-array", -@@ -3989,8 +4062,8 @@ +@@ -3818,8 +3901,8 @@ dependencies = [ [[package]] name = "lance-derive" @@ -530,7 +560,7 @@ diff --git a/Cargo.lock b/Cargo.lock dependencies = [ "proc-macro2", "quote", -@@ -3999,8 +4072,8 @@ +@@ -3828,8 +3911,8 @@ dependencies = [ [[package]] name = "lance-encoding" @@ -541,7 +571,7 @@ diff --git a/Cargo.lock b/Cargo.lock dependencies = [ "arrow-arith", "arrow-array", -@@ -4025,8 +4098,6 @@ +@@ -3854,8 +3937,6 @@ dependencies = [ "num-traits", "prost", "prost-build", @@ -550,7 +580,7 @@ diff --git a/Cargo.lock b/Cargo.lock "tokio", "tracing", "xxhash-rust", -@@ -4035,12 +4106,13 @@ +@@ -3864,12 +3945,13 @@ dependencies = [ [[package]] name = "lance-file" @@ -566,7 +596,7 @@ diff --git a/Cargo.lock b/Cargo.lock "arrow-data", "arrow-schema", "arrow-select", -@@ -4066,8 +4138,8 @@ +@@ -3895,8 +3977,8 @@ dependencies = [ [[package]] name = "lance-geo" @@ -577,7 +607,7 @@ diff --git a/Cargo.lock b/Cargo.lock dependencies = [ "datafusion", "geo-traits", -@@ -4081,13 +4153,14 @@ +@@ -3910,13 +3992,14 @@ dependencies = [ [[package]] name = "lance-index" @@ -594,7 +624,7 @@ diff --git a/Cargo.lock b/Cargo.lock "arrow-ord", "arrow-schema", "arrow-select", -@@ -4096,7 +4169,6 @@ +@@ -3925,7 +4008,6 @@ dependencies = [ "async-trait", "bitvec", "bytes", @@ -602,7 +632,7 @@ diff --git a/Cargo.lock b/Cargo.lock "crossbeam-queue", "datafusion", "datafusion-common", -@@ -4116,7 +4188,6 @@ +@@ -3945,7 +4027,6 @@ dependencies = [ "lance-bitpacking", "lance-core", "lance-datafusion", @@ -610,7 +640,7 @@ diff --git a/Cargo.lock b/Cargo.lock "lance-encoding", "lance-file", "lance-geo", -@@ -4146,13 +4217,12 @@ +@@ -3975,13 +4056,12 @@ dependencies = [ "tempfile", "tokio", "tracing", @@ -626,7 +656,7 @@ diff --git a/Cargo.lock b/Cargo.lock dependencies = [ "arrow-array", "arrow-schema", -@@ -4174,18 +4244,12 @@ +@@ -4003,18 +4083,12 @@ dependencies = [ [[package]] name = "lance-io" @@ -647,7 +677,7 @@ diff --git a/Cargo.lock b/Cargo.lock "async-trait", "aws-config", "aws-credential-types", -@@ -4193,10 +4257,8 @@ +@@ -4022,10 +4096,8 @@ dependencies = [ "bytes", "chrono", "futures", @@ -658,7 +688,7 @@ diff --git a/Cargo.lock b/Cargo.lock "lance-core", "lance-namespace", "log", -@@ -4208,34 +4270,37 @@ +@@ -4037,34 +4109,37 @@ dependencies = [ "pin-project", "prost", "rand 0.9.2", @@ -702,20 +732,19 @@ diff --git a/Cargo.lock b/Cargo.lock dependencies = [ "arrow", "async-trait", -@@ -4247,9 +4312,9 @@ +@@ -4076,9 +4151,9 @@ dependencies = [ [[package]] name = "lance-namespace-reqwest-client" -version = "0.8.6" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "ba3f0a235e3ed5f8805205649ccc7d7d0f3df23ce1294242c9265ad488d7f19d" +version = "0.11.1" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "ba3f0a235e3ed5f8805205649ccc7d7d0f3df23ce1294242c9265ad488d7f19d" +checksum = "1d06b1fbb5d41f93bc652b61e2872af92e8a6c5f6b4ce8839a8ecfa05365d359" dependencies = [ "reqwest 0.12.28", "serde", -@@ -4261,14 +4326,13 @@ +@@ -4090,14 +4165,13 @@ dependencies = [ [[package]] name = "lance-select" @@ -732,7 +761,7 @@ diff --git a/Cargo.lock b/Cargo.lock "itertools 0.14.0", "lance-core", "roaring", -@@ -4277,8 +4341,8 @@ +@@ -4106,8 +4180,8 @@ dependencies = [ [[package]] name = "lance-table" @@ -743,7 +772,7 @@ diff --git a/Cargo.lock b/Cargo.lock dependencies = [ "arrow", "arrow-array", -@@ -4286,6 +4350,7 @@ +@@ -4115,6 +4189,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "async-trait", @@ -751,23 +780,22 @@ diff --git a/Cargo.lock b/Cargo.lock "byteorder", "bytes", "chrono", -@@ -4315,11 +4380,11 @@ +@@ -4144,11 +4219,11 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "9.1.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" --dependencies = [ +version = "11.0.0" +source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" -+dependencies = [ + dependencies = [ + "frostem", "icu_segmenter", - "rust-stemmers", "serde", "stop-words", "unicode-normalization", -@@ -4331,7 +4396,7 @@ +@@ -4160,7 +4235,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ @@ -776,23 +804,23 @@ diff --git a/Cargo.lock b/Cargo.lock ] [[package]] -@@ -4462,9 +4527,9 @@ +@@ -4291,9 +4366,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +version = "0.4.34" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "loom" -@@ -4480,6 +4545,15 @@ +@@ -4308,6 +4383,15 @@ dependencies = [ + "tracing-subscriber", ] - [[package]] ++[[package]] +name = "lru" +version = "0.18.4" +source = "registry+https://github.com/rust-lang/crates.io-index" @@ -801,15 +829,13 @@ diff --git a/Cargo.lock b/Cargo.lock + "hashbrown 0.17.1", +] + -+[[package]] + [[package]] name = "lru-slab" version = "0.1.2" - source = "registry+https://github.com/rust-lang/crates.io-index" -@@ -4569,6 +4643,15 @@ - version = "2.8.0" +@@ -4399,6 +4483,15 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" -+ + +[[package]] +name = "memmap2" +version = "0.9.11" @@ -818,10 +844,11 @@ diff --git a/Cargo.lock b/Cargo.lock +dependencies = [ + "libc", +] - ++ [[package]] - name = "memoffset" -@@ -4809,7 +4892,7 @@ + name = "mime" + version = "0.3.17" +@@ -4619,7 +4712,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ @@ -830,7 +857,7 @@ diff --git a/Cargo.lock b/Cargo.lock ] [[package]] -@@ -4838,7 +4921,7 @@ +@@ -4648,7 +4741,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", @@ -839,7 +866,7 @@ diff --git a/Cargo.lock b/Cargo.lock "bytes", "chrono", "form_urlencoded", -@@ -4854,7 +4937,7 @@ +@@ -4664,7 +4757,7 @@ dependencies = [ "md-5 0.10.6", "parking_lot", "percent-encoding", @@ -848,28 +875,26 @@ diff --git a/Cargo.lock b/Cargo.lock "rand 0.10.1", "reqwest 0.12.28", "ring", -@@ -4873,9 +4956,9 @@ +@@ -4683,9 +4776,9 @@ dependencies = [ [[package]] name = "object_store_opendal" -version = "0.57.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8" +version = "0.58.0" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8" +checksum = "88f165780495c17aa3ce86846600504198c3fffd99073521552751c2430fa6ac" dependencies = [ "async-trait", "bytes", -@@ -4908,12 +4991,13 @@ +@@ -4718,12 +4811,13 @@ checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" [[package]] name = "opendal" -version = "0.57.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" +version = "0.58.2" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" +checksum = "33dbff14cc9bb085224256d6a81289d2f3202e85b06f408d42534b42162a4231" dependencies = [ "ctor 1.0.13", @@ -878,15 +903,14 @@ diff --git a/Cargo.lock b/Cargo.lock "opendal-layer-concurrent-limit", "opendal-layer-logging", "opendal-layer-retry", -@@ -4931,24 +5015,22 @@ +@@ -4741,24 +4835,22 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.57.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309" +version = "0.58.2" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309" +checksum = "48dbcef97d3eb7591db2c18d5cae95c836bcce07359b98d98dd6f4e861eb77b7" dependencies = [ "anyhow", @@ -909,91 +933,84 @@ diff --git a/Cargo.lock b/Cargo.lock "serde", "serde_json", "tokio", -@@ -4958,22 +5040,36 @@ +@@ -4767,23 +4859,37 @@ dependencies = [ + "web-time", ] - [[package]] --name = "opendal-layer-concurrent-limit" --version = "0.57.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f" --dependencies = [ ++[[package]] +name = "opendal-http-transport-reqwest" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85663452ea32bbc17e8f79ab29788c846d116ec7de31451be9c787e462dcb36c" +dependencies = [ + "bytes", - "futures", - "http 1.4.0", -- "mea", ++ "futures", ++ "http 1.4.0", + "http-body 1.0.1", - "opendal-core", ++ "opendal-core", + "reqwest 0.13.4", +] + -+[[package]] -+name = "opendal-layer-concurrent-limit" + [[package]] + name = "opendal-layer-concurrent-limit" +-version = "0.57.0" +version = "0.58.2" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f" +checksum = "03f9e144b5228d741c3763ade8711d9b72e5fb6d998e779f2d7a09da0b5a3eba" -+dependencies = [ + dependencies = [ + "asyncband", -+ "futures", -+ "http 1.4.0", -+ "opendal-core", + "futures", + "http 1.4.0", +- "mea", + "opendal-core", ] [[package]] name = "opendal-layer-logging" -version = "0.57.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2" +version = "0.58.2" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2" +checksum = "c2de17c61cd32e9d8d7d8efb91795e714dbccbafbc3c4e219e1542f4d6324161" dependencies = [ "log", "opendal-core", -@@ -4981,9 +5077,9 @@ +@@ -4791,9 +4897,9 @@ dependencies = [ [[package]] name = "opendal-layer-retry" -version = "0.57.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7" +version = "0.58.2" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7" +checksum = "e94db301964a25366090484d61e6da16d5979cc8faf02c3e12210dc74fafed38" dependencies = [ "backon", "log", -@@ -4992,9 +5088,9 @@ +@@ -4802,9 +4908,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.57.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3" +version = "0.58.2" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3" +checksum = "08956ddda07465449bfd48825f4f0f25e0351278ac974eaa659895d9d74f2c80" dependencies = [ "opendal-core", "tokio", -@@ -5002,17 +5098,17 @@ +@@ -4812,17 +4918,17 @@ dependencies = [ [[package]] name = "opendal-service-azblob" -version = "0.57.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a" --dependencies = [ -- "base64", +version = "0.58.2" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a" +checksum = "6d278d2fb57661947d1c9fb44047e432b2782c48dc085b7abc99fe6e18c26cf8" -+dependencies = [ + dependencies = [ +- "base64", + "base64 0.23.1", "bytes", "http 1.4.0", @@ -1005,19 +1022,17 @@ diff --git a/Cargo.lock b/Cargo.lock "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", -@@ -5023,17 +5119,18 @@ +@@ -4833,17 +4939,18 @@ dependencies = [ [[package]] name = "opendal-service-azdls" -version = "0.57.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361" --dependencies = [ -- "base64", +version = "0.58.2" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361" +checksum = "2d564484a8f7d091827e825cfc91ed45bd48e64d262451ee041fe843db81bd8a" -+dependencies = [ + dependencies = [ +- "base64", + "asyncband", + "base64 0.23.1", "bytes", @@ -1030,28 +1045,26 @@ diff --git a/Cargo.lock b/Cargo.lock "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", -@@ -5043,9 +5140,9 @@ +@@ -4853,9 +4960,9 @@ dependencies = [ [[package]] name = "opendal-service-azure-common" -version = "0.57.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" +version = "0.58.2" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" +checksum = "cfcc1bfdac4f54d9018c462dd32ad9e2f68fdf584f825c811550c8ffc87894cd" dependencies = [ "http 1.4.0", "opendal-core", -@@ -5053,15 +5150,15 @@ +@@ -4863,15 +4970,15 @@ dependencies = [ [[package]] name = "opendal-service-cos" -version = "0.57.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901" +version = "0.58.2" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901" +checksum = "bb021c128ebde42e6f3e719d4cfed27e994017aa503a7a1e5818bdb61fd67dc9" dependencies = [ "bytes", @@ -1063,20 +1076,19 @@ diff --git a/Cargo.lock b/Cargo.lock "reqsign-core", "reqsign-file-read-tokio", "reqsign-tencent-cos", -@@ -5070,9 +5167,9 @@ +@@ -4880,9 +4987,9 @@ dependencies = [ [[package]] name = "opendal-service-gcs" -version = "0.57.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47" +version = "0.58.2" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47" +checksum = "da1f2a8c975fd22fea01f0409bcb8774f1ac02ad79863a3490098203121555d3" dependencies = [ "async-trait", "bytes", -@@ -5080,7 +5177,7 @@ +@@ -4890,7 +4997,7 @@ dependencies = [ "log", "opendal-core", "percent-encoding", @@ -1085,33 +1097,31 @@ diff --git a/Cargo.lock b/Cargo.lock "reqsign-core", "reqsign-file-read-tokio", "reqsign-google", -@@ -5091,9 +5188,9 @@ +@@ -4901,9 +5008,9 @@ dependencies = [ [[package]] name = "opendal-service-goosefs" -version = "0.57.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4" +version = "0.58.2" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4" +checksum = "89fd71b80078f2983bd363e322fbebfa6a46d5fc56f17e4f23d76d5edb31226b" dependencies = [ "bytes", "goosefs-sdk", -@@ -5105,9 +5202,9 @@ +@@ -4915,9 +5022,9 @@ dependencies = [ [[package]] name = "opendal-service-hf" -version = "0.57.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848" +version = "0.58.2" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848" +checksum = "c8a3c8ec0c2918fa23f258fa28822f222ec8c1e3a501ded665b78bc65a9d7734" dependencies = [ "bytes", "hf-xet", -@@ -5115,22 +5212,21 @@ +@@ -4925,22 +5032,21 @@ dependencies = [ "log", "opendal-core", "percent-encoding", @@ -1123,10 +1133,9 @@ diff --git a/Cargo.lock b/Cargo.lock [[package]] name = "opendal-service-oss" -version = "0.57.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace" +version = "0.58.2" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace" +checksum = "8e3ce7a2ceb925e0f28f545b169eb57d04cec6116aae94b8584d2bdbe7d456c6" dependencies = [ "bytes", @@ -1138,19 +1147,17 @@ diff --git a/Cargo.lock b/Cargo.lock "reqsign-aliyun-oss", "reqsign-core", "reqsign-file-read-tokio", -@@ -5139,18 +5235,18 @@ +@@ -4949,18 +5055,18 @@ dependencies = [ [[package]] name = "opendal-service-s3" -version = "0.57.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb" --dependencies = [ -- "base64", +version = "0.58.2" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb" +checksum = "c64335f9f24ccb62ac36f1d976342b48611a75ba61979813a4f78a4ebd94de42" -+dependencies = [ + dependencies = [ +- "base64", + "base64 0.23.1", "bytes", - "crc32c", @@ -1164,15 +1171,14 @@ diff --git a/Cargo.lock b/Cargo.lock "reqsign-aws-v4", "reqsign-core", "reqsign-file-read-tokio", -@@ -5160,14 +5256,14 @@ +@@ -4970,14 +5076,14 @@ dependencies = [ [[package]] name = "opendal-service-tos" -version = "0.57.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "6f2f7a4c32e5202eb4ac72e76c4b5e30c86ab60762811172f4111103b9d673a1" +version = "0.58.2" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "6f2f7a4c32e5202eb4ac72e76c4b5e30c86ab60762811172f4111103b9d673a1" +checksum = "70c3c507c3a565b2feb4c7b5f653436acd34ca59ffc842a7671b5498b13b76bf" dependencies = [ "bytes", @@ -1183,7 +1189,7 @@ diff --git a/Cargo.lock b/Cargo.lock "reqsign-core", "reqsign-file-read-tokio", "reqsign-volcengine-tos", -@@ -5274,7 +5370,7 @@ +@@ -5084,7 +5190,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "898bac3fa00d0ba57a4e8289837e965baa2dee8c3749f3b11d45a64b4223d9c3" dependencies = [ @@ -1192,7 +1198,7 @@ diff --git a/Cargo.lock b/Cargo.lock "serde", ] -@@ -5312,16 +5408,16 @@ +@@ -5122,16 +5228,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ "digest 0.10.7", @@ -1203,24 +1209,20 @@ diff --git a/Cargo.lock b/Cargo.lock [[package]] name = "pem" -version = "3.0.6" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" --dependencies = [ -- "base64", +version = "4.0.0" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" -+dependencies = [ + dependencies = [ +- "base64", + "base64 0.23.1", "serde_core", ] -@@ -5580,6 +5676,28 @@ - dependencies = [ - "memchr", +@@ -5392,6 +5498,28 @@ dependencies = [ "serde", -+] -+ + ] + +[[package]] +name = "quick-xml" +version = "0.41.0" @@ -1241,10 +1243,12 @@ diff --git a/Cargo.lock b/Cargo.lock + "equivalent", + "hashbrown 0.16.1", + "parking_lot", - ] - ++] ++ [[package]] -@@ -5806,7 +5924,7 @@ + name = "quinn" + version = "0.11.9" +@@ -5616,7 +5744,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ @@ -1253,34 +1257,31 @@ diff --git a/Cargo.lock b/Cargo.lock ] [[package]] -@@ -5887,9 +6005,9 @@ +@@ -5697,9 +5825,9 @@ dependencies = [ [[package]] name = "reqsign-aliyun-oss" -version = "3.0.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "57ac2757f3140aa2e213b554148ae0b52733e624fc6723f0cc6bb3d440176c95" +version = "3.1.4" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "57ac2757f3140aa2e213b554148ae0b52733e624fc6723f0cc6bb3d440176c95" +checksum = "68d24d281f734a463093b7b93aae8b16f5f8496a54fbeec4d2d10b0488295296" dependencies = [ "anyhow", "form_urlencoded", -@@ -5903,18 +6021,18 @@ +@@ -5713,18 +5841,18 @@ dependencies = [ ] [[package]] -name = "reqsign-aws-v4" -version = "3.0.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "44eaca382e94505a49f1a4849658d153aebf79d9c1a58e5dd3b10361511e9f43" --dependencies = [ -- "anyhow", +name = "reqsign-aws-core" +version = "3.1.0" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "44eaca382e94505a49f1a4849658d153aebf79d9c1a58e5dd3b10361511e9f43" +checksum = "4d63b56638bb3cc7bd376a7cdce1ba3089777a08f47e4097888f2d784cc3f46c" -+dependencies = [ + dependencies = [ +- "anyhow", "bytes", "form_urlencoded", + "hex", @@ -1292,10 +1293,11 @@ diff --git a/Cargo.lock b/Cargo.lock "reqsign-core", "rust-ini", "serde", -@@ -5924,17 +6042,31 @@ +@@ -5733,18 +5861,32 @@ dependencies = [ + "sha1", ] - [[package]] ++[[package]] +name = "reqsign-aws-v4" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1310,13 +1312,12 @@ diff --git a/Cargo.lock b/Cargo.lock + "serde", +] + -+[[package]] + [[package]] name = "reqsign-azure-storage" -version = "3.0.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "7a321980405d596bd34aaf95c4722a3de4128a67fd19e74a81a83aa3fdf082e6" +version = "3.2.0" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "7a321980405d596bd34aaf95c4722a3de4128a67fd19e74a81a83aa3fdf082e6" +checksum = "e8177b4f08620ab7f2e9cab7d7ccb9da1b61c66b889fed46cc0880b0fe75eb6b" dependencies = [ "anyhow", @@ -1329,15 +1330,14 @@ diff --git a/Cargo.lock b/Cargo.lock "log", "pem", "percent-encoding", -@@ -5947,31 +6079,33 @@ +@@ -5757,31 +5899,33 @@ dependencies = [ [[package]] name = "reqsign-core" -version = "3.0.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "b10302cf0a7d7e7352ba211fc92c3c5bebf1286153e49cc5aa87348078a8e102" +version = "3.3.0" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "b10302cf0a7d7e7352ba211fc92c3c5bebf1286153e49cc5aa87348078a8e102" +checksum = "f4ac1510872d9481205975d264deb39c109797e5068cc882ed9064270eaae5fa" dependencies = [ "anyhow", @@ -1365,26 +1365,23 @@ diff --git a/Cargo.lock b/Cargo.lock [[package]] name = "reqsign-file-read-tokio" -version = "3.0.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "e2d89295b3d17abea31851cc8de55d843d89c52132c864963c38d41920613dc5" +version = "3.0.5" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "e2d89295b3d17abea31851cc8de55d843d89c52132c864963c38d41920613dc5" +checksum = "95c3371bfc7e5c7f9627a04133af3583fd6c28715e7c83f79db38f3b384f535f" dependencies = [ "anyhow", "reqsign-core", -@@ -5980,13 +6114,13 @@ +@@ -5790,13 +5934,13 @@ dependencies = [ [[package]] name = "reqsign-google" -version = "3.0.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "35cc609b49c69e76ecaceb775a03f792d1ed3e7755ab3548d4534fd801e3242e" --dependencies = [ +version = "3.1.0" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "35cc609b49c69e76ecaceb775a03f792d1ed3e7755ab3548d4534fd801e3242e" +checksum = "f81a9d38870892443489c0abb5332edfa81d5a14c437af9caef7c194897c92c1" -+dependencies = [ + dependencies = [ + "bytes", "form_urlencoded", "http 1.4.0", @@ -1392,7 +1389,7 @@ diff --git a/Cargo.lock b/Cargo.lock "log", "percent-encoding", "reqsign-aws-v4", -@@ -5994,15 +6128,14 @@ +@@ -5804,15 +5948,14 @@ dependencies = [ "rsa", "serde", "serde_json", @@ -1403,28 +1400,26 @@ diff --git a/Cargo.lock b/Cargo.lock [[package]] name = "reqsign-tencent-cos" -version = "3.0.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "e128f19525861dbded59e1e7c17653a8ed63d573ca04aed708d552dbef5bb32a" +version = "3.0.5" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "e128f19525861dbded59e1e7c17653a8ed63d573ca04aed708d552dbef5bb32a" +checksum = "b15c5a4df7c3f16823ae242675c5ebfb52d640cc9a50d1fcf943263247fa1730" dependencies = [ "anyhow", "http 1.4.0", -@@ -6015,9 +6148,9 @@ +@@ -5825,9 +5968,9 @@ dependencies = [ [[package]] name = "reqsign-volcengine-tos" -version = "3.0.0" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "f9d757602a7ef2b6025c0da77e6d2e23fbdef35930fa466b15ffbf0a3f13acf7" +version = "3.1.1" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "f9d757602a7ef2b6025c0da77e6d2e23fbdef35930fa466b15ffbf0a3f13acf7" +checksum = "173387eb5ae4cf6a0a7098665ebcc6729862819dc3d95a81aee840767803e3d9" dependencies = [ "anyhow", "http 1.4.0", -@@ -6032,7 +6165,7 @@ +@@ -5842,7 +5985,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ @@ -1433,24 +1428,22 @@ diff --git a/Cargo.lock b/Cargo.lock "bytes", "encoding_rs", "futures-core", -@@ -6074,11 +6207,11 @@ +@@ -5884,11 +6027,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.13.3" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" --dependencies = [ -- "base64", +version = "0.13.4" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" -+dependencies = [ + dependencies = [ +- "base64", + "base64 0.22.1", "bytes", "futures-core", "futures-util", -@@ -6121,7 +6254,7 @@ +@@ -5931,7 +6074,7 @@ dependencies = [ "anyhow", "async-trait", "http 1.4.0", @@ -1459,7 +1452,7 @@ diff --git a/Cargo.lock b/Cargo.lock "thiserror 2.0.18", "tower-service", ] -@@ -6136,7 +6269,7 @@ +@@ -5946,7 +6089,7 @@ dependencies = [ "cfg-if 1.0.4", "getrandom 0.2.17", "libc", @@ -1468,10 +1461,11 @@ diff --git a/Cargo.lock b/Cargo.lock "windows-sys 0.52.0", ] -@@ -6199,16 +6332,6 @@ +@@ -6008,16 +6151,6 @@ dependencies = [ + "ordered-multimap", ] - [[package]] +-[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1481,11 +1475,10 @@ diff --git a/Cargo.lock b/Cargo.lock - "serde_derive", -] - --[[package]] + [[package]] name = "rustc-hash" version = "2.1.1" - source = "registry+https://github.com/rust-lang/crates.io-index" -@@ -6229,7 +6352,7 @@ +@@ -6039,7 +6172,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ @@ -1494,7 +1487,7 @@ diff --git a/Cargo.lock b/Cargo.lock "errno", "libc", "linux-raw-sys", -@@ -6309,7 +6432,7 @@ +@@ -6119,7 +6252,7 @@ dependencies = [ "aws-lc-rs", "ring", "rustls-pki-types", @@ -1503,7 +1496,7 @@ diff --git a/Cargo.lock b/Cargo.lock ] [[package]] -@@ -6434,7 +6557,7 @@ +@@ -6244,7 +6377,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ @@ -1512,7 +1505,7 @@ diff --git a/Cargo.lock b/Cargo.lock "core-foundation 0.10.1", "core-foundation-sys", "libc", -@@ -6563,7 +6686,7 @@ +@@ -6373,7 +6506,7 @@ version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" dependencies = [ @@ -1521,15 +1514,14 @@ diff --git a/Cargo.lock b/Cargo.lock "bs58", "chrono", "hex", -@@ -6604,13 +6727,13 @@ +@@ -6414,13 +6547,13 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.6" --source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +version = "0.11.0" -+source = "registry+https://github.com/rust-lang/crates.io-index" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if 1.0.4", @@ -1540,10 +1532,11 @@ diff --git a/Cargo.lock b/Cargo.lock ] [[package]] -@@ -6714,18 +6837,6 @@ +@@ -6523,18 +6656,6 @@ version = "0.1.5" + source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" - [[package]] +-[[package]] -name = "simple_asn1" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1555,28 +1548,26 @@ diff --git a/Cargo.lock b/Cargo.lock - "time", -] - --[[package]] + [[package]] name = "siphasher" version = "1.0.2" +@@ -6602,6 +6723,12 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -@@ -6799,6 +6910,12 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" - [[package]] ++[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + -+[[package]] + [[package]] name = "spki" version = "0.7.3" - source = "registry+https://github.com/rust-lang/crates.io-index" -@@ -6877,28 +6994,6 @@ - version = "0.11.1" +@@ -6682,28 +6809,6 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -- + -[[package]] -name = "strum" -version = "0.26.3" @@ -1598,10 +1589,11 @@ diff --git a/Cargo.lock b/Cargo.lock - "rustversion", - "syn 2.0.117", -] - +- [[package]] name = "substrait" -@@ -7000,7 +7095,7 @@ + version = "0.63.0" +@@ -6804,7 +6909,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ @@ -1610,7 +1602,7 @@ diff --git a/Cargo.lock b/Cargo.lock "core-foundation 0.9.4", "system-configuration-sys", ] -@@ -7274,7 +7369,7 @@ +@@ -7078,7 +7183,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", @@ -1619,7 +1611,7 @@ diff --git a/Cargo.lock b/Cargo.lock "bytes", "h2", "http 1.4.0", -@@ -7332,7 +7427,7 @@ +@@ -7136,7 +7241,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ "async-compression", @@ -1628,20 +1620,20 @@ diff --git a/Cargo.lock b/Cargo.lock "bytes", "futures-core", "futures-util", -@@ -7568,12 +7663,6 @@ +@@ -7370,12 +7475,6 @@ version = "0.2.11" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - [[package]] - name = "untrusted" +-[[package]] +-name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - --[[package]] --name = "untrusted" + [[package]] + name = "untrusted" version = "0.9.0" - source = "registry+https://github.com/rust-lang/crates.io-index" - checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -@@ -7818,7 +7907,7 @@ +@@ -7622,7 +7721,7 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ @@ -1650,7 +1642,7 @@ diff --git a/Cargo.lock b/Cargo.lock "hashbrown 0.15.5", "indexmap 2.14.0", "semver", -@@ -8259,7 +8348,7 @@ +@@ -8054,7 +8153,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", @@ -1659,7 +1651,7 @@ diff --git a/Cargo.lock b/Cargo.lock "indexmap 2.14.0", "log", "serde", -@@ -8337,7 +8426,7 @@ +@@ -8132,7 +8231,7 @@ checksum = "3e1e496dcbe6a09017acdfaf48e1a646735e7ff5b2a49e2c7e081cca77a59bc8" dependencies = [ "anyhow", "async-trait", @@ -1668,7 +1660,7 @@ diff --git a/Cargo.lock b/Cargo.lock "bytes", "clap", "crc32fast", -@@ -8348,7 +8437,7 @@ +@@ -8143,7 +8242,7 @@ dependencies = [ "more-asserts", "rand 0.10.1", "redb", @@ -1677,7 +1669,7 @@ diff --git a/Cargo.lock b/Cargo.lock "reqwest-middleware", "serde", "serde_json", -@@ -8374,7 +8463,7 @@ +@@ -8169,7 +8268,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb838aa8eb67d730af301584cf003caad407487606058292a6750711b603fbee" dependencies = [ "async-trait", @@ -1686,7 +1678,7 @@ diff --git a/Cargo.lock b/Cargo.lock "blake3", "bytemuck", "bytes", -@@ -8461,7 +8550,7 @@ +@@ -8256,7 +8355,7 @@ dependencies = [ "oneshot", "pin-project", "rand 0.10.1", @@ -1695,7 +1687,7 @@ diff --git a/Cargo.lock b/Cargo.lock "serde", "serde_json", "shellexpand", -@@ -8557,20 +8646,6 @@ +@@ -8352,20 +8451,6 @@ name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" @@ -1717,9 +1709,10 @@ diff --git a/Cargo.lock b/Cargo.lock [[package]] name = "zerotrie" diff --git a/Cargo.toml b/Cargo.toml +index d072a5d..3920a65 100644 --- a/Cargo.toml +++ b/Cargo.toml -@@ -18,14 +18,14 @@ +@@ -18,14 +18,14 @@ rust-version = "1.91.0" crate-type = ["cdylib", "staticlib", "rlib"] [dependencies] @@ -1742,7 +1735,7 @@ diff --git a/Cargo.toml b/Cargo.toml datafusion = { version = "54.0.0", default-features = false } arrow = { version = "58.0.0", features = ["prettyprint", "ffi"] } arrow-array = "58.0.0" -@@ -49,9 +49,9 @@ +@@ -45,9 +45,9 @@ snafu = "0.9" uuid = { version = "1", features = ["v4"] } [dev-dependencies] @@ -1756,9 +1749,10 @@ diff --git a/Cargo.toml b/Cargo.toml arrow-array = "58.0.0" arrow-schema = "58.0.0" diff --git a/src/fts_query.rs b/src/fts_query.rs +index c9d3a43..71874e0 100644 --- a/src/fts_query.rs +++ b/src/fts_query.rs -@@ -246,7 +246,7 @@ +@@ -246,7 +246,7 @@ async fn prepare_fts_query_context( .with_max_expansions(match_query.max_expansions) .with_prefix_length(match_query.prefix_length); PreparedFtsQuery::Match(Arc::new( @@ -1767,7 +1761,7 @@ diff --git a/src/fts_query.rs b/src/fts_query.rs )) } FtsQuery::Phrase(phrase_query) => { -@@ -260,7 +260,7 @@ +@@ -260,7 +260,7 @@ async fn prepare_fts_query_context( let query_tokens = collect_query_tokens(&phrase_query.terms, &mut tokenizer); let params = query.params().with_phrase_slop(Some(phrase_query.slop)); PreparedFtsQuery::Phrase(Arc::new( @@ -1777,9 +1771,10 @@ diff --git a/src/fts_query.rs b/src/fts_query.rs } _ => { diff --git a/src/index_segment.rs b/src/index_segment.rs +index a46c4f3..d4e143c 100644 --- a/src/index_segment.rs +++ b/src/index_segment.rs -@@ -889,7 +889,7 @@ +@@ -889,7 +889,7 @@ unsafe fn new_vector_builder_inner( // TODO(upstream-lance): Remove this fail-fast once Lance's distributed // vector-index path reconstructs a supplied PQ codebook with an L2 // ProductQuantizer, matching the ordinary full-dataset path. Pinned Lance @@ -1788,7 +1783,7 @@ diff --git a/src/index_segment.rs b/src/index_segment.rs // `make_global_pq`, which silently switches PQ code assignment away from // the L2 contract shared by full-dataset builds and index readers. if matches!( -@@ -910,7 +910,7 @@ +@@ -910,7 +910,7 @@ unsafe fn new_vector_builder_inner( let selected_fragment_ids: HashSet = fragment_ids.iter().copied().collect(); if selected_fragment_ids != all_fragment_ids { return Err(invalid_input(format!( @@ -1798,9 +1793,10 @@ diff --git a/src/index_segment.rs b/src/index_segment.rs parsed.mode, selected_fragment_ids.len(), diff --git a/src/scanner.rs b/src/scanner.rs +index 7e898ce..d3ef3be 100644 --- a/src/scanner.rs +++ b/src/scanner.rs -@@ -661,7 +661,7 @@ +@@ -529,7 +529,7 @@ fn rewrite_prepared_fts_plan( exec.params().clone(), exec.prefilter_source().clone(), segments.to_vec(), @@ -1809,7 +1805,7 @@ diff --git a/src/scanner.rs b/src/scanner.rs .with_base_scorer(Arc::clone(scorer)); return Ok((Arc::new(replacement), rewritten)); } -@@ -681,7 +681,7 @@ +@@ -549,7 +549,7 @@ fn rewrite_prepared_fts_plan( exec.params().clone(), exec.prefilter_source().clone(), segments.to_vec(), @@ -1819,9 +1815,10 @@ diff --git a/src/scanner.rs b/src/scanner.rs return Ok((Arc::new(replacement), rewritten)); } diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index b763cef..bde742d 100644 --- a/tests/c_api_test.rs +++ b/tests/c_api_test.rs -@@ -2845,8 +2845,7 @@ +@@ -2449,8 +2449,7 @@ fn test_robotics_e2e_write_then_finalize() { format!("data/{}", filename), field_ids, column_indices, @@ -1831,7 +1828,7 @@ diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs None, // file_size_bytes None, // base_id ); -@@ -3919,6 +3918,7 @@ +@@ -3405,6 +3404,7 @@ fn test_index_segment_metadata_parse_rejects_malformed_and_dangerous_input() { created_at: Some(u64::MAX), base_id: None, files: Vec::new(), @@ -1839,7 +1836,7 @@ diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs } .encode_to_vec(); assert_eq!( -@@ -3943,6 +3943,7 @@ +@@ -3429,6 +3429,7 @@ fn test_index_segment_metadata_parse_rejects_malformed_and_dangerous_input() { created_at: None, base_id: None, files: Vec::new(), @@ -1847,7 +1844,7 @@ diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs } .encode_to_vec(); assert_eq!( -@@ -3970,6 +3971,7 @@ +@@ -3456,6 +3457,7 @@ fn test_index_segment_metadata_parse_rejects_malformed_and_dangerous_input() { created_at: None, base_id: None, files: Vec::new(), @@ -1855,7 +1852,7 @@ diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs } .encode_to_vec(); assert_eq!( -@@ -4547,7 +4549,7 @@ +@@ -4033,7 +4035,7 @@ fn test_vector_index_segment_rejects_strict_subset_dot_pq() { let message = take_last_error_message(); assert!(message.contains("metric=DOT"), "{message}"); assert!(message.contains("strict fragment subset"), "{message}"); From 7dfe2eba49f96a7550c317c91dbfb81e90a7faf8 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Wed, 9 Sep 2026 21:41:50 +0800 Subject: [PATCH 06/12] formatter --- be/src/common/config.cpp | 4 ++-- be/src/format_v2/lance/lance_session_manager.cpp | 12 +++++------- be/src/format_v2/table/lance_reader.cpp | 12 ++++++------ .../format_v2/lance/lance_session_manager_test.cpp | 8 +++----- be/test/format_v2/table/lance_reader_test.cpp | 10 +++++----- .../doris/datasource/lance/source/LanceScanNode.java | 3 ++- .../java/org/apache/doris/qe/SessionVariable.java | 3 ++- 7 files changed, 25 insertions(+), 27 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 94ac90b0d2cf82..5c2e0ebf8d63f1 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1212,12 +1212,12 @@ DEFINE_Validator(variant_storage_parse_mode, // Lance uses one BE-wide session so metadata/index caches and the optional Foyer data-file cache // can be shared by all Lance dataset readers. -DEFINE_Int64(lance_index_cache_size_bytes, "10737418240"); // 10 GiB +DEFINE_Int64(lance_index_cache_size_bytes, "10737418240"); // 10 GiB DEFINE_Int64(lance_metadata_cache_size_bytes, "1073741824"); // 1GB DEFINE_Bool(enable_lance_data_cache, "true"); DEFINE_String(lance_data_cache_path, "${DORIS_HOME}/lance_data_cache"); DEFINE_Int64(lance_data_cache_disk_capacity_bytes, "107374182400"); // 100GB -DEFINE_Int64(lance_data_cache_read_block_size_bytes, "1048576"); // 1MB +DEFINE_Int64(lance_data_cache_read_block_size_bytes, "1048576"); // 1MB // I/O buffering budget per Lance scanner, not a cap on its total memory usage. // Runtime changes apply to newly created scanners. diff --git a/be/src/format_v2/lance/lance_session_manager.cpp b/be/src/format_v2/lance/lance_session_manager.cpp index c10142f9e05f45..58b52f398fa069 100644 --- a/be/src/format_v2/lance/lance_session_manager.cpp +++ b/be/src/format_v2/lance/lance_session_manager.cpp @@ -153,16 +153,14 @@ LanceSessionManager& LanceSessionManager::instance() { LanceSessionManager::LanceSessionManager(Config config) : _config(std::move(config)) { LOG(INFO) << "Creating BE-wide Lance session manager: lance_index_cache_size_bytes=" << _config.lance_index_cache_size_bytes - << ", lance_metadata_cache_size_bytes=" - << _config.lance_metadata_cache_size_bytes + << ", lance_metadata_cache_size_bytes=" << _config.lance_metadata_cache_size_bytes << ", enable_lance_data_cache=" << _config.enable_lance_data_cache << ", lance_data_cache_path=" << _config.lance_data_cache_path << ", lance_data_cache_disk_capacity_bytes=" << _config.lance_data_cache_disk_capacity_bytes << ", lance_data_cache_read_block_size_bytes=" << _config.lance_data_cache_read_block_size_bytes - << ", foyer_memory_capacity_bytes=" - << _config.lance_data_cache_read_block_size_bytes; + << ", foyer_memory_capacity_bytes=" << _config.lance_data_cache_read_block_size_bytes; } LanceSessionManager::~LanceSessionManager() { @@ -189,9 +187,9 @@ Status LanceSessionManager::_initialize() { static_cast(_config.lance_metadata_cache_size_bytes), &data_cache_options); } else { - _session = lance_session_new( - static_cast(_config.lance_index_cache_size_bytes), - static_cast(_config.lance_metadata_cache_size_bytes)); + _session = + lance_session_new(static_cast(_config.lance_index_cache_size_bytes), + static_cast(_config.lance_metadata_cache_size_bytes)); } if (_session == nullptr) { return lance_error("create shared Lance session"); diff --git a/be/src/format_v2/table/lance_reader.cpp b/be/src/format_v2/table/lance_reader.cpp index fb1bb182674276..68b97797b8cdad 100644 --- a/be/src/format_v2/table/lance_reader.cpp +++ b/be/src/format_v2/table/lance_reader.cpp @@ -141,12 +141,12 @@ Status LanceTableReader::init(TableReadOptions&& options) { TUnit::UNIT, LANCE_READER_PROFILE, 1); _execution_bytes_read = ADD_CHILD_COUNTER_WITH_LEVEL( _scanner_profile, "LanceExecutionIOBytesRead", TUnit::BYTES, LANCE_READER_PROFILE, 1); - _data_cache_bytes_read_from_cache = ADD_CHILD_COUNTER_WITH_LEVEL( - _scanner_profile, "LanceDataCacheBytesReadFromCache", TUnit::BYTES, - LANCE_READER_PROFILE, 1); - _data_cache_bytes_read_from_remote = ADD_CHILD_COUNTER_WITH_LEVEL( - _scanner_profile, "LanceDataCacheBytesReadFromRemote", TUnit::BYTES, - LANCE_READER_PROFILE, 1); + _data_cache_bytes_read_from_cache = + ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LanceDataCacheBytesReadFromCache", + TUnit::BYTES, LANCE_READER_PROFILE, 1); + _data_cache_bytes_read_from_remote = + ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LanceDataCacheBytesReadFromRemote", + TUnit::BYTES, LANCE_READER_PROFILE, 1); _index_partition_cache_miss_loads = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LanceIndexPartitionCacheMissLoads", TUnit::UNIT, LANCE_READER_PROFILE, 1); diff --git a/be/test/format_v2/lance/lance_session_manager_test.cpp b/be/test/format_v2/lance/lance_session_manager_test.cpp index 9e873f61d8c938..7d9975fc0066b9 100644 --- a/be/test/format_v2/lance/lance_session_manager_test.cpp +++ b/be/test/format_v2/lance/lance_session_manager_test.cpp @@ -57,9 +57,7 @@ TEST(LanceSessionManagerTest, SessionAndDataCacheConfigurationsAreIndependent) { }; LanceSessionManager manager(std::move(config)); LanceDataset* raw_dataset = nullptr; - ASSERT_TRUE(manager - .open_dataset(lance_fixture_path().c_str(), nullptr, 0, &raw_dataset) - .ok()); + ASSERT_TRUE(manager.open_dataset(lance_fixture_path().c_str(), nullptr, 0, &raw_dataset).ok()); LanceDatasetPtr dataset(raw_dataset, lance_dataset_close); ASSERT_NE(dataset, nullptr); } @@ -117,8 +115,8 @@ TEST(LanceSessionManagerTest, CreatesFoyerBackedSession) { { LanceSessionManager manager(std::move(config)); LanceDataset* raw_dataset = nullptr; - const auto status = manager.open_dataset(lance_fixture_path().c_str(), nullptr, 0, - &raw_dataset); + const auto status = + manager.open_dataset(lance_fixture_path().c_str(), nullptr, 0, &raw_dataset); LanceDatasetPtr dataset(raw_dataset, lance_dataset_close); EXPECT_TRUE(status.ok()) << status; EXPECT_NE(dataset, nullptr); diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp index 9fac579b92d919..a6dd9b444f2fcf 100644 --- a/be/test/format_v2/table/lance_reader_test.cpp +++ b/be/test/format_v2/table/lance_reader_test.cpp @@ -1055,11 +1055,11 @@ TEST(LanceTableReaderVectorSearchTest, ReturnsStableGlobalRowIdsAndFetchesPayloa EXPECT_NE(fetch_profile.get_counter("LanceRowIdTakeReadTime"), nullptr); EXPECT_NE(fetch_profile.get_counter("LanceArrowToDorisBlockTime"), nullptr); EXPECT_NE(fetch_profile.get_counter("LanceRowIdFetchTotalTime"), nullptr); - expect_lance_profile_hierarchy(&fetch_profile, - {"LanceDatasetOpenTime", "LanceRowIdTakeReadTime", - "LanceArrowToDorisBlockTime", "LanceRowIdFetchTotalTime", - "LanceDataCacheBytesReadFromCache", - "LanceDataCacheBytesReadFromRemote"}); + expect_lance_profile_hierarchy( + &fetch_profile, + {"LanceDatasetOpenTime", "LanceRowIdTakeReadTime", "LanceArrowToDorisBlockTime", + "LanceRowIdFetchTotalTime", "LanceDataCacheBytesReadFromCache", + "LanceDataCacheBytesReadFromRemote"}); EXPECT_TRUE(payload_reader.close().ok()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java index 45b37560f3f633..52c9e064371ace 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java @@ -716,7 +716,8 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { result.append(prefix).append("lanceScalarIndexScan=SEGMENT\n"); result.append(prefix).append("lanceGroupingIndex=").append(scalarIndexPlan.indexName).append("\n"); result.append(prefix).append("lanceGroupingIndexSegments=").append(plannedIndexSegments).append("\n"); - result.append(prefix).append("lanceGroupingIndexedFragments=").append(plannedIndexFragments).append("\n"); + result.append(prefix).append("lanceGroupingIndexedFragments=") + .append(plannedIndexFragments).append("\n"); result.append(prefix).append("lanceGroupingUnindexedFragments=") .append(plannedUnindexedFragments).append("\n"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index b06f2cc08b2138..4432844120bdaa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -2591,7 +2591,8 @@ public Map getForceEagerAggHintMap() { description = { "是否启用 Lance 两阶段延迟读取，默认开启，不受 topn_lazy_materialization_threshold 控制。" + "当前支持 vector_search 和 full_text_search 中可安全延迟读取的列。", - "Enable Lance two-phase lazy materialization, independently of topn_lazy_materialization_threshold. " + "Enable Lance two-phase lazy materialization, " + + "independently of topn_lazy_materialization_threshold. " + "Enabled by default for eligible columns in vector_search and full_text_search."}) public boolean enableLanceLazyMaterialization = true; From 68a41b2dc59c56476301281d49c4fd2a5b0a6318 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Thu, 10 Sep 2026 11:46:56 +0800 Subject: [PATCH 07/12] update --- .../format_v2/lance/lance_session_manager.cpp | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/be/src/format_v2/lance/lance_session_manager.cpp b/be/src/format_v2/lance/lance_session_manager.cpp index 58b52f398fa069..884eb5db18c5e7 100644 --- a/be/src/format_v2/lance/lance_session_manager.cpp +++ b/be/src/format_v2/lance/lance_session_manager.cpp @@ -170,11 +170,15 @@ LanceSessionManager::~LanceSessionManager() { Status LanceSessionManager::_initialize() { if (_config.enable_lance_data_cache) { + // Treat the configured cache mode as a process-level requirement. If an enabled + // data cache cannot initialize, report the failure instead of silently creating a + // session without it. This keeps directory/configuration/device failures visible + // to the operator. Disabling the cache is an explicit configuration change. const LanceDataCacheOptions data_cache_options { .directory = _config.lance_data_cache_path.c_str(), // Foyer's HybridCache requires a memory tier. Keep it at the minimum useful - // capacity of exactly one range-cache block; entries use WriteOnInsertion and - // are persisted to the disk tier immediately. + // capacity of exactly one range-cache block; WriteOnInsertion enqueues disk + // writes on insertion rather than waiting for memory-tier eviction. .memory_capacity_bytes = static_cast(_config.lance_data_cache_read_block_size_bytes), .disk_capacity_bytes = @@ -192,7 +196,18 @@ Status LanceSessionManager::_initialize() { static_cast(_config.lance_metadata_cache_size_bytes)); } if (_session == nullptr) { - return lance_error("create shared Lance session"); + // Capture the Lance-C error on the initializing thread before another FFI call can + // replace it. Keep the original cause (including Foyer's directory/I/O details) and + // explain how to recover from the initialization status retained by call_once below. + auto status = lance_error("create shared Lance session"); + if (_config.enable_lance_data_cache) { + status.append( + "; Check the Lance data cache configuration and storage. After fixing the " + "issue, restart this BE. Alternatively, set enable_lance_data_cache=false " + "in be.conf and restart this BE. Session initialization will not be retried " + "in this BE process."); + } + return status; } _metrics = std::make_unique(_session, _config); return Status::OK(); @@ -205,6 +220,12 @@ Status LanceSessionManager::open_dataset(const char* uri, const char* const* sto } *dataset = nullptr; + // Initialize lazily on the first dataset open, not at BE startup. A failed Status is a + // normal return from this lambda, so call_once completes and retains that failure just + // like a successful initialization. All subsequent readers receive the same copied + // error; fixing the cache directory alone does not trigger another attempt. This is + // intentional: repair the cache configuration/storage (or disable the data cache), then + // restart the BE to recreate the process-wide manager and session. std::call_once(_initialize_once, [this] { _initialize_status = _initialize(); }); RETURN_IF_ERROR(_initialize_status); From 4b0553b46f1172057d76a77683c5b8db982192a2 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Thu, 10 Sep 2026 18:07:35 +0800 Subject: [PATCH 08/12] update --- .../lance/lance_session_manager_test.cpp | 3 + .../lance/LanceIndexMetadataLoader.java | 38 ++- thirdparty/download-thirdparty.sh | 4 +- thirdparty/patches/lance-c-0.1.9-pr-80.patch | 225 ++++++++++++++++++ 4 files changed, 255 insertions(+), 15 deletions(-) create mode 100644 thirdparty/patches/lance-c-0.1.9-pr-80.patch diff --git a/be/test/format_v2/lance/lance_session_manager_test.cpp b/be/test/format_v2/lance/lance_session_manager_test.cpp index 7d9975fc0066b9..e1274219347352 100644 --- a/be/test/format_v2/lance/lance_session_manager_test.cpp +++ b/be/test/format_v2/lance/lance_session_manager_test.cpp @@ -69,6 +69,9 @@ TEST(LanceSessionManagerTest, PublishesSessionCacheMetrics) { .lance_index_cache_size_bytes = INDEX_CACHE_CAPACITY, .lance_metadata_cache_size_bytes = METADATA_CACHE_CAPACITY, .enable_lance_data_cache = false, + .lance_data_cache_path = "", + .lance_data_cache_disk_capacity_bytes = 0, + .lance_data_cache_read_block_size_bytes = 0, }; LanceSessionManager manager(std::move(config)); LanceDataset* raw_dataset = nullptr; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexMetadataLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexMetadataLoader.java index fedfcee22ca862..7870237accc515 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexMetadataLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexMetadataLoader.java @@ -114,7 +114,7 @@ static List collectPhysicalEntries(Dataset dataset) { if (indexes.size() > MAX_PHYSICAL_INDEX_ENTRIES) { throw new IllegalArgumentException( "Lance physical index entry count exceeds limit " - + MAX_PHYSICAL_INDEX_ENTRIES); + + MAX_PHYSICAL_INDEX_ENTRIES + " (actual=" + indexes.size() + ")"); } List entries = new ArrayList<>(indexes.size()); Set seenUuids = new HashSet<>(); @@ -246,7 +246,7 @@ static List describeUserIndexes(Dataset dataset) { if (listedNames.size() > MAX_PHYSICAL_INDEX_ENTRIES) { throw new IllegalArgumentException( "Lance physical index entry count exceeds limit " - + MAX_PHYSICAL_INDEX_ENTRIES); + + MAX_PHYSICAL_INDEX_ENTRIES + " (actual=" + listedNames.size() + ")"); } Set userIndexNames = new LinkedHashSet<>(); @@ -259,7 +259,9 @@ static List describeUserIndexes(Dataset dataset) { userIndexNames.add(name); if (userIndexNames.size() > MAX_LOGICAL_INDEXES) { throw new IllegalArgumentException( - "Lance logical index count exceeds limit " + MAX_LOGICAL_INDEXES); + "Lance logical index count exceeds limit " + MAX_LOGICAL_INDEXES + + " (observed=" + userIndexNames.size() + + ", listed_entries=" + listedNames.size() + ")"); } } @@ -273,7 +275,8 @@ static List describeUserIndexes(Dataset dataset) { // A criteria query is an exact lookup; any other cardinality is inconsistent metadata. if (matching.size() != 1) { throw new IllegalArgumentException( - "Lance index criteria must return exactly one description"); + "Lance index criteria must return exactly one description (actual=" + + matching.size() + ")"); } IndexDescription description = matching.get(0); if (description == null) { @@ -297,7 +300,8 @@ static Map buildFieldNamesById(List fields) { } if (fields.size() > MAX_SCHEMA_FIELDS) { throw new IllegalArgumentException( - "Lance schema field count exceeds limit " + MAX_SCHEMA_FIELDS); + "Lance schema field count exceeds limit " + MAX_SCHEMA_FIELDS + + " (actual=" + fields.size() + ")"); } Map fieldNames = new HashMap<>(); SchemaTraversalState traversalState = new SchemaTraversalState(); @@ -311,7 +315,8 @@ private static void collectFieldNames(LanceField field, String parentPath, int d Map fieldNames, SchemaTraversalState traversalState) { if (depth > MAX_SCHEMA_DEPTH) { throw new IllegalArgumentException( - "Lance schema depth exceeds limit " + MAX_SCHEMA_DEPTH); + "Lance schema depth exceeds limit " + MAX_SCHEMA_DEPTH + + " (observed=" + depth + ")"); } if (field == null) { throw new IllegalArgumentException("Lance schema field must not be null"); @@ -319,7 +324,8 @@ private static void collectFieldNames(LanceField field, String parentPath, int d ++traversalState.fieldCount; if (traversalState.fieldCount > MAX_SCHEMA_FIELDS) { throw new IllegalArgumentException( - "Lance schema field count exceeds limit " + MAX_SCHEMA_FIELDS); + "Lance schema field count exceeds limit " + MAX_SCHEMA_FIELDS + + " (observed=" + traversalState.fieldCount + ")"); } String segment = formatFieldPathSegment( requireExternalString(field.getName(), "Lance schema field name")); @@ -358,7 +364,8 @@ static List normalize(List descriptions, } if (descriptions.size() > MAX_LOGICAL_INDEXES) { throw new IllegalArgumentException( - "Lance logical index count exceeds limit " + MAX_LOGICAL_INDEXES); + "Lance logical index count exceeds limit " + MAX_LOGICAL_INDEXES + + " (actual=" + descriptions.size() + ")"); } if (fieldNames == null) { throw new IllegalArgumentException("Lance field names must not be null"); @@ -383,7 +390,8 @@ static List normalize(List descriptions, } if (fieldIds.size() > MAX_COLUMNS_PER_INDEX) { throw new IllegalArgumentException( - "Lance logical index column count exceeds limit " + MAX_COLUMNS_PER_INDEX); + "Lance logical index column count exceeds limit " + MAX_COLUMNS_PER_INDEX + + " (actual=" + fieldIds.size() + ")"); } List columns = new ArrayList<>(fieldIds.size()); @@ -407,7 +415,8 @@ static List normalize(List descriptions, if (aggregateColumnNamesBytes > MAX_COLUMN_NAMES_BYTES) { throw new IllegalArgumentException( "Lance logical index column names exceed aggregate limit " - + MAX_COLUMN_NAMES_BYTES + " UTF-8 bytes"); + + MAX_COLUMN_NAMES_BYTES + " UTF-8 bytes (observed=" + + aggregateColumnNamesBytes + ")"); } columns.add(column); } @@ -442,7 +451,8 @@ private static String normalizeProperties(String indexName, String detailsJson) if (utf8Length(detailsJson) > MAX_EXTERNAL_STRING_BYTES) { throw new IllegalArgumentException( "Lance index details JSON exceeds limit " - + MAX_EXTERNAL_STRING_BYTES + " UTF-8 bytes"); + + MAX_EXTERNAL_STRING_BYTES + " UTF-8 bytes (actual=" + + utf8Length(detailsJson) + ")"); } if (StringUtils.isBlank(detailsJson)) { return "{}"; @@ -476,7 +486,8 @@ private static String normalizeProperties(String indexName, String detailsJson) if (utf8Length(properties) > MAX_PROPERTIES_BYTES) { throw new IllegalArgumentException( "Lance index properties exceed limit " - + MAX_PROPERTIES_BYTES + " UTF-8 bytes"); + + MAX_PROPERTIES_BYTES + " UTF-8 bytes (actual=" + + utf8Length(properties) + ")"); } return properties; } @@ -531,7 +542,8 @@ private static String requireExternalString(String value, String valueType) { } if (utf8Length(value) > MAX_EXTERNAL_STRING_BYTES) { throw new IllegalArgumentException(valueType + " exceeds limit " - + MAX_EXTERNAL_STRING_BYTES + " UTF-8 bytes"); + + MAX_EXTERNAL_STRING_BYTES + " UTF-8 bytes (actual=" + + utf8Length(value) + ")"); } return value; } diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index 6239ab8cc3b4f3..99e5f7f228dc03 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -723,8 +723,8 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then cd "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}" if [[ ! -f "${PATCHED_MARK}" ]]; then # Apply the merged PRs first; the latest PR #73 and #79 both require Lance v11. - # This order keeps the upstream patches unchanged, including Cargo.lock. - for lance_patch in pr-74 pr-75-pr-78 pr-77 pr-73 pr-79; do + # PR #80 explicitly initializes OpenDAL for statically linked C/C++ callers. + for lance_patch in pr-74 pr-75-pr-78 pr-77 pr-73 pr-79 pr-80; do patch --batch --forward --reject-file=- --fuzz=0 --no-backup-if-mismatch -s \ -p1 <"${TP_PATCH_DIR}/${LANCE_C_SOURCE}-${lance_patch}.patch" done diff --git a/thirdparty/patches/lance-c-0.1.9-pr-80.patch b/thirdparty/patches/lance-c-0.1.9-pr-80.patch new file mode 100644 index 00000000000000..edc42cad0a1271 --- /dev/null +++ b/thirdparty/patches/lance-c-0.1.9-pr-80.patch @@ -0,0 +1,225 @@ +From 7fcd9c4ff7c03c10bdc9d8a600b3f7b0cf28a9ad Mon Sep 17 00:00:00 2001 +From: zhangstar333 +Date: Thu, 10 Sep 2026 15:50:53 +0800 +Subject: [PATCH] oss provider error + +Doris integration: rebase only Cargo.toml/Cargo.lock hunk context over PR #73. +All added and removed lines are identical to upstream PR #80 at 7fcd9c4. + +--- + Cargo.lock | 1 + + Cargo.toml | 2 + + src/runtime.rs | 5 ++ + tests/compile_and_run_test.rs | 16 ++++++ + tests/cpp/test_oss_transport.c | 40 +++++++++++++ + tests/static_oss_transport_test.py | 91 ++++++++++++++++++++++++++++++ + 6 files changed, 155 insertions(+) + create mode 100644 tests/cpp/test_oss_transport.c + create mode 100644 tests/static_oss_transport_test.py + +diff --git a/Cargo.lock b/Cargo.lock +--- a/Cargo.lock ++++ b/Cargo.lock +@@ -3970,6 +3970,7 @@ + "libc", + "log", + "object_store", ++ "opendal", + "pin-project", + "prost", + "snafu", +diff --git a/Cargo.toml b/Cargo.toml +--- a/Cargo.toml ++++ b/Cargo.toml +@@ -43,6 +43,8 @@ + log = "0.4" + libc = "0.2" + object_store = "0.13.2" ++# Explicitly install the HTTP transport when embedded in a static C/C++ executable. ++opendal = { version = "=0.58.2", default-features = false, features = ["http-transport-reqwest"] } + pin-project = "1.0" + prost = "0.14" + snafu = "0.9" +diff --git a/src/runtime.rs b/src/runtime.rs +index 0153d3f..3bd8964 100644 +--- a/src/runtime.rs ++++ b/src/runtime.rs +@@ -8,6 +8,11 @@ use std::sync::LazyLock; + /// Global multi-threaded Tokio runtime, shared across all FFI calls. + /// Initialized lazily on first access. + pub static RT: LazyLock = LazyLock::new(|| { ++ // A native linker can omit OpenDAL's automatic constructor from liblance_c.a. ++ // Keep initialization reachable from the FFI entry points, before any HTTP I/O. ++ // Installation is idempotent and preserves an already installed transport. ++ opendal::install_default(); ++ + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() +diff --git a/tests/compile_and_run_test.rs b/tests/compile_and_run_test.rs +index b419ac9..8566d10 100644 +--- a/tests/compile_and_run_test.rs ++++ b/tests/compile_and_run_test.rs +@@ -249,3 +249,19 @@ fn test_cpp_compilation_and_execution() { + + run_test_binary(&binary, &dataset_uri, &write_uri); + } ++ ++/// A fresh C executable must initialize OpenDAL even when archive constructors are omitted. ++#[cfg(target_os = "linux")] ++#[test] ++#[ignore = "requires a C compiler, Python 3, and building the static library"] ++fn test_static_oss_transport() { ++ let (shared_library, _) = build_lance_c(); ++ let static_library = shared_library.with_file_name("liblance_c.a"); ++ assert!(static_library.exists(), "static library was not built"); ++ let status = Command::new("python3") ++ .arg(Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/static_oss_transport_test.py")) ++ .arg(static_library) ++ .status() ++ .expect("failed to run the static OSS transport test"); ++ assert!(status.success(), "static OSS HTTP transport test failed"); ++} +diff --git a/tests/cpp/test_oss_transport.c b/tests/cpp/test_oss_transport.c +new file mode 100644 +index 0000000..c96ee7e +--- /dev/null ++++ b/tests/cpp/test_oss_transport.c +@@ -0,0 +1,40 @@ ++/* SPDX-License-Identifier: Apache-2.0 */ ++/* SPDX-FileCopyrightText: Copyright The Lance Authors */ ++ ++#include "lance/lance.h" ++#include ++#include ++ ++/* The Python harness serves a missing manifest on a local HTTP endpoint. */ ++int main(int argc, char **argv) { ++ if (argc != 3) return 2; ++ const char *options[] = { ++ "oss_endpoint", argv[1], ++ "oss_region", "cn-test", ++ "oss_access_key_id", "test-key", ++ "oss_secret_access_key", "test-secret", ++ "addressing_style", "path", ++ NULL ++ }; ++ LanceSession *session = NULL; ++ LanceDataset *dataset = NULL; ++ const char *uri = "oss://test-bucket/missing.lance"; ++ if (strcmp(argv[2], "shared") == 0) { ++ session = lance_session_new(0, 0); ++ if (session == NULL) return 3; ++ dataset = lance_dataset_open_with_session(uri, options, 1, session); ++ } else { ++ dataset = lance_dataset_open(uri, options, 1); ++ } ++ /* The object does not exist, but the request must reach the HTTP server. */ ++ const char *error = lance_last_error_message(); ++ int failed = dataset != NULL || error == NULL; ++ if (error != NULL) { ++ fprintf(stderr, "%s\n", error); ++ failed |= strstr(error, "default HTTP transport is not installed") != NULL; ++ lance_free_string(error); ++ } ++ lance_dataset_close(dataset); ++ lance_session_close(session); ++ return failed ? 1 : 0; ++} +diff --git a/tests/static_oss_transport_test.py b/tests/static_oss_transport_test.py +new file mode 100644 +index 0000000..7d0e87c +--- /dev/null ++++ b/tests/static_oss_transport_test.py +@@ -0,0 +1,91 @@ ++# SPDX-License-Identifier: Apache-2.0 ++# SPDX-FileCopyrightText: Copyright The Lance Authors ++ ++"""Exercise native OSS HTTP initialization from fresh, statically linked C processes. ++ ++Build lance-c first, then run on Linux: ++ python3 tests/static_oss_transport_test.py target/release/liblance_c.a ++ ++No OSS account is needed. A local HTTP server returns 404 for a missing manifest. ++The assertion is that an HTTP request reaches it, not merely that opening fails. ++Unlike a Rust test binary, the C executable must pull initialization from the archive. ++""" ++ ++import argparse ++from http.server import BaseHTTPRequestHandler, HTTPServer ++import os ++from pathlib import Path ++import shlex ++import subprocess ++import sys ++import tempfile ++import threading ++ ++ ++def main(): ++ parser = argparse.ArgumentParser(description=__doc__) ++ parser.add_argument("library", type=Path) ++ args = parser.parse_args() ++ if not sys.platform.startswith("linux"): ++ parser.error("this static-link regression test currently supports Linux") ++ library = args.library.resolve(strict=True) ++ root = Path(__file__).resolve().parents[1] ++ requests = [] ++ ++ class Handler(BaseHTTPRequestHandler): ++ def missing(self): ++ requests.append((self.command, self.path)) ++ self.send_response(404) ++ self.send_header("Content-Length", "0") ++ self.send_header("Connection", "close") ++ self.end_headers() ++ ++ do_HEAD = missing ++ do_GET = missing ++ ++ def log_message(self, *_args): ++ pass ++ ++ with tempfile.TemporaryDirectory(prefix="lance-static-oss-") as directory: ++ executable = Path(directory) / "test_oss_transport" ++ # Pass the archive explicitly; -llance_c could silently select the shared library. ++ # Do not use --whole-archive: ordinary native linking must retain initialization. ++ subprocess.run( ++ shlex.split(os.environ.get("CC", "cc")) ++ + ["-std=c11", "-Wall", "-Wextra", "-Werror", "-Wl,--gc-sections", ++ "-I", str(root / "include"), str(root / "tests/cpp/test_oss_transport.c"), ++ str(library), "-lgcc_s", "-lutil", "-lrt", "-lpthread", "-lm", "-ldl", ++ "-o", str(executable)], ++ check=True, ++ ) ++ environment = { ++ key: value for key, value in os.environ.items() ++ if not key.startswith(("AWS_", "OSS_", "ALIBABA_CLOUD_")) ++ and key.lower() not in ("http_proxy", "https_proxy", "all_proxy", "no_proxy") ++ } ++ environment["NO_PROXY"] = "127.0.0.1,localhost" ++ # Each mode starts a new process so an earlier call cannot hide missing initialization. ++ for mode in ("ordinary", "shared"): ++ requests.clear() ++ with HTTPServer(("127.0.0.1", 0), Handler) as server: ++ thread = threading.Thread( ++ target=server.serve_forever, kwargs={"poll_interval": 0.05}, daemon=True ++ ) ++ thread.start() ++ try: ++ result = subprocess.run( ++ [str(executable), f"http://127.0.0.1:{server.server_port}", mode], ++ env=environment, capture_output=True, text=True, timeout=30, ++ ) ++ finally: ++ server.shutdown() ++ thread.join() ++ assert result.returncode == 0, f"{mode}: {result.stderr}" ++ assert any("/_versions/" in path for _, path in requests), ( ++ f"{mode}: no manifest HTTP request reached the server: {result.stderr}" ++ ) ++ print(f"PASS: {mode} OSS open reached the local HTTP server") ++ ++ ++if __name__ == "__main__": ++ main() From bd6d6316de7cf3d89e786172b0ddc1cd9d2f15c3 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Thu, 10 Sep 2026 22:57:39 +0800 Subject: [PATCH 09/12] update case --- .../lance/LanceExternalCatalog.java | 10 +++- .../datasource/lance/LanceMetadataLoader.java | 48 +++++++++++++++---- .../test_lance_vector_search_index_types.out | 16 +++---- ...est_lance_vector_search_index_types.groovy | 38 ++++----------- 4 files changed, 65 insertions(+), 47 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java index ae0f8f717e0226..50d824d41e1c5a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java @@ -336,7 +336,15 @@ public LanceTableMetadata loadTableMetadata(String dbName, String tableName) { } public LanceTableMetadata loadTableMetadataForSearch(String dbName, String tableName) { - return loadTableMetadata(dbName, tableName, Optional.empty()); + makeSureInitialized(); + ResolvedTableAccess tableAccess = resolveTableAccess(dbName, tableName); + try { + return LanceMetadataLoader.loadLatestForSearch( + tableAccess.datasetUri, tableAccess.storageOptions, allocator); + } catch (Exception e) { + throw new RuntimeException("Failed to load Lance table metadata for " + dbName + "." + tableName + + ": " + sanitizedRootCauseMessage(e), safeCause(e)); + } } public LanceTableMetadata loadTableMetadata(String dbName, String tableName, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceMetadataLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceMetadataLoader.java index 1e9c6aa83b677c..1e376333bf9039 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceMetadataLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceMetadataLoader.java @@ -23,11 +23,14 @@ import com.fasterxml.jackson.databind.JsonNode; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.lance.Dataset; import org.lance.Fragment; import org.lance.index.Index; import org.lance.index.IndexDescription; import org.lance.schema.LanceField; +import org.lance.schema.LanceSchema; import java.util.ArrayList; import java.util.Collections; @@ -38,6 +41,7 @@ /** Loads one fixed Lance dataset snapshot through the Lance Java SDK. */ public final class LanceMetadataLoader { + private static final Logger LOG = LogManager.getLogger(LanceMetadataLoader.class); private static final long ALLOCATOR_LIMIT = 256L * 1024 * 1024; private LanceMetadataLoader() { @@ -56,7 +60,7 @@ public static LanceTableMetadata loadLatestForTvf( // S3 TVFs do not plan FE index-segment groups; only the dataset snapshot is needed. return loadInternal(datasetUri, LanceStorageOptions.fromDorisStorageProperties(datasetUri, storageProperties), - OptionalLong.empty(), allocator, false); + OptionalLong.empty(), allocator, false, false); } } @@ -66,12 +70,20 @@ public static LanceTableMetadata loadLatestForTvf( *

Called by * {@link LanceExternalCatalog#loadTableMetadata(String, String, java.util.Optional)} when no * time-travel version is requested. Schema, version, and fragments are read from the same - * opened dataset snapshot, together with index coverage for fragment grouping and external searches. + * opened dataset snapshot, together with index coverage for fragment grouping. The known SDK + * schema-conversion failure disables scalar segment grouping for this snapshot only. */ public static LanceTableMetadata loadLatest(String datasetUri, Map lanceStorageOptions, BufferAllocator allocator) throws Exception { return loadInternal( - datasetUri, lanceStorageOptions, OptionalLong.empty(), allocator, true); + datasetUri, lanceStorageOptions, OptionalLong.empty(), allocator, true, true); + } + + /** Search-index planning requires field IDs; SDK schema conversion failures remain fatal. */ + public static LanceTableMetadata loadLatestForSearch(String datasetUri, + Map lanceStorageOptions, BufferAllocator allocator) throws Exception { + return loadInternal( + datasetUri, lanceStorageOptions, OptionalLong.empty(), allocator, true, false); } /** @@ -85,13 +97,13 @@ public static LanceTableMetadata loadVersion(String datasetUri, Map lanceStorageOptions, long version, BufferAllocator allocator) throws Exception { return loadInternal( - datasetUri, lanceStorageOptions, OptionalLong.of(version), allocator, true); + datasetUri, lanceStorageOptions, OptionalLong.of(version), allocator, true, true); } /** Shared implementation for the latest-version and explicit-version public entry points. */ private static LanceTableMetadata loadInternal(String datasetUri, Map lanceStorageOptions, OptionalLong version, - BufferAllocator allocator, boolean includeIndexSegments) throws Exception { + BufferAllocator allocator, boolean includeIndexSegments, boolean allowSchemaFallback) throws Exception { try (Dataset dataset = Dataset.open().allocator(allocator).uri(datasetUri) .readOptions(LanceReadOptions.build(lanceStorageOptions, version)).build()) { long resolvedVersion = dataset.version(); @@ -102,7 +114,9 @@ private static LanceTableMetadata loadInternal(String datasetUri, fragment.metadata().getPhysicalRows())); } Map lanceFieldIds = includeIndexSegments - ? loadTopLevelFieldIds(dataset) : Collections.emptyMap(); + ? loadTopLevelFieldIds(dataset, allowSchemaFallback) : Collections.emptyMap(); + // Index discovery stays strict even when SDK schema conversion prevents field-ID + // mapping. Do not hide inconsistent index metadata behind the schema workaround. List indexSegments = includeIndexSegments ? loadIndexSegments(dataset) : Collections.emptyList(); return includeIndexSegments @@ -114,9 +128,27 @@ private static LanceTableMetadata loadInternal(String datasetUri, } } - private static Map loadTopLevelFieldIds(Dataset dataset) { + private static Map loadTopLevelFieldIds(Dataset dataset, boolean allowSchemaFallback) { + LanceSchema schema; + try { + schema = dataset.getLanceSchema(); + } catch (IllegalArgumentException e) { + if (!allowSchemaFallback || !"ArrowSchema conversion error".equals(e.getMessage())) { + throw e; + } + // Lance v11's JNI converter cannot represent some types (notably Dictionary), + // even though getSchema() can import the dataset's Arrow schema. An empty field-ID + // map makes LanceScalarIndexPlanner choose fragment scans; filters still reach + // Lance. This does not add support for reading Dictionary values in Doris. + // Restrict the catch to the SDK call: invalid IDs and duplicate names below must + // remain errors, as must all index-description failures in loadIndexSegments(). + LOG.warn("Lance SDK schema conversion failed at dataset version {}; " + + "disabling FE scalar index segment planning for this snapshot: {}", + dataset.version(), e.getMessage()); + return Collections.emptyMap(); + } Map result = new LinkedHashMap<>(); - for (LanceField field : dataset.getLanceSchema().fields()) { + for (LanceField field : schema.fields()) { if (field.getId() < 0) { throw new IllegalStateException( "Lance field '" + field.getName() + "' has invalid id " + field.getId()); diff --git a/regression-test/data/external_table_p0/lance/test_lance_vector_search_index_types.out b/regression-test/data/external_table_p0/lance/test_lance_vector_search_index_types.out index 878891ea48bec0..31987592600d78 100644 --- a/regression-test/data/external_table_p0/lance/test_lance_vector_search_index_types.out +++ b/regression-test/data/external_table_p0/lance/test_lance_vector_search_index_types.out @@ -53,15 +53,15 @@ row_id bigint No false \N -- !ivf_hnsw_sq_ef_5 -- 518 item-0518 0.0 -517 item-0517 66.278755 -516 item-0516 132.55751 -515 item-0515 198.83627 -514 item-0514 248.54533 +519 item-0519 49.70906 +517 item-0517 66.27875 +520 item-0520 115.98781 +516 item-0516 132.5575 -- !ivf_hnsw_sq_ef_50 -- 518 item-0518 0.0 -519 item-0519 49.70907 -517 item-0517 66.278755 -520 item-0520 115.98782 -516 item-0516 132.55751 +519 item-0519 49.70906 +517 item-0517 66.27875 +520 item-0520 115.98781 +516 item-0516 132.5575 diff --git a/regression-test/suites/external_table_p0/lance/test_lance_vector_search_index_types.groovy b/regression-test/suites/external_table_p0/lance/test_lance_vector_search_index_types.groovy index 105e75a7791404..2bf1a327805bb6 100644 --- a/regression-test/suites/external_table_p0/lance/test_lance_vector_search_index_types.groovy +++ b/regression-test/suites/external_table_p0/lance/test_lance_vector_search_index_types.groovy @@ -51,15 +51,12 @@ suite("test_lance_vector_search_index_types", "p0,external") { // 257 rather than 256 because it discriminates by a wider margin on all seven collinear // tables; it is pinned as the collinear profile's boundary_row in the generator. String boundaryQuery = "[256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271]" - // Row 518's vector, in the middle of the data, where the graph traversal has room to - // settle for a worse neighbour when its candidate width is narrow. The generator pins - // this row for the ef discriminator below; which rows react to ef is decided by the - // graph draw, so it moves whenever the fixture is rebuilt and the generator reports the - // rows that still work when it does. + // Row 518's vector, used to cover narrow and wide HNSW candidate searches. Their + // results may coincide; whether ef changes the answer depends on the graph and Lance version. String midQuery = "[517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532]" - // ef is what a graph index cannot be searched without; refine_factor is what makes a - // lossy index comparable to an exact distance. Both stay out of the discriminators. + // Set ef explicitly for graph searches; refine_factor reranks quantized candidates + // using original vectors. The ef-only queries below retain their approximate distances. Map refineOptions = [ "vs_ivf_sq_f32" : ', "refine_factor"="10"', "vs_ivf_hnsw_flat_f32": ', "refine_factor"="10", "ef"="100"', @@ -141,29 +138,10 @@ suite("test_lance_vector_search_index_types", "p0,external") { ORDER BY _distance, row_id """ - // The graph counterpart of the nprobes discriminator: with ef=5 the traversal has to - // settle for a worse fifth neighbour than with ef=50, so ef demonstrably reached the - // index instead of being dropped on the way. Both queries probe all four partitions and - // neither reranks, because refine_factor with exact distances is precisely what would - // hide the effect. IVF_HNSW_SQ is the table the fixture generator pins this on: on 1024 - // collinear vectors the FLAT and PQ graphs still return the exact rows at ef=5, so only - // this one can carry the assertion. - def narrowEf = sql """ - SELECT row_id, _distance - FROM ${search("vs_ivf_hnsw_sq_f32", midQuery, "5", "4", ', "ef"="5"')} - ORDER BY _distance, row_id - """ - def wideEf = sql """ - SELECT row_id, _distance - FROM ${search("vs_ivf_hnsw_sq_f32", midQuery, "5", "4", ', "ef"="50"')} - ORDER BY _distance, row_id - """ - assertEquals(5, narrowEf.size()) - assertEquals(5, wideEf.size()) - assertFalse(narrowEf.collect { it[1] }.equals(wideEf.collect { it[1] }), - "IVF_HNSW_SQ returned the same distances for ef=5 and ef=50, so ef never reached " - + "the graph search. ef=5=" + narrowEf + " ef=50=" + wideEf) - + // Cover both ef values without requiring different answers: on this frozen fixture, + // Lance v11 returns the same five candidates for ef=5 and ef=50. The goldens below + // record their unrefined SQ distances, not exact squared L2 distances. The ef lower-bound + // error case below separately checks that the supplied ef reaches HNSW search. qt_ivf_hnsw_sq_ef_5 """ SELECT row_id, label, _distance FROM ${search("vs_ivf_hnsw_sq_f32", midQuery, "5", "4", ', "ef"="5"')} From 2a98a5bac840285405911870ef94458119ab1da9 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Fri, 11 Sep 2026 14:51:36 +0800 Subject: [PATCH 10/12] update without index details --- be/src/format_v2/table/lance_reader.cpp | 2 +- .../lance/LanceIndexMetadataLoader.java | 189 +++++++++++++----- .../datasource/lance/LanceMetadataLoader.java | 7 +- 3 files changed, 147 insertions(+), 51 deletions(-) diff --git a/be/src/format_v2/table/lance_reader.cpp b/be/src/format_v2/table/lance_reader.cpp index 68b97797b8cdad..7629d61d969908 100644 --- a/be/src/format_v2/table/lance_reader.cpp +++ b/be/src/format_v2/table/lance_reader.cpp @@ -38,8 +38,8 @@ #include "exec/common/endian.h" #include "format_v2/lance/lance_reader_helper.h" #include "format_v2/lance/lance_runtime_filter_helper.h" -#include "runtime/exec_env.h" #include "format_v2/lance/lance_session_manager.h" +#include "runtime/exec_env.h" #include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "storage/utils.h" diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexMetadataLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexMetadataLoader.java index 7870237accc515..2fa2fe7d5ab915 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexMetadataLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexMetadataLoader.java @@ -26,6 +26,8 @@ import com.google.gson.stream.JsonToken; import org.apache.arrow.memory.BufferAllocator; import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.lance.Dataset; import org.lance.index.Index; import org.lance.index.IndexCriteria; @@ -51,6 +53,8 @@ /** Loads and normalizes logical and physical index metadata from one latest Lance dataset snapshot. */ public final class LanceIndexMetadataLoader { + private static final Logger LOG = LogManager.getLogger(LanceIndexMetadataLoader.class); + // Bounds are package-visible so the admission snapshot revalidates against the same limits. static final int MAX_LOGICAL_INDEXES = 256; private static final int MAX_COLUMNS_PER_INDEX = 64; @@ -107,7 +111,8 @@ public static List loadPhysicalEntries(String datasetUr static List collectPhysicalEntries(Dataset dataset) { List indexes = dataset.getIndexes(); if (indexes == null) { - throw new IllegalArgumentException("Lance physical index entries must not be null"); + throw new IllegalArgumentException( + "Lance physical index entries must not be null: Dataset.getIndexes() returned null"); } // Bound the raw provider response before any filtering so a flood of system // entries still fails closed instead of consuming unbounded memory. @@ -118,27 +123,31 @@ static List collectPhysicalEntries(Dataset dataset) { } List entries = new ArrayList<>(indexes.size()); Set seenUuids = new HashSet<>(); - for (Index index : indexes) { + for (int position = 0; position < indexes.size(); ++position) { + Index index = indexes.get(position); if (index == null) { throw new IllegalArgumentException( - "Lance physical index entry must not be null"); + "Lance physical index entry must not be null (entry_position=" + position + + ", total_entries=" + indexes.size() + ", positions are zero-based)"); } String name = requireExternalString( - index.name(), "Lance physical index entry name"); + index.name(), "Lance physical index entry name at position " + position); UUID uuid = index.uuid(); if (uuid == null) { throw new IllegalArgumentException( - "Lance physical index entry uuid must not be null"); + "Lance physical index entry uuid must not be null for index '" + name + "'"); } long datasetVersion = index.datasetVersion(); if (datasetVersion <= 0) { throw new IllegalArgumentException( - "Lance physical index entry dataset version must be positive"); + "Lance physical index entry dataset version must be positive for index '" + + name + "' (uuid=" + uuid + ", actual=" + datasetVersion + ")"); } String uuidString = uuid.toString(); if (!seenUuids.add(uuidString)) { throw new IllegalArgumentException( - "Duplicate Lance physical index entry uuid '" + uuidString + "'"); + "Duplicate Lance physical index entry uuid '" + uuidString + + "' at index '" + name + "'; each physical segment must have a unique UUID"); } // Validate every raw entry before filtering so malformed system metadata or a UUID // collision between a system and user entry cannot be hidden from the all-or-error read. @@ -237,11 +246,14 @@ static List collectPhysicalIndexI /** * Describes only user-created indexes. The Lance JNI bulk describe path also tries to * materialize details for internal indexes, whose details are not supported by the SDK. + * Legacy indexes that cannot be described because all their segments lack details are + * omitted. This disables their FE segment planning, not the scanner's automatic index use. */ static List describeUserIndexes(Dataset dataset) { List listedNames = dataset.listIndexes(); if (listedNames == null) { - throw new IllegalArgumentException("Lance index names must not be null"); + throw new IllegalArgumentException( + "Lance index names must not be null: Dataset.listIndexes() returned null"); } if (listedNames.size() > MAX_PHYSICAL_INDEX_ENTRIES) { throw new IllegalArgumentException( @@ -261,42 +273,106 @@ static List describeUserIndexes(Dataset dataset) { throw new IllegalArgumentException( "Lance logical index count exceeds limit " + MAX_LOGICAL_INDEXES + " (observed=" + userIndexNames.size() - + ", listed_entries=" + listedNames.size() + ")"); + + ", listed_entries=" + listedNames.size() + ", last_index='" + name + + "'); the limit counts distinct user index names, not physical segments"); } } List descriptions = new ArrayList<>(userIndexNames.size()); + Set indexesWithoutDetails = null; for (String name : userIndexNames) { IndexCriteria criteria = new IndexCriteria.Builder().hasName(name).build(); List matching = dataset.describeIndices(criteria); if (matching == null) { - throw new IllegalArgumentException("Lance index descriptions must not be null"); + throw new IllegalArgumentException( + "Lance index descriptions must not be null: describeIndices returned null for index '" + + name + "' at dataset version " + dataset.version()); } - // A criteria query is an exact lookup; any other cardinality is inconsistent metadata. + // Lance attempts to infer legacy vector details before describing indexes. Only + // check raw metadata after that attempt: skipping them upfront would also discard + // vector segments whose details Lance can successfully recover from index files. + if (matching.isEmpty()) { + if (indexesWithoutDetails == null) { + indexesWithoutDetails = loadIndexesWithoutDetails(dataset); + } + if (indexesWithoutDetails.contains(name)) { + LOG.warn("Skipping FE metadata and segment planning for legacy Lance index '{}' " + + "at dataset version {}: all physical segments lack index details; " + + "scanner index selection remains unchanged", name, dataset.version()); + continue; + } + } + // Keep all other missing or ambiguous descriptions strict, including mixed + // legacy/current segments. An empty description alone is not evidence of age. if (matching.size() != 1) { throw new IllegalArgumentException( "Lance index criteria must return exactly one description (actual=" - + matching.size() + ")"); + + matching.size() + ") for index '" + name + + "' at dataset version " + dataset.version() + + "; expected one logical index description, which may contain multiple segments. " + + (matching.isEmpty() + ? "The listed index could not be described and was not confirmed to have " + + "missing details on every segment; inspect the Lance native warnings " + + "in fe.out for the underlying reason." + : "Multiple descriptions matched the same exact index name; " + + "check the index metadata for this snapshot.")); } IndexDescription description = matching.get(0); if (description == null) { throw new IllegalArgumentException( - "Lance logical index description must not be null"); + "Lance logical index description must not be null for index '" + name + + "' at dataset version " + dataset.version()); } String describedName = requireExternalString( - description.getName(), "Lance logical index name"); + description.getName(), "Lance logical index description name requested for '" + name + "'"); if (!name.equals(describedName)) { throw new IllegalArgumentException( - "Lance index description name does not match requested name"); + "Lance index description name does not match requested name (expected='" + + name + "', actual='" + describedName + "', dataset_version=" + + dataset.version() + ")"); } descriptions.add(description); } return descriptions; } + /** Names with at least one physical entry and no segment carrying index details. */ + private static Set loadIndexesWithoutDetails(Dataset dataset) { + List indexes = dataset.getIndexes(); + if (indexes == null) { + throw new IllegalArgumentException("Lance physical index entries must not be null: " + + "Dataset.getIndexes() returned null while checking legacy indexes without details"); + } + if (indexes.size() > MAX_PHYSICAL_INDEX_ENTRIES) { + throw new IllegalArgumentException( + "Lance physical index entry count exceeds limit " + + MAX_PHYSICAL_INDEX_ENTRIES + " (actual=" + indexes.size() + ")"); + } + Set withoutDetails = new HashSet<>(); + Set withDetails = new HashSet<>(); + for (int position = 0; position < indexes.size(); ++position) { + Index index = indexes.get(position); + if (index == null) { + throw new IllegalArgumentException("Lance physical index entry must not be null " + + "while checking legacy indexes (entry_position=" + position + + ", total_entries=" + indexes.size() + ", positions are zero-based)"); + } + String name = requireExternalString(index.name(), + "Lance physical index entry name at position " + position); + if (index.indexDetails().isPresent()) { + withDetails.add(name); + } else { + withoutDetails.add(name); + } + } + withoutDetails.removeAll(withDetails); + return withoutDetails; + } + static Map buildFieldNamesById(List fields) { if (fields == null) { - throw new IllegalArgumentException("Lance schema fields must not be null"); + throw new IllegalArgumentException("Lance schema fields must not be null: " + + "expected the schema field list for resolving index column IDs"); } if (fields.size() > MAX_SCHEMA_FIELDS) { throw new IllegalArgumentException( @@ -316,28 +392,33 @@ private static void collectFieldNames(LanceField field, String parentPath, int d if (depth > MAX_SCHEMA_DEPTH) { throw new IllegalArgumentException( "Lance schema depth exceeds limit " + MAX_SCHEMA_DEPTH - + " (observed=" + depth + ")"); + + " (observed=" + depth + ", parent_path='" + parentPath + "')"); } if (field == null) { - throw new IllegalArgumentException("Lance schema field must not be null"); + throw new IllegalArgumentException("Lance schema field must not be null " + + "(parent_path='" + parentPath + "', depth=" + depth + ")"); } ++traversalState.fieldCount; if (traversalState.fieldCount > MAX_SCHEMA_FIELDS) { throw new IllegalArgumentException( "Lance schema field count exceeds limit " + MAX_SCHEMA_FIELDS - + " (observed=" + traversalState.fieldCount + ")"); + + " (observed=" + traversalState.fieldCount + ", parent_path='" + + parentPath + "'); the limit includes nested fields"); } String segment = formatFieldPathSegment( - requireExternalString(field.getName(), "Lance schema field name")); + requireExternalString(field.getName(), "Lance schema field name for field ID " + field.getId())); String path = requireExternalString( parentPath.isEmpty() ? segment : parentPath + "." + segment, - "Lance schema field path"); - if (fieldNames.put(field.getId(), path) != null) { - throw new IllegalArgumentException("Duplicate Lance schema field id " + field.getId()); + "Lance schema field path for field ID " + field.getId()); + String previousPath = fieldNames.put(field.getId(), path); + if (previousPath != null) { + throw new IllegalArgumentException("Duplicate Lance schema field id " + field.getId() + + " (first_path='" + previousPath + "', duplicate_path='" + path + "')"); } List children = field.getChildren(); if (children == null) { - throw new IllegalArgumentException("Lance schema field children must not be null"); + throw new IllegalArgumentException("Lance schema field children must not be null for field '" + + path + "' (field_id=" + field.getId() + "); expected an empty list for a leaf field"); } for (LanceField child : children) { collectFieldNames(child, path, depth + 1, fieldNames, traversalState); @@ -360,7 +441,8 @@ static String formatFieldPathSegment(String segment) { static List normalize(List descriptions, Map fieldNames) { if (descriptions == null) { - throw new IllegalArgumentException("Lance index descriptions must not be null"); + throw new IllegalArgumentException("Lance index descriptions must not be null: " + + "expected a list of logical index descriptions to normalize"); } if (descriptions.size() > MAX_LOGICAL_INDEXES) { throw new IllegalArgumentException( @@ -368,7 +450,8 @@ static List normalize(List descriptions, + " (actual=" + descriptions.size() + ")"); } if (fieldNames == null) { - throw new IllegalArgumentException("Lance field names must not be null"); + throw new IllegalArgumentException("Lance field names must not be null: " + + "expected a mapping from schema field IDs to column paths"); } List normalized = new ArrayList<>(descriptions.size()); @@ -376,22 +459,25 @@ static List normalize(List descriptions, for (int position = 0; position < descriptions.size(); ++position) { IndexDescription description = descriptions.get(position); if (description == null) { - throw new IllegalArgumentException("Lance logical index description must not be null"); + throw new IllegalArgumentException("Lance logical index description must not be null " + + "(description_position=" + position + ", total_descriptions=" + + descriptions.size() + ", positions are zero-based)"); } String name = requireExternalString( - description.getName(), "Lance logical index name"); + description.getName(), "Lance logical index name at description position " + position); String indexType = requireExternalString( - description.getIndexType(), "Lance logical index type"); + description.getIndexType(), "Lance logical index type for index '" + name + "'"); List fieldIds = description.getFieldIds(); if (fieldIds == null || fieldIds.isEmpty()) { throw new IllegalArgumentException( - "Lance logical index field IDs must not be null or empty"); + "Lance logical index field IDs must not be null or empty for index '" + name + + "' (actual=" + (fieldIds == null ? "null" : "empty list") + ")"); } if (fieldIds.size() > MAX_COLUMNS_PER_INDEX) { throw new IllegalArgumentException( "Lance logical index column count exceeds limit " + MAX_COLUMNS_PER_INDEX - + " (actual=" + fieldIds.size() + ")"); + + " (actual=" + fieldIds.size() + ", index='" + name + "')"); } List columns = new ArrayList<>(fieldIds.size()); @@ -399,24 +485,28 @@ static List normalize(List descriptions, for (Integer fieldId : fieldIds) { if (fieldId == null) { throw new IllegalArgumentException( - "Lance logical index field ID must not be null"); + "Lance logical index field ID must not be null for index '" + name + "'"); } if (!uniqueFieldIds.add(fieldId)) { throw new IllegalArgumentException( - "Duplicate field id " + fieldId + " in Lance logical index metadata"); + "Duplicate field id " + fieldId + " in Lance logical index metadata for index '" + + name + "'; each indexed column must appear only once"); } if (!fieldNames.containsKey(fieldId)) { throw new IllegalArgumentException( - "Lance index metadata references unknown field id " + fieldId); + "Lance index metadata references unknown field id " + fieldId + + " for index '" + name + "'; this ID is absent from the dataset schema"); } String column = requireExternalString( - fieldNames.get(fieldId), "Lance logical index column name"); + fieldNames.get(fieldId), "Lance logical index column name for index '" + name + + "', field ID " + fieldId); aggregateColumnNamesBytes += utf8Length(column); if (aggregateColumnNamesBytes > MAX_COLUMN_NAMES_BYTES) { throw new IllegalArgumentException( "Lance logical index column names exceed aggregate limit " + MAX_COLUMN_NAMES_BYTES + " UTF-8 bytes (observed=" - + aggregateColumnNamesBytes + ")"); + + aggregateColumnNamesBytes + ", index='" + name + "', column='" + + column + "'); the limit covers column names across all logical indexes"); } columns.add(column); } @@ -431,7 +521,8 @@ static List normalize(List descriptions, String name = indexed.index.getName(); if (!logicalIndexNames.add(name)) { throw new IllegalArgumentException( - "Duplicate Lance logical index name '" + name + "'"); + "Duplicate Lance logical index name '" + name + + "'; same-name physical segments must belong to one logical description"); } } normalized.sort(Comparator.comparing( @@ -452,7 +543,7 @@ private static String normalizeProperties(String indexName, String detailsJson) throw new IllegalArgumentException( "Lance index details JSON exceeds limit " + MAX_EXTERNAL_STRING_BYTES + " UTF-8 bytes (actual=" - + utf8Length(detailsJson) + ")"); + + utf8Length(detailsJson) + ", index='" + indexName + "')"); } if (StringUtils.isBlank(detailsJson)) { return "{}"; @@ -463,13 +554,15 @@ private static String normalizeProperties(String indexName, String detailsJson) reader.setLenient(false); parsed = GsonUtils.GSON.getAdapter(JsonElement.class).read(reader); if (reader.peek() != JsonToken.END_DOCUMENT) { - throw invalidDetailsJson(indexName); + throw invalidDetailsJson(indexName, "expected one JSON object without trailing content"); } } catch (IOException | RuntimeException e) { - throw invalidDetailsJson(indexName); + // Parser exceptions may contain raw JSON (including credentials). Report the + // expected format without copying the parser message or retaining its cause. + throw invalidDetailsJson(indexName, "expected one well-formed JSON object without trailing content"); } if (!parsed.isJsonObject()) { - throw invalidDetailsJson(indexName); + throw invalidDetailsJson(indexName, "expected a JSON object at the root"); } TreeMap allowedProperties = new TreeMap<>(); @@ -487,7 +580,7 @@ private static String normalizeProperties(String indexName, String detailsJson) throw new IllegalArgumentException( "Lance index properties exceed limit " + MAX_PROPERTIES_BYTES + " UTF-8 bytes (actual=" - + utf8Length(properties) + ")"); + + utf8Length(properties) + ", index='" + indexName + "')"); } return properties; } @@ -499,7 +592,7 @@ private static void copyNestedProperties(JsonObject source, String propertyName, return; } if (!nested.isJsonObject()) { - throw invalidDetailsJson(indexName); + throw invalidDetailsJson(indexName, "property '" + propertyName + "' must be a JSON object"); } TreeMap allowedNested = new TreeMap<>(); @@ -525,20 +618,22 @@ private static void copyPrimitiveProperties(JsonObject source, Set allow continue; } if (!value.isJsonPrimitive()) { - throw invalidDetailsJson(indexName); + throw invalidDetailsJson(indexName, "property '" + entry.getKey() + + "' must be a string, number or boolean"); } target.put(entry.getKey(), value); } } - private static IllegalArgumentException invalidDetailsJson(String indexName) { + private static IllegalArgumentException invalidDetailsJson(String indexName, String reason) { return new IllegalArgumentException( - "Invalid Lance index details JSON for '" + indexName + "'"); + "Invalid Lance index details JSON for '" + indexName + "': " + reason); } private static String requireExternalString(String value, String valueType) { if (value == null || value.isEmpty()) { - throw new IllegalArgumentException(valueType + " must not be null or empty"); + throw new IllegalArgumentException(valueType + " must not be null or empty (actual=" + + (value == null ? "null" : "empty string") + ")"); } if (utf8Length(value) > MAX_EXTERNAL_STRING_BYTES) { throw new IllegalArgumentException(valueType + " exceeds limit " diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceMetadataLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceMetadataLoader.java index 1e376333bf9039..3be84957f4822d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceMetadataLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceMetadataLoader.java @@ -115,8 +115,8 @@ private static LanceTableMetadata loadInternal(String datasetUri, } Map lanceFieldIds = includeIndexSegments ? loadTopLevelFieldIds(dataset, allowSchemaFallback) : Collections.emptyMap(); - // Index discovery stays strict even when SDK schema conversion prevents field-ID - // mapping. Do not hide inconsistent index metadata behind the schema workaround. + // Index discovery still validates metadata when schema conversion prevents field-ID + // mapping. Only confirmed legacy indexes without details are omitted by the loader. List indexSegments = includeIndexSegments ? loadIndexSegments(dataset) : Collections.emptyList(); return includeIndexSegments @@ -141,7 +141,8 @@ private static Map loadTopLevelFieldIds(Dataset dataset, boolea // map makes LanceScalarIndexPlanner choose fragment scans; filters still reach // Lance. This does not add support for reading Dictionary values in Doris. // Restrict the catch to the SDK call: invalid IDs and duplicate names below must - // remain errors, as must all index-description failures in loadIndexSegments(). + // remain errors. Legacy indexes without details are handled separately by the + // index loader; this schema workaround must not suppress other index errors. LOG.warn("Lance SDK schema conversion failed at dataset version {}; " + "disabling FE scalar index segment planning for this snapshot: {}", dataset.version(), e.getMessage()); From ca3e5c5f58667a9ac71d3b525f82eaa4ce5e5b65 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Wed, 16 Sep 2026 15:51:16 +0800 Subject: [PATCH 11/12] update to upstream --- .../org/apache/doris/datasource/lance/source/LanceScanNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java index 52c9e064371ace..4c12ba24ddd5ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java @@ -417,7 +417,7 @@ private Optional> createVectorIndexSegmentSplits(LanceTableMetadata plannedIndexSegments = plan.splitCount(); plannedIndexFragments = plan.indexSegmentFragmentCount(); plannedUnindexedFragments = plannedFragments - plannedIndexFragments; - appendUnindexedFragmentSplits(plan, visibleFragments); + plan.addUncoveredFragments(visibleFragments.values(), 1); return Optional.of(plan.buildSplits()); } return Optional.empty(); From 817ef5ca3cd1f485e6c16b78046a5091986a39a7 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Wed, 16 Sep 2026 22:37:02 +0800 Subject: [PATCH 12/12] update ut case --- .../lance_runtime_filter_helper_test.cpp | 10 +- be/test/format_v2/table/lance_reader_test.cpp | 131 ++---------------- 2 files changed, 11 insertions(+), 130 deletions(-) diff --git a/be/test/format_v2/lance/lance_runtime_filter_helper_test.cpp b/be/test/format_v2/lance/lance_runtime_filter_helper_test.cpp index 273078ad0ab147..c1b7bdcc1af085 100644 --- a/be/test/format_v2/lance/lance_runtime_filter_helper_test.cpp +++ b/be/test/format_v2/lance/lance_runtime_filter_helper_test.cpp @@ -43,7 +43,6 @@ #include "exprs/vliteral.h" #include "exprs/vslot_ref.h" #include "format/format_common.h" -#include "runtime/runtime_profile.h" namespace doris::format::lance { namespace { @@ -175,7 +174,7 @@ TEST(LanceRuntimeFilterHelperTest, ConvertsSupportedFiltersToLanceSql) { EXPECT_TRUE(result->skipped_filter_ids.empty()); } -TEST(LanceRuntimeFilterHelperTest, RecordsUnsupportedRuntimeFilters) { +TEST(LanceRuntimeFilterHelperTest, SeparatesSupportedAndUnsupportedRuntimeFilters) { const VExprContextSPtrs conjuncts { int64_runtime_in("id", {2}, 3), unsupported_bloom_runtime_filter("id", 8), @@ -187,13 +186,6 @@ TEST(LanceRuntimeFilterHelperTest, RecordsUnsupportedRuntimeFilters) { EXPECT_EQ("(`id` IN (2))", result->expression); EXPECT_EQ((std::vector {3}), result->pushable_filter_ids); EXPECT_EQ((std::vector {8}), result->skipped_filter_ids); - - RuntimeProfile profile("lance_runtime_filter_profile"); - record_lance_runtime_filter_pushdown(&profile, *result); - ASSERT_NE(profile.get_info_string("LanceRuntimeFilterPushedIds"), nullptr); - EXPECT_EQ("3", *profile.get_info_string("LanceRuntimeFilterPushedIds")); - ASSERT_NE(profile.get_info_string("LanceRuntimeFilterSkippedIds"), nullptr); - EXPECT_EQ("8", *profile.get_info_string("LanceRuntimeFilterSkippedIds")); } TEST(LanceRuntimeFilterHelperTest, IgnoresNonRuntimeFilterConjuncts) { diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp index a6dd9b444f2fcf..cb2b42faf1c5de 100644 --- a/be/test/format_v2/table/lance_reader_test.cpp +++ b/be/test/format_v2/table/lance_reader_test.cpp @@ -109,24 +109,6 @@ struct LanceFixtureInfo { std::vector fragment_ids; }; -void expect_lance_profile_hierarchy(RuntimeProfile* profile, - const std::vector& metric_names) { - TRuntimeProfileTree tree; - profile->to_thrift(&tree, 3); - ASSERT_FALSE(tree.nodes.empty()); - const auto& children = tree.nodes[0].child_counters_map; - ASSERT_TRUE(children.contains(RuntimeProfile::ROOT_COUNTER)); - EXPECT_TRUE(children.at(RuntimeProfile::ROOT_COUNTER).contains("FileScannerV2")); - ASSERT_TRUE(children.contains("FileScannerV2")); - EXPECT_TRUE(children.at("FileScannerV2").contains("TableReader")); - ASSERT_TRUE(children.contains("TableReader")); - EXPECT_TRUE(children.at("TableReader").contains("LanceReader")); - ASSERT_TRUE(children.contains("LanceReader")); - for (const auto& metric_name : metric_names) { - EXPECT_TRUE(children.at("LanceReader").contains(metric_name)) << metric_name; - } -} - Status get_fixture_info(const std::filesystem::path& dataset_uri, LanceFixtureInfo* info) { std::unique_ptr dataset( lance_dataset_open(dataset_uri.c_str(), nullptr, 0), lance_dataset_close); @@ -497,18 +479,6 @@ TEST(LanceTableReaderFullTextSearchTest, ValidatesRequestAndScoreTypeBeforeDatas auto valid_params = make_full_text_search_params("lance", 4, 1); LanceTableReader valid_reader; ASSERT_TRUE(init_reader(&valid_reader, columns, &state, &valid_profile, &valid_params).ok()); - ASSERT_NE(valid_profile.get_info_string("LanceSearchType"), nullptr); - EXPECT_EQ("FULL_TEXT", *valid_profile.get_info_string("LanceSearchType")); - ASSERT_NE(valid_profile.get_info_string("LanceFtsCoverageMode"), nullptr); - EXPECT_EQ("STRICT", *valid_profile.get_info_string("LanceFtsCoverageMode")); - ASSERT_NE(valid_profile.get_info_string("LanceFtsQueryType"), nullptr); - EXPECT_EQ("MATCH", *valid_profile.get_info_string("LanceFtsQueryType")); - ASSERT_NE(valid_profile.get_info_string("LanceFtsMatchOperator"), nullptr); - EXPECT_EQ("OR", *valid_profile.get_info_string("LanceFtsMatchOperator")); - ASSERT_NE(valid_profile.get_info_string("LanceFtsMaxFuzzyDistance"), nullptr); - EXPECT_EQ("0", *valid_profile.get_info_string("LanceFtsMaxFuzzyDistance")); - ASSERT_NE(valid_profile.get_info_string("LanceTopKPlusOffset"), nullptr); - EXPECT_EQ("5", *valid_profile.get_info_string("LanceTopKPlusOffset")); RuntimeProfile empty_query_profile("lance_fts_empty_query"); auto empty_query_params = make_full_text_search_params("", 4, 0); @@ -552,10 +522,6 @@ TEST(LanceTableReaderFullTextSearchTest, ValidatesQuerySpecificParameters) { auto phrase_params = make_phrase_search_params("lance search", 4, 0, 1); LanceTableReader phrase_reader; ASSERT_TRUE(init_reader(&phrase_reader, columns, &state, &phrase_profile, &phrase_params).ok()); - ASSERT_NE(phrase_profile.get_info_string("LanceFtsQueryType"), nullptr); - EXPECT_EQ("PHRASE", *phrase_profile.get_info_string("LanceFtsQueryType")); - ASSERT_NE(phrase_profile.get_info_string("LanceFtsPhraseSlop"), nullptr); - EXPECT_EQ("1", *phrase_profile.get_info_string("LanceFtsPhraseSlop")); RuntimeProfile fuzzy_profile("lance_fts_fuzzy_request"); auto fuzzy_params = make_full_text_search_params("lance", 4, 0, TFtsCoverageMode::STRICT, @@ -830,12 +796,6 @@ TEST(LanceTableReaderVectorSearchTest, SearchesWholeSnapshotWithOffsetAndDistanc EXPECT_FLOAT_EQ(1.0F, rows[0].second); EXPECT_EQ(4, rows[1].first); EXPECT_FLOAT_EQ(8.25F, rows[1].second); - ASSERT_NE(profile.get_info_string("LanceTopK"), nullptr); - EXPECT_EQ("2", *profile.get_info_string("LanceTopK")); - ASSERT_NE(profile.get_info_string("LanceOffset"), nullptr); - EXPECT_EQ("1", *profile.get_info_string("LanceOffset")); - ASSERT_NE(profile.get_info_string("LanceTopKPlusOffset"), nullptr); - EXPECT_EQ("3", *profile.get_info_string("LanceTopKPlusOffset")); EXPECT_TRUE(reader.close().ok()); } @@ -903,51 +863,6 @@ TEST(LanceTableReaderVectorSearchTest, SearchesMultipleFragmentSplits) { } std::ranges::sort(row_ids); EXPECT_EQ((std::vector {1, 2, 3, 4}), row_ids); - ASSERT_NE(profile.get_counter("LancePlannedIndexSegmentCount"), nullptr); - EXPECT_EQ(0, profile.get_counter("LancePlannedIndexSegmentCount")->value()); - ASSERT_NE(profile.get_counter("LancePlannedIndexedFragmentCount"), nullptr); - EXPECT_EQ(0, profile.get_counter("LancePlannedIndexedFragmentCount")->value()); - ASSERT_NE(profile.get_counter("LancePlannedFlatSearchFragmentCount"), nullptr); - EXPECT_EQ(profile.get_counter("LancePlannedFlatSearchFragmentCount")->value(), - static_cast(fixture.fragment_ids.size())); - ASSERT_NE(profile.get_info_string("LanceTopK"), nullptr); - EXPECT_EQ("4", *profile.get_info_string("LanceTopK")); - ASSERT_NE(profile.get_info_string("LanceOffset"), nullptr); - EXPECT_EQ("0", *profile.get_info_string("LanceOffset")); - ASSERT_NE(profile.get_info_string("LanceTopKPlusOffset"), nullptr); - EXPECT_EQ("4", *profile.get_info_string("LanceTopKPlusOffset")); - EXPECT_NE(profile.get_counter("LanceDatasetOpenTime"), nullptr); - EXPECT_NE(profile.get_counter("LanceScannerConfigureTime"), nullptr); - EXPECT_NE(profile.get_counter("LanceScannerReadTime"), nullptr); - EXPECT_NE(profile.get_counter("LanceRowOffsetRangesScanned"), nullptr); - EXPECT_NE(profile.get_counter("LanceTaskWaitTime"), nullptr); - EXPECT_EQ(profile.get_counter("LanceExecutionIndexCacheMissLoads"), nullptr); - EXPECT_EQ(profile.get_counter("LanceRowIdTakeReadTime"), nullptr); - EXPECT_EQ(profile.get_counter("LanceRowIdFetchTotalTime"), nullptr); - EXPECT_EQ(profile.get_counter("LanceScalarIndexQueryTime"), nullptr); - EXPECT_EQ(profile.get_counter("LanceScalarIndexResultSerializationTime"), nullptr); - expect_lance_profile_hierarchy(&profile, {"LanceDatasetOpenTime", - "LanceScannerConfigureTime", - "LanceScannerReadTime", - "LanceArrowToDorisBlockTime", - "LanceExecutionIOOps", - "LanceExecutionIORequests", - "LanceExecutionIOBytesRead", - "LanceDataCacheBytesReadFromCache", - "LanceDataCacheBytesReadFromRemote", - "LanceIndexPartitionCacheMissLoads", - "LanceIndexComparisons", - "LanceFragmentsScanned", - "LanceRowOffsetRangesScanned", - "LanceRowsScanned", - "LanceIVFPartitionsRanked", - "LanceIVFPartitionsSearched", - "LanceVectorIndexSegmentsSearched", - "LanceTaskWaitTime", - "LanceIVFPartitionRankingTime", - "LancePlannedIndexSegmentCount", - "LancePlannedIndexedFragmentCount", - "LancePlannedFlatSearchFragmentCount"}); EXPECT_TRUE(reader.close().ok()); } @@ -1051,15 +966,6 @@ TEST(LanceTableReaderVectorSearchTest, ReturnsStableGlobalRowIdsAndFetchesPayloa EXPECT_EQ("extra", label_values.get_data_at(0).to_string()); EXPECT_EQ("unit-x", label_values.get_data_at(1).to_string()); EXPECT_EQ("extra", label_values.get_data_at(2).to_string()); - EXPECT_NE(fetch_profile.get_counter("LanceDatasetOpenTime"), nullptr); - EXPECT_NE(fetch_profile.get_counter("LanceRowIdTakeReadTime"), nullptr); - EXPECT_NE(fetch_profile.get_counter("LanceArrowToDorisBlockTime"), nullptr); - EXPECT_NE(fetch_profile.get_counter("LanceRowIdFetchTotalTime"), nullptr); - expect_lance_profile_hierarchy( - &fetch_profile, - {"LanceDatasetOpenTime", "LanceRowIdTakeReadTime", "LanceArrowToDorisBlockTime", - "LanceRowIdFetchTotalTime", "LanceDataCacheBytesReadFromCache", - "LanceDataCacheBytesReadFromRemote"}); EXPECT_TRUE(payload_reader.close().ok()); } @@ -1218,14 +1124,13 @@ TEST(LanceTableReaderFilterTest, CombinesStaticSubstraitFilterWithRuntimeFilter) row_ids.get_data().end()); } EXPECT_EQ((std::vector {4}), combined_row_ids); - ASSERT_NE(combined_profile.get_info_string("LanceRuntimeFilterPushedIds"), nullptr); - EXPECT_EQ("42", *combined_profile.get_info_string("LanceRuntimeFilterPushedIds")); EXPECT_TRUE(combined_reader.close().ok()); } TEST(LanceTableReaderScalarSegmentTest, FiltersIndexedAndUncoveredDomainsWithoutLosingRows) { const auto unique_suffix = std::chrono::steady_clock::now().time_since_epoch().count(); for (const auto index_type : {LANCE_SCALAR_BTREE, LANCE_SCALAR_BITMAP}) { + SCOPED_TRACE(index_type == LANCE_SCALAR_BTREE ? "BTREE" : "BITMAP"); const auto dataset_uri = std::filesystem::temp_directory_path() / ("doris_lance_scalar_segment_" + std::to_string(unique_suffix) + "_" + std::to_string(index_type) + ".lance"); @@ -1261,8 +1166,9 @@ TEST(LanceTableReaderScalarSegmentTest, FiltersIndexedAndUncoveredDomainsWithout ASSERT_EQ(1, segments.size()); const auto read_domain = [&](const std::vector& fragments, bool use_segment, - const std::vector& expected, int64_t searched, - int64_t fallbacks) { + const std::vector& expected) { + SCOPED_TRACE(::testing::PrintToString(fragments)); + SCOPED_TRACE(use_segment ? "selected segment" : "scalar index disabled"); TQueryGlobals globals; RuntimeState state(globals); RuntimeProfile profile("lance_scalar_segment"); @@ -1294,19 +1200,15 @@ TEST(LanceTableReaderScalarSegmentTest, FiltersIndexedAndUncoveredDomainsWithout } std::ranges::sort(actual); EXPECT_EQ(expected, actual); - EXPECT_EQ(use_segment ? 1 : 0, - profile.get_counter("LanceScalarIndexSegmentsRequested")->value()); - EXPECT_EQ(searched, profile.get_counter("LanceScalarIndexSegmentsSearched")->value()); - EXPECT_EQ(fallbacks, profile.get_counter("LanceScalarIndexSegmentFallbacks")->value()); - if (!use_segment) { - EXPECT_EQ(0, profile.get_counter("LanceIndexComparisons")->value()); - } EXPECT_TRUE(reader.close().ok()); }; - read_domain({fixture.fragment_ids[0]}, true, {2}, 1, 0); - read_domain({fixture.fragment_ids[1]}, false, {4}, 0, 0); + read_domain({fixture.fragment_ids[0]}, true, {2}); + read_domain({fixture.fragment_ids[1]}, false, {4}); // A segment which cannot cover the whole task must filter the entire explicit domain. - read_domain(fixture.fragment_ids, true, {2, 4}, 0, 1); + read_domain(fixture.fragment_ids, true, {2, 4}); + // Disabling the index must produce the same rows in each read domain. + read_domain({fixture.fragment_ids[0]}, false, {2}); + read_domain(fixture.fragment_ids, false, {2, 4}); } } @@ -1384,9 +1286,6 @@ TEST(LanceTableReaderFilterTest, PushesRuntimeInFilterIntoLanceScanner) { } std::ranges::sort(actual_row_ids); EXPECT_EQ((std::vector {2, 4}), actual_row_ids); - ASSERT_NE(profile.get_info_string("LanceRuntimeFilterPushedIds"), nullptr); - EXPECT_EQ("41", *profile.get_info_string("LanceRuntimeFilterPushedIds")); - EXPECT_EQ(profile.get_info_string("LanceRuntimeFilterSkippedIds"), nullptr); EXPECT_TRUE(reader.close().ok()); } @@ -1428,9 +1327,6 @@ TEST(LanceTableReaderFilterTest, SkipsNullAwareRuntimeRangeBeforeLanceScanner) { } std::ranges::sort(row_ids); EXPECT_EQ((std::vector {1, 2, 3, 4}), row_ids); - ASSERT_NE(profile.get_info_string("LanceRuntimeFilterSkippedIds"), nullptr); - EXPECT_EQ("43", *profile.get_info_string("LanceRuntimeFilterSkippedIds")); - EXPECT_EQ(profile.get_info_string("LanceRuntimeFilterPushedIds"), nullptr); EXPECT_TRUE(reader.close().ok()); } @@ -1467,9 +1363,6 @@ TEST(LanceTableReaderFilterTest, SkipsUnsafeStringRuntimeFiltersBeforeLanceCStri } } EXPECT_EQ(4U, rows); - ASSERT_NE(profile.get_info_string("LanceRuntimeFilterSkippedIds"), nullptr); - EXPECT_EQ("44,45", *profile.get_info_string("LanceRuntimeFilterSkippedIds")); - EXPECT_EQ(profile.get_info_string("LanceRuntimeFilterPushedIds"), nullptr); EXPECT_TRUE(reader.close().ok()); } @@ -1529,8 +1422,6 @@ TEST(LanceTableReaderFilterTest, SkipsTimestampNanoRuntimeFilterBeforeMaterializ // Therefore the residual <= .123456 accepts this row and the pre-materialization SQL must not // remove it. EXPECT_EQ("1970-01-01 00:00:00.123456", columns[1].type->to_string(timestamp, 0)); - ASSERT_NE(profile.get_info_string("LanceRuntimeFilterSkippedIds"), nullptr); - EXPECT_EQ("46", *profile.get_info_string("LanceRuntimeFilterSkippedIds")); EXPECT_TRUE(reader.close().ok()); } @@ -1611,8 +1502,6 @@ TEST(LanceTableReaderFilterTest, SkipsPhysicalNumericTypesUnsupportedByPinnedPla ASSERT_TRUE(reader.get_block(&block, &eos).ok()); ASSERT_FALSE(eos); EXPECT_EQ(1U, block.rows()); - ASSERT_NE(profile.get_info_string("LanceRuntimeFilterSkippedIds"), nullptr); - EXPECT_EQ("47,48,49,50", *profile.get_info_string("LanceRuntimeFilterSkippedIds")); EXPECT_TRUE(reader.close().ok()); }