diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index ca20cb63990447..5990908e58cb74 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1218,6 +1218,8 @@ DEFINE_Bool(enable_file_cache, "false"); // Both will use the directory "memory" on the disk instead of the real RAM. DEFINE_String(file_cache_path, "[{\"path\":\"${DORIS_HOME}/file_cache\"}]"); DEFINE_Int64(file_cache_each_block_size, "1048576"); // 1MB +DEFINE_Bool(enable_external_file_meta_disk_cache, "true"); +DEFINE_mInt64(external_file_meta_disk_cache_max_entry_bytes, "67108864"); // 64MB DEFINE_Bool(clear_file_cache, "false"); DEFINE_mBool(enable_file_cache_query_limit, "false"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 2a6b21ccf8c0e3..66ec0fe09cc7e2 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1261,6 +1261,8 @@ DECLARE_Bool(enable_file_cache); // Both will use the directory "memory" on the disk instead of the real RAM. DECLARE_String(file_cache_path); DECLARE_Int64(file_cache_each_block_size); +DECLARE_Bool(enable_external_file_meta_disk_cache); +DECLARE_mInt64(external_file_meta_disk_cache_max_entry_bytes); DECLARE_Bool(clear_file_cache); DECLARE_mBool(enable_file_cache_query_limit); DECLARE_mBool(enable_file_cache_query_limit_segment_meta); diff --git a/be/src/exec/scan/file_scanner.cpp b/be/src/exec/scan/file_scanner.cpp index fcd0e8e863ded0..efa1967f8a63ec 100644 --- a/be/src/exec/scan/file_scanner.cpp +++ b/be/src/exec/scan/file_scanner.cpp @@ -1351,6 +1351,12 @@ Status FileScanner::_init_parquet_reader(FileMetaCache* file_meta_cache_ptr, std::unique_ptr parquet_reader) { const TFileRangeDesc& range = _current_range; Status init_status = Status::OK(); + // Row-id fetch scanners read one mapped range without a split source and disable memory cache. + const bool enable_file_meta_memory_cache = + _split_source != nullptr && _should_enable_file_meta_memory_cache(file_meta_cache_ptr); + auto configure_file_meta_memory_cache = [&](GenericReader* reader) { + reader->set_enable_file_meta_memory_cache(enable_file_meta_memory_cache); + }; phmap::flat_hash_map>> slot_id_to_predicates = _local_state @@ -1376,6 +1382,7 @@ Status FileScanner::_init_parquet_reader(FileMetaCache* file_meta_cache_ptr, [this]() -> std::shared_ptr { return _create_row_id_column_iterator(); }); + configure_file_meta_memory_cache(iceberg_reader.get()); init_status = static_cast(iceberg_reader.get())->init_reader(&pctx); _cur_reader = std::move(iceberg_reader); } else if (range.__isset.table_format_params && @@ -1384,6 +1391,7 @@ Status FileScanner::_init_parquet_reader(FileMetaCache* file_meta_cache_ptr, auto paimon_reader = PaimonParquetReader::create_unique( _profile, *_params, range, _state->batch_size(), &_state->timezone_obj(), _kv_cache, _io_ctx, _state, file_meta_cache_ptr); + configure_file_meta_memory_cache(paimon_reader.get()); init_status = static_cast(paimon_reader.get())->init_reader(&pctx); _cur_reader = std::move(paimon_reader); } else if (range.__isset.table_format_params && @@ -1392,6 +1400,7 @@ Status FileScanner::_init_parquet_reader(FileMetaCache* file_meta_cache_ptr, auto hudi_reader = HudiParquetReader::create_unique( _profile, *_params, range, _state->batch_size(), &_state->timezone_obj(), _io_ctx, _state, file_meta_cache_ptr); + configure_file_meta_memory_cache(hudi_reader.get()); init_status = static_cast(hudi_reader.get())->init_reader(&pctx); _cur_reader = std::move(hudi_reader); } else if (range.table_format_params.table_format_type == "hive") { @@ -1403,6 +1412,7 @@ Status FileScanner::_init_parquet_reader(FileMetaCache* file_meta_cache_ptr, [this]() -> std::shared_ptr { return _create_row_id_column_iterator(); }); + configure_file_meta_memory_cache(hive_reader.get()); init_status = static_cast(hive_reader.get())->init_reader(&pctx); _cur_reader = std::move(hive_reader); } else if (range.table_format_params.table_format_type == "tvf") { @@ -1416,6 +1426,7 @@ Status FileScanner::_init_parquet_reader(FileMetaCache* file_meta_cache_ptr, [this]() -> std::shared_ptr { return _create_row_id_column_iterator(); }); + configure_file_meta_memory_cache(parquet_reader.get()); init_status = static_cast(parquet_reader.get())->init_reader(&pctx); _cur_reader = std::move(parquet_reader); } else if (_is_load) { @@ -1425,6 +1436,7 @@ Status FileScanner::_init_parquet_reader(FileMetaCache* file_meta_cache_ptr, _io_ctx, _state, file_meta_cache_ptr, _state->query_options().enable_parquet_lazy_mat); } + configure_file_meta_memory_cache(parquet_reader.get()); init_status = static_cast(parquet_reader.get())->init_reader(&pctx); _cur_reader = std::move(parquet_reader); } @@ -1436,6 +1448,12 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, std::unique_ptr orc_reader) { const TFileRangeDesc& range = _current_range; Status init_status = Status::OK(); + // Row-id fetch scanners read one mapped range without a split source and disable memory cache. + const bool enable_file_meta_memory_cache = + _split_source != nullptr && _should_enable_file_meta_memory_cache(file_meta_cache_ptr); + auto configure_file_meta_memory_cache = [&](GenericReader* reader) { + reader->set_enable_file_meta_memory_cache(enable_file_meta_memory_cache); + }; // Build unified OrcInitContext (shared by all ORC reader variants) OrcInitContext octx; @@ -1454,6 +1472,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, [this]() -> std::shared_ptr { return _create_row_id_column_iterator(); }); + configure_file_meta_memory_cache(tran_orc_reader.get()); init_status = static_cast(tran_orc_reader.get())->init_reader(&octx); _cur_reader = std::move(tran_orc_reader); @@ -1467,6 +1486,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, [this]() -> std::shared_ptr { return _create_row_id_column_iterator(); }); + configure_file_meta_memory_cache(iceberg_reader.get()); init_status = static_cast(iceberg_reader.get())->init_reader(&octx); _cur_reader = std::move(iceberg_reader); @@ -1476,6 +1496,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, auto paimon_reader = PaimonOrcReader::create_unique( _profile, _state, *_params, range, _state->batch_size(), _state->timezone(), _kv_cache, _io_ctx, file_meta_cache_ptr); + configure_file_meta_memory_cache(paimon_reader.get()); init_status = static_cast(paimon_reader.get())->init_reader(&octx); _cur_reader = std::move(paimon_reader); @@ -1485,6 +1506,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, auto hudi_reader = HudiOrcReader::create_unique(_profile, _state, *_params, range, _state->batch_size(), _state->timezone(), _io_ctx, file_meta_cache_ptr); + configure_file_meta_memory_cache(hudi_reader.get()); init_status = static_cast(hudi_reader.get())->init_reader(&octx); _cur_reader = std::move(hudi_reader); @@ -1498,6 +1520,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, [this]() -> std::shared_ptr { return _create_row_id_column_iterator(); }); + configure_file_meta_memory_cache(hive_reader.get()); init_status = static_cast(hive_reader.get())->init_reader(&octx); _cur_reader = std::move(hive_reader); @@ -1512,6 +1535,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, [this]() -> std::shared_ptr { return _create_row_id_column_iterator(); }); + configure_file_meta_memory_cache(orc_reader.get()); init_status = static_cast(orc_reader.get())->init_reader(&octx); _cur_reader = std::move(orc_reader); } else if (_is_load) { @@ -1520,6 +1544,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, _profile, _state, *_params, range, _state->batch_size(), _state->timezone(), _io_ctx, file_meta_cache_ptr, _state->query_options().enable_orc_lazy_mat); } + configure_file_meta_memory_cache(orc_reader.get()); init_status = static_cast(orc_reader.get())->init_reader(&octx); _cur_reader = std::move(orc_reader); } diff --git a/be/src/exec/scan/file_scanner.h b/be/src/exec/scan/file_scanner.h index 87a54243f16c96..41890856bcfbe1 100644 --- a/be/src/exec/scan/file_scanner.h +++ b/be/src/exec/scan/file_scanner.h @@ -339,12 +339,16 @@ class FileScanner : public Scanner { : _local_state->get_push_down_agg_type(); } - // enable the file meta cache only when - // 1. max_external_file_meta_cache_num is > 0 - // 2. the file number is less than 1/3 of cache's capacibility - // Otherwise, the cache miss rate will be high bool _should_enable_file_meta_cache() { - return ExecEnv::GetInstance()->file_meta_cache()->enabled() && + auto* file_meta_cache = ExecEnv::GetInstance()->file_meta_cache(); + return file_meta_cache != nullptr && + (file_meta_cache->enabled() || FileMetaCache::is_persistent_cache_enabled()); + } + + // Enable memory file meta cache only when the file number is less than 1/3 of cache capacity. + // Otherwise, the cache miss rate will be high. Persistent cache is not gated by this rule. + bool _should_enable_file_meta_memory_cache(FileMetaCache* file_meta_cache) { + return file_meta_cache != nullptr && file_meta_cache->enabled() && _split_source->num_scan_ranges() < config::max_external_file_meta_cache_num / 3; } }; diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index 9e5811af8b0a51..76f2cbed7cff60 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -449,6 +449,11 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) { VExprContextSPtrs table_conjuncts; RETURN_IF_ERROR(_build_table_conjuncts(&table_conjuncts)); + FileMetaCache* file_meta_cache = nullptr; + if (_should_enable_file_meta_cache()) { + file_meta_cache = ExecEnv::GetInstance()->file_meta_cache(); + DORIS_CHECK(file_meta_cache != nullptr); + } RETURN_IF_ERROR(_table_reader->init({ .projected_columns = _projected_columns, .conjuncts = std::move(table_conjuncts), @@ -460,6 +465,8 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) { .file_slot_descs = &_file_slot_descs, .push_down_agg_type = _local_state->get_push_down_agg_type(), .condition_cache_digest = _local_state->get_condition_cache_digest(), + .file_meta_cache = file_meta_cache, + .enable_file_meta_memory_cache = _should_enable_file_meta_memory_cache(file_meta_cache), })); return Status::OK(); } @@ -529,7 +536,13 @@ bool FileScannerV2::_should_skip_not_found(const Status& status, bool ignore_not } bool FileScannerV2::_should_enable_file_meta_cache() const { - return ExecEnv::GetInstance()->file_meta_cache()->enabled() && + auto* file_meta_cache = ExecEnv::GetInstance()->file_meta_cache(); + return file_meta_cache != nullptr && + (file_meta_cache->enabled() || FileMetaCache::is_persistent_cache_enabled()); +} + +bool FileScannerV2::_should_enable_file_meta_memory_cache(FileMetaCache* file_meta_cache) const { + return file_meta_cache != nullptr && file_meta_cache->enabled() && _split_source->num_scan_ranges() < config::max_external_file_meta_cache_num / 3; } diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index 2bddc5d5e69e6c..4c4a5663b8335d 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -122,6 +122,7 @@ class FileScannerV2 final : public Scanner { std::map partition_values); static bool _should_skip_not_found(const Status& status, bool ignore_not_found); bool _should_enable_file_meta_cache() const; + bool _should_enable_file_meta_memory_cache(FileMetaCache* file_meta_cache) const; std::optional _create_global_rowid_context( const TFileRangeDesc& range) const; Status _generate_partition_values(const TFileRangeDesc& range, diff --git a/be/src/format/generic_reader.h b/be/src/format/generic_reader.h index cc21ce82c2805e..2f0589c4e6a235 100644 --- a/be/src/format/generic_reader.h +++ b/be/src/format/generic_reader.h @@ -258,6 +258,8 @@ class GenericReader : public ProfileCollector { // Pass condition cache context to the reader for HIT/MISS tracking. virtual void set_condition_cache_context(std::shared_ptr ctx) {} + void set_enable_file_meta_memory_cache(bool enable) { _enable_file_meta_memory_cache = enable; } + // Returns true if this reader can produce an accurate total row count from metadata // without reading actual data. Used to determine if CountReader decorator can be applied. // Only ORC and Parquet readers support this (via file footer metadata). @@ -279,6 +281,7 @@ class GenericReader : public ProfileCollector { // Cache to save some common part such as file footer. // Maybe null if not used FileMetaCache* _meta_cache = nullptr; + bool _enable_file_meta_memory_cache = true; // ---- Column descriptors (set by init_reader, owned by FileScanner) ---- const std::vector* _column_descs = nullptr; diff --git a/be/src/format/orc/vorc_reader.cpp b/be/src/format/orc/vorc_reader.cpp index 28ef930057f3a2..fcc9cc26779a5f 100644 --- a/be/src/format/orc/vorc_reader.cpp +++ b/be/src/format/orc/vorc_reader.cpp @@ -336,6 +336,18 @@ void OrcReader::_collect_profile_before_close() { COUNTER_UPDATE(_orc_profile.lazy_read_filtered_rows, _statistics.lazy_read_filtered_rows); COUNTER_UPDATE(_orc_profile.file_footer_read_calls, _statistics.file_footer_read_calls); COUNTER_UPDATE(_orc_profile.file_footer_hit_cache, _statistics.file_footer_hit_cache); + COUNTER_UPDATE(_orc_profile.file_footer_hit_memory_cache, + _statistics.file_footer_hit_memory_cache); + COUNTER_UPDATE(_orc_profile.file_footer_hit_disk_cache, + _statistics.file_footer_hit_disk_cache); + COUNTER_UPDATE(_orc_profile.file_footer_miss_disk_cache, + _statistics.file_footer_miss_disk_cache); + COUNTER_UPDATE(_orc_profile.file_footer_write_disk_cache, + _statistics.file_footer_write_disk_cache); + COUNTER_UPDATE(_orc_profile.file_footer_read_disk_cache_time, + _statistics.file_footer_read_disk_cache_time); + COUNTER_UPDATE(_orc_profile.file_footer_write_disk_cache_time, + _statistics.file_footer_write_disk_cache_time); if (_file_input_stream != nullptr) { _file_input_stream->collect_profile_before_close(); } @@ -376,6 +388,18 @@ void OrcReader::_init_profile() { ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterReadCalls", TUnit::UNIT, 1); _orc_profile.file_footer_hit_cache = ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitCache", TUnit::UNIT, 1); + _orc_profile.file_footer_hit_memory_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitMemoryCache", TUnit::UNIT, 1); + _orc_profile.file_footer_hit_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitDiskCache", TUnit::UNIT, 1); + _orc_profile.file_footer_miss_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterMissDiskCache", TUnit::UNIT, 1); + _orc_profile.file_footer_write_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterWriteDiskCache", TUnit::UNIT, 1); + _orc_profile.file_footer_read_disk_cache_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "FileFooterReadDiskCacheTime", orc_profile, 1); + _orc_profile.file_footer_write_disk_cache_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, "FileFooterWriteDiskCacheTime", orc_profile, 1); } } @@ -439,21 +463,62 @@ Status OrcReader::_create_file_reader() { RETURN_IF_ERROR(create_orc_reader()); } else { auto inner_file_reader = _file_input_stream->get_inner_reader(); - const auto& file_meta_cache_key = + const std::string file_meta_cache_key = FileMetaCache::get_key(inner_file_reader, _file_description); + const int64_t file_size = _file_description.file_size == -1 ? inner_file_reader->size() + : _file_description.file_size; + const FileMetaCacheContext file_meta_cache_context { + .format = FileMetaCacheFormat::ORC, + .key = file_meta_cache_key, + .modification_time = _file_description.mtime, + .file_size = file_size, + .enable_memory_cache = _enable_file_meta_memory_cache && _meta_cache->enabled()}; + FileMetaCacheProfile file_meta_cache_profile { + .hit_cache = &_statistics.file_footer_hit_cache, + .hit_memory_cache = &_statistics.file_footer_hit_memory_cache, + .hit_disk_cache = &_statistics.file_footer_hit_disk_cache, + .miss_disk_cache = &_statistics.file_footer_miss_disk_cache, + .write_disk_cache = &_statistics.file_footer_write_disk_cache, + .read_disk_cache_time = &_statistics.file_footer_read_disk_cache_time, + .write_disk_cache_time = &_statistics.file_footer_write_disk_cache_time}; + const std::string memory_cache_key = + FileMetaCache::get_memory_cache_key(file_meta_cache_context); // Local variables can be required because setSerializedFileTail is an assignment operation, not a reference. ObjLRUCache::CacheHandle _meta_cache_handle; - if (_meta_cache->lookup(file_meta_cache_key, &_meta_cache_handle)) { - const std::string* footer_ptr = _meta_cache_handle.data(); + std::string serialized_file_tail; + const FileMetaCacheLookupResult lookup_result = + _meta_cache->lookup(file_meta_cache_context, &_meta_cache_handle, + &serialized_file_tail, &file_meta_cache_profile); + if (lookup_result.state == FileMetaCacheLookupState::MEMORY_HIT) { + const std::string* footer_ptr = _meta_cache_handle.data(); options.setSerializedFileTail(*footer_ptr); RETURN_IF_ERROR(create_orc_reader()); - _statistics.file_footer_hit_cache++; + } else if (lookup_result.state == FileMetaCacheLookupState::PERSISTED_HIT) { + auto footer_ptr = std::make_unique(std::move(serialized_file_tail)); + options.setSerializedFileTail(*footer_ptr); + Status status = create_orc_reader(); + if (!status.ok()) { + VLOG_DEBUG << "ignore invalid orc file meta persistent cache payload: " << status; + _meta_cache->invalidate_persistent_cache(file_meta_cache_context); + _reader.reset(); + _file_input_stream.reset(); + return _create_file_reader(); + } + if (file_meta_cache_context.enable_memory_cache) { + _meta_cache->insert(memory_cache_key, footer_ptr, &_meta_cache_handle); + } } else { _statistics.file_footer_read_calls++; RETURN_IF_ERROR(create_orc_reader()); - std::string* footer_ptr = new std::string {_reader->getSerializedFileTail()}; - _meta_cache->insert(file_meta_cache_key, footer_ptr, &_meta_cache_handle); + const uint64_t file_tail_size = + _reader->getFileFooterLength() + _reader->getFilePostscriptLength(); + if (file_meta_cache_context.enable_memory_cache || + FileMetaCache::is_persistent_cache_payload_size_allowed(file_tail_size)) { + auto footer_ptr = std::make_unique(_reader->getSerializedFileTail()); + _meta_cache->insert(file_meta_cache_context, footer_ptr, &_meta_cache_handle, + *footer_ptr, &file_meta_cache_profile); + } } } @@ -1547,6 +1612,8 @@ void OrcReader::_init_file_description() { if (_scan_range.__isset.fs_name) { _file_description.fs_name = _scan_range.fs_name; } + _file_description.fs_name = + FileFactory::get_file_system_identity(_system_properties, _file_description); if (_scan_range.__isset.file_cache_admission) { _file_description.file_cache_admission = _scan_range.file_cache_admission; } diff --git a/be/src/format/orc/vorc_reader.h b/be/src/format/orc/vorc_reader.h index 6d64f604cdee5a..e8c006e33dc2fe 100644 --- a/be/src/format/orc/vorc_reader.h +++ b/be/src/format/orc/vorc_reader.h @@ -150,6 +150,12 @@ class OrcReader : public TableFormatReader, public RowPositionProvider { int64_t lazy_read_filtered_rows = 0; int64_t file_footer_read_calls = 0; int64_t file_footer_hit_cache = 0; + int64_t file_footer_hit_memory_cache = 0; + int64_t file_footer_hit_disk_cache = 0; + int64_t file_footer_miss_disk_cache = 0; + int64_t file_footer_write_disk_cache = 0; + int64_t file_footer_read_disk_cache_time = 0; + int64_t file_footer_write_disk_cache_time = 0; }; OrcReader(RuntimeProfile* profile, RuntimeState* state, const TFileScanRangeParams& params, @@ -317,6 +323,12 @@ class OrcReader : public TableFormatReader, public RowPositionProvider { RuntimeProfile::Counter* evaluated_row_group_count = nullptr; RuntimeProfile::Counter* file_footer_read_calls = nullptr; RuntimeProfile::Counter* file_footer_hit_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_memory_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_miss_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_read_disk_cache_time = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache_time = nullptr; }; class ORCFilterImpl : public orc::ORCFilter { diff --git a/be/src/format/parquet/vparquet_reader.cpp b/be/src/format/parquet/vparquet_reader.cpp index 26caf7faac66f2..5bb9657d713775 100644 --- a/be/src/format/parquet/vparquet_reader.cpp +++ b/be/src/format/parquet/vparquet_reader.cpp @@ -65,6 +65,7 @@ #include "storage/index/zone_map/zonemap_eval_context.h" #include "util/slice.h" #include "util/string_util.h" +#include "util/thrift_util.h" #include "util/timezone_utils.h" namespace cctz { @@ -255,6 +256,18 @@ void ParquetReader::_init_profile() { ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterReadCalls", TUnit::UNIT, 1); _parquet_profile.file_footer_hit_cache = ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitCache", TUnit::UNIT, 1); + _parquet_profile.file_footer_hit_memory_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitMemoryCache", TUnit::UNIT, 1); + _parquet_profile.file_footer_hit_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitDiskCache", TUnit::UNIT, 1); + _parquet_profile.file_footer_miss_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterMissDiskCache", TUnit::UNIT, 1); + _parquet_profile.file_footer_write_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterWriteDiskCache", TUnit::UNIT, 1); + _parquet_profile.file_footer_read_disk_cache_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, "FileFooterReadDiskCacheTime", parquet_profile, 1); + _parquet_profile.file_footer_write_disk_cache_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, "FileFooterWriteDiskCacheTime", parquet_profile, 1); _parquet_profile.decompress_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecompressTime", parquet_profile, 1); _parquet_profile.decompress_cnt = ADD_CHILD_COUNTER_WITH_LEVEL( @@ -367,21 +380,90 @@ Status ParquetReader::_open_file() { // parse magic number & parse meta data _reader_statistics.file_footer_read_calls += 1; } else { - const auto& file_meta_cache_key = + const std::string file_meta_cache_key = FileMetaCache::get_key(_tracing_file_reader, _file_description); - if (!_meta_cache->lookup(file_meta_cache_key, &_meta_cache_handle)) { + const int64_t file_size = _file_description.file_size == -1 + ? _tracing_file_reader->size() + : _file_description.file_size; + const FileMetaCacheContext file_meta_cache_context { + .format = FileMetaCacheFormat::PARQUET, + .key = file_meta_cache_key, + .modification_time = _file_description.mtime, + .file_size = file_size, + .enable_memory_cache = + _enable_file_meta_memory_cache && _meta_cache->enabled()}; + FileMetaCacheProfile file_meta_cache_profile { + .hit_cache = &_reader_statistics.file_footer_hit_cache, + .hit_memory_cache = &_reader_statistics.file_footer_hit_memory_cache, + .hit_disk_cache = &_reader_statistics.file_footer_hit_disk_cache, + .miss_disk_cache = &_reader_statistics.file_footer_miss_disk_cache, + .write_disk_cache = &_reader_statistics.file_footer_write_disk_cache, + .read_disk_cache_time = &_reader_statistics.file_footer_read_disk_cache_time, + .write_disk_cache_time = &_reader_statistics.file_footer_write_disk_cache_time}; + const std::string memory_cache_key = + FileMetaCache::get_memory_cache_key(file_meta_cache_context); + std::string footer_payload; + const FileMetaCacheLookupResult lookup_result = + _meta_cache->lookup(file_meta_cache_context, &_meta_cache_handle, + &footer_payload, &file_meta_cache_profile); + if (lookup_result.state == FileMetaCacheLookupState::MEMORY_HIT) { + _file_metadata = _meta_cache_handle.data(); + } else if (lookup_result.state == FileMetaCacheLookupState::PERSISTED_HIT) { + uint32_t metadata_size = static_cast(footer_payload.size()); + tparquet::FileMetaData t_metadata; + Status status = deserialize_thrift_msg( + reinterpret_cast(footer_payload.data()), &metadata_size, + true, &t_metadata); + if (status.ok()) { + _file_metadata_ptr = std::make_unique(t_metadata, metadata_size); + status = _file_metadata_ptr->init_schema(enable_mapping_varbinary, + enable_mapping_timestamp_tz); + } + if (status.ok()) { + if (file_meta_cache_context.enable_memory_cache) { + _meta_cache->insert(memory_cache_key, _file_metadata_ptr.release(), + &_meta_cache_handle); + _file_metadata = _meta_cache_handle.data(); + } else { + _file_metadata = _file_metadata_ptr.get(); + } + } else { + VLOG_DEBUG << "ignore invalid parquet file meta persistent cache payload: " + << status; + _meta_cache->invalidate_persistent_cache(file_meta_cache_context); + _file_metadata_ptr.reset(); + footer_payload.clear(); + } + } + if (_file_metadata == nullptr) { RETURN_IF_ERROR(parse_thrift_footer(_tracing_file_reader, &_file_metadata_ptr, &meta_size, _io_ctx, enable_mapping_varbinary, enable_mapping_timestamp_tz)); - // _file_metadata_ptr.release() : move control of _file_metadata to _meta_cache_handle - _meta_cache->insert(file_meta_cache_key, _file_metadata_ptr.release(), - &_meta_cache_handle); - _file_metadata = _meta_cache_handle.data(); + const size_t serialized_meta_size = meta_size >= PARQUET_FOOTER_SIZE + ? meta_size - PARQUET_FOOTER_SIZE + : meta_size; + if (FileMetaCache::is_persistent_cache_payload_size_allowed( + static_cast(serialized_meta_size))) { + tparquet::FileMetaData thrift_metadata = _file_metadata_ptr->to_thrift(); + ThriftSerializer serializer(true, static_cast(serialized_meta_size)); + RETURN_IF_ERROR(serializer.serialize(&thrift_metadata, &footer_payload)); + const auto insert_result = _meta_cache->insert( + file_meta_cache_context, _file_metadata_ptr, &_meta_cache_handle, + footer_payload, &file_meta_cache_profile); + if (insert_result.memory_inserted) { + _file_metadata = _meta_cache_handle.data(); + } else { + _file_metadata = _file_metadata_ptr.get(); + } + } else if (file_meta_cache_context.enable_memory_cache && + _meta_cache->insert(memory_cache_key, _file_metadata_ptr, + &_meta_cache_handle)) { + _file_metadata = _meta_cache_handle.data(); + } else { + _file_metadata = _file_metadata_ptr.get(); + } _reader_statistics.file_footer_read_calls += 1; - } else { - _reader_statistics.file_footer_hit_cache++; } - _file_metadata = _meta_cache_handle.data(); } if (_file_metadata == nullptr) { @@ -420,6 +502,8 @@ void ParquetReader::_init_file_description() { if (_scan_range.__isset.fs_name) { _file_description.fs_name = _scan_range.fs_name; } + _file_description.fs_name = + FileFactory::get_file_system_identity(_system_properties, _file_description); if (_scan_range.__isset.file_cache_admission) { _file_description.file_cache_admission = _scan_range.file_cache_admission; } @@ -1742,6 +1826,18 @@ void ParquetReader::_collect_profile() { _reader_statistics.file_footer_read_calls); COUNTER_UPDATE(_parquet_profile.file_footer_hit_cache, _reader_statistics.file_footer_hit_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_hit_memory_cache, + _reader_statistics.file_footer_hit_memory_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_hit_disk_cache, + _reader_statistics.file_footer_hit_disk_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_miss_disk_cache, + _reader_statistics.file_footer_miss_disk_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_write_disk_cache, + _reader_statistics.file_footer_write_disk_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_read_disk_cache_time, + _reader_statistics.file_footer_read_disk_cache_time); + COUNTER_UPDATE(_parquet_profile.file_footer_write_disk_cache_time, + _reader_statistics.file_footer_write_disk_cache_time); COUNTER_UPDATE(_parquet_profile.skip_page_header_num, _column_statistics.skip_page_header_num); COUNTER_UPDATE(_parquet_profile.parse_page_header_num, diff --git a/be/src/format/parquet/vparquet_reader.h b/be/src/format/parquet/vparquet_reader.h index 9046641c6c0c51..e6c768c2847784 100644 --- a/be/src/format/parquet/vparquet_reader.h +++ b/be/src/format/parquet/vparquet_reader.h @@ -108,6 +108,12 @@ class ParquetReader : public TableFormatReader { int64_t parse_footer_time = 0; int64_t file_footer_read_calls = 0; int64_t file_footer_hit_cache = 0; + int64_t file_footer_hit_memory_cache = 0; + int64_t file_footer_hit_disk_cache = 0; + int64_t file_footer_miss_disk_cache = 0; + int64_t file_footer_write_disk_cache = 0; + int64_t file_footer_read_disk_cache_time = 0; + int64_t file_footer_write_disk_cache_time = 0; int64_t file_reader_create_time = 0; int64_t open_file_num = 0; int64_t row_group_filter_time = 0; @@ -281,6 +287,12 @@ class ParquetReader : public TableFormatReader { RuntimeProfile::Counter* parse_page_index_time = nullptr; RuntimeProfile::Counter* file_footer_read_calls = nullptr; RuntimeProfile::Counter* file_footer_hit_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_memory_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_miss_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_read_disk_cache_time = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache_time = nullptr; RuntimeProfile::Counter* decompress_time = nullptr; RuntimeProfile::Counter* decompress_cnt = nullptr; RuntimeProfile::Counter* page_read_counter = nullptr; diff --git a/be/src/format/table/iceberg_reader.cpp b/be/src/format/table/iceberg_reader.cpp index 38bb395120df24..4c0e47e2b14356 100644 --- a/be/src/format/table/iceberg_reader.cpp +++ b/be/src/format/table/iceberg_reader.cpp @@ -420,6 +420,7 @@ Status IcebergParquetReader::_read_position_delete_file(const TFileRangeDesc* de ParquetReader parquet_delete_reader(get_profile(), get_scan_params(), *delete_range, READ_DELETE_FILE_BATCH_SIZE, &get_state()->timezone_obj(), get_io_ctx(), get_state(), _meta_cache); + parquet_delete_reader.set_enable_file_meta_memory_cache(_enable_file_meta_memory_cache); // The delete file range has size=-1 (read whole file). We must disable // row group filtering before init; otherwise _do_init_reader returns EndOfFile // when _filter_groups && _range_size < 0. @@ -747,6 +748,7 @@ Status IcebergOrcReader::_read_position_delete_file(const TFileRangeDesc* delete OrcReader orc_delete_reader(get_profile(), get_state(), get_scan_params(), *delete_range, READ_DELETE_FILE_BATCH_SIZE, get_state()->timezone(), get_io_ctx(), _meta_cache); + orc_delete_reader.set_enable_file_meta_memory_cache(_enable_file_meta_memory_cache); OrcInitContext delete_ctx; delete_ctx.column_names = delete_file_col_names; delete_ctx.col_name_to_block_idx = diff --git a/be/src/format/table/iceberg_reader_mixin.h b/be/src/format/table/iceberg_reader_mixin.h index aee211aee019cd..647956a80ec088 100644 --- a/be/src/format/table/iceberg_reader_mixin.h +++ b/be/src/format/table/iceberg_reader_mixin.h @@ -505,6 +505,7 @@ Status IcebergReaderMixin::_equality_delete_base( } std::unique_ptr delete_reader = _create_equality_reader(delete_desc); + delete_reader->set_enable_file_meta_memory_cache(this->_enable_file_meta_memory_cache); RETURN_IF_ERROR(delete_reader->init_schema_reader()); std::vector equality_delete_col_names; diff --git a/be/src/format/table/transactional_hive_reader.cpp b/be/src/format/table/transactional_hive_reader.cpp index 8e31ff9b37dfdb..5282e70ac11f44 100644 --- a/be/src/format/table/transactional_hive_reader.cpp +++ b/be/src/format/table/transactional_hive_reader.cpp @@ -206,6 +206,7 @@ Status TransactionalHiveReader::on_after_init_reader(ReaderInitContext* /*ctx*/) OrcReader delete_reader(get_profile(), get_state(), get_scan_params(), delete_range, 256 /*batch_size*/, get_state()->timezone(), get_io_ctx(), _meta_cache, false); + delete_reader.set_enable_file_meta_memory_cache(_enable_file_meta_memory_cache); auto acid_info_node = std::make_shared(); for (auto idx = 0; idx < TransactionalHive::DELETE_ROW_COLUMN_NAMES_LOWER_CASE.size(); diff --git a/be/src/format_v2/file_reader.cpp b/be/src/format_v2/file_reader.cpp index 4f5b247c791efd..fb939d068f6b4c 100644 --- a/be/src/format_v2/file_reader.cpp +++ b/be/src/format_v2/file_reader.cpp @@ -70,6 +70,8 @@ Status FileReader::init(RuntimeState* state) { _init_profile(); SCOPED_RAW_TIMER(&_reader_statistics.file_reader_create_time); ++_reader_statistics.open_file_num; + _file_description->fs_name = + FileFactory::get_file_system_identity(*_system_properties, *_file_description); io::FileReaderOptions reader_options = FileFactory::get_reader_options(state->query_options(), *_file_description); _file_reader = DORIS_TRY(io::DelegateReader::create_file_reader( diff --git a/be/src/format_v2/file_reader.h b/be/src/format_v2/file_reader.h index 59f684121ce1e3..2057b9aa338f97 100644 --- a/be/src/format_v2/file_reader.h +++ b/be/src/format_v2/file_reader.h @@ -228,6 +228,12 @@ class FileReader { int64_t parse_footer_time = 0; int64_t file_footer_read_calls = 0; int64_t file_footer_hit_cache = 0; + int64_t file_footer_hit_memory_cache = 0; + int64_t file_footer_hit_disk_cache = 0; + int64_t file_footer_miss_disk_cache = 0; + int64_t file_footer_write_disk_cache = 0; + int64_t file_footer_read_disk_cache_time = 0; + int64_t file_footer_write_disk_cache_time = 0; int64_t file_reader_create_time = 0; int64_t open_file_num = 0; int64_t row_group_filter_time = 0; diff --git a/be/src/format_v2/orc/orc_reader.cpp b/be/src/format_v2/orc/orc_reader.cpp index b728c6fbedb700..bdc7ee23696881 100644 --- a/be/src/format_v2/orc/orc_reader.cpp +++ b/be/src/format_v2/orc/orc_reader.cpp @@ -72,6 +72,7 @@ #include "format_v2/orc/orc_file_input_stream.h" #include "format_v2/orc/orc_search_argument.h" #include "format_v2/timestamp_statistics.h" +#include "io/fs/file_meta_cache.h" #include "io/fs/file_reader.h" #include "runtime/exec_env.h" #include "runtime/runtime_profile.h" @@ -732,10 +733,13 @@ OrcReader::OrcReader(std::shared_ptr& system_propertie std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context, - bool enable_mapping_timestamp_tz) + bool enable_mapping_timestamp_tz, FileMetaCache* file_meta_cache, + bool enable_file_meta_memory_cache) : FileReader(system_properties, file_description, io_ctx, profile), _global_rowid_context(std::move(global_rowid_context)), - _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz) {} + _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz), + _file_meta_cache(file_meta_cache), + _enable_file_meta_memory_cache(enable_file_meta_memory_cache) {} OrcReader::~OrcReader() = default; @@ -788,6 +792,42 @@ void OrcReader::_init_profile() { ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FilteredBytes", TUnit::BYTES, orc_profile, 1); _orc_profile.open_file_num = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FileNum", TUnit::UNIT, orc_profile, 1); + _orc_profile.file_footer_read_calls = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterReadCalls", TUnit::UNIT, 1); + _orc_profile.file_footer_hit_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitCache", TUnit::UNIT, 1); + _orc_profile.file_footer_hit_memory_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitMemoryCache", TUnit::UNIT, 1); + _orc_profile.file_footer_hit_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitDiskCache", TUnit::UNIT, 1); + _orc_profile.file_footer_miss_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterMissDiskCache", TUnit::UNIT, 1); + _orc_profile.file_footer_write_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterWriteDiskCache", TUnit::UNIT, 1); + _orc_profile.file_footer_read_disk_cache_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "FileFooterReadDiskCacheTime", orc_profile, 1); + _orc_profile.file_footer_write_disk_cache_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "FileFooterWriteDiskCacheTime", orc_profile, 1); +} + +void OrcReader::_update_file_meta_cache_profile() const { + if (_profile == nullptr) { + return; + } + COUNTER_UPDATE(_orc_profile.file_footer_read_calls, _reader_statistics.file_footer_read_calls); + COUNTER_UPDATE(_orc_profile.file_footer_hit_cache, _reader_statistics.file_footer_hit_cache); + COUNTER_UPDATE(_orc_profile.file_footer_hit_memory_cache, + _reader_statistics.file_footer_hit_memory_cache); + COUNTER_UPDATE(_orc_profile.file_footer_hit_disk_cache, + _reader_statistics.file_footer_hit_disk_cache); + COUNTER_UPDATE(_orc_profile.file_footer_miss_disk_cache, + _reader_statistics.file_footer_miss_disk_cache); + COUNTER_UPDATE(_orc_profile.file_footer_write_disk_cache, + _reader_statistics.file_footer_write_disk_cache); + COUNTER_UPDATE(_orc_profile.file_footer_read_disk_cache_time, + _reader_statistics.file_footer_read_disk_cache_time); + COUNTER_UPDATE(_orc_profile.file_footer_write_disk_cache_time, + _reader_statistics.file_footer_write_disk_cache_time); } void OrcReader::_collect_profile() const { @@ -855,9 +895,96 @@ Status OrcReader::init(RuntimeState* state) { _state->timezone_obj = state->timezone_obj(); } + RETURN_IF_ERROR(_init_orc_reader(state)); + _update_file_meta_cache_profile(); + return Status::OK(); +} + +Status OrcReader::_init_orc_reader(RuntimeState* state) { + if (_file_meta_cache == nullptr) { + ++_reader_statistics.file_footer_read_calls; + return _create_orc_reader(nullptr, state); + } + + const std::string file_meta_cache_key = + FileMetaCache::get_key(_tracing_file_reader, *_file_description); + const int64_t file_size = _file_description->file_size == -1 + ? static_cast(_tracing_file_reader->size()) + : _file_description->file_size; + const FileMetaCacheContext file_meta_cache_context { + .format = FileMetaCacheFormat::ORC_V2, + .key = file_meta_cache_key, + .modification_time = _file_description->mtime, + .file_size = file_size, + .enable_memory_cache = _enable_file_meta_memory_cache && _file_meta_cache->enabled()}; + FileMetaCacheProfile file_meta_cache_profile { + .hit_cache = &_reader_statistics.file_footer_hit_cache, + .hit_memory_cache = &_reader_statistics.file_footer_hit_memory_cache, + .hit_disk_cache = &_reader_statistics.file_footer_hit_disk_cache, + .miss_disk_cache = &_reader_statistics.file_footer_miss_disk_cache, + .write_disk_cache = &_reader_statistics.file_footer_write_disk_cache, + .read_disk_cache_time = &_reader_statistics.file_footer_read_disk_cache_time, + .write_disk_cache_time = &_reader_statistics.file_footer_write_disk_cache_time}; + const std::string memory_cache_key = + FileMetaCache::get_memory_cache_key(file_meta_cache_context); + ObjLRUCache::CacheHandle meta_cache_handle; + std::string serialized_file_tail; + const FileMetaCacheLookupResult lookup_result = + _file_meta_cache->lookup(file_meta_cache_context, &meta_cache_handle, + &serialized_file_tail, &file_meta_cache_profile); + if (lookup_result.state == FileMetaCacheLookupState::MEMORY_HIT) { + const auto* cached_file_tail = meta_cache_handle.data(); + DORIS_CHECK(cached_file_tail != nullptr); + RETURN_IF_ERROR(_create_orc_reader(cached_file_tail, state)); + } else if (lookup_result.state == FileMetaCacheLookupState::PERSISTED_HIT) { + auto cached_file_tail = std::make_unique(std::move(serialized_file_tail)); + Status status = _create_orc_reader(cached_file_tail.get(), state); + if (!status.ok()) { + if (status.is()) { + return status; + } + VLOG_DEBUG << "ignore invalid ORC v2 file meta persistent cache payload: " << status; + ++_reader_statistics.file_footer_read_calls; + RETURN_IF_ERROR(_create_orc_reader(nullptr, state)); + _file_meta_cache->invalidate_persistent_cache(file_meta_cache_context); + const uint64_t file_tail_size = _state->reader->getFileFooterLength() + + _state->reader->getFilePostscriptLength(); + if (file_meta_cache_context.enable_memory_cache || + FileMetaCache::is_persistent_cache_payload_size_allowed(file_tail_size)) { + cached_file_tail = + std::make_unique(_state->reader->getSerializedFileTail()); + _file_meta_cache->insert(file_meta_cache_context, cached_file_tail, + &meta_cache_handle, *cached_file_tail, + &file_meta_cache_profile); + } + } else if (file_meta_cache_context.enable_memory_cache) { + _file_meta_cache->insert(memory_cache_key, cached_file_tail, &meta_cache_handle); + } + } else { + ++_reader_statistics.file_footer_read_calls; + RETURN_IF_ERROR(_create_orc_reader(nullptr, state)); + const uint64_t file_tail_size = + _state->reader->getFileFooterLength() + _state->reader->getFilePostscriptLength(); + if (file_meta_cache_context.enable_memory_cache || + FileMetaCache::is_persistent_cache_payload_size_allowed(file_tail_size)) { + auto cached_file_tail = + std::make_unique(_state->reader->getSerializedFileTail()); + _file_meta_cache->insert(file_meta_cache_context, cached_file_tail, &meta_cache_handle, + *cached_file_tail, &file_meta_cache_profile); + } + } + return Status::OK(); +} + +Status OrcReader::_create_orc_reader(const std::string* serialized_file_tail, RuntimeState* state) { + _state->reader.reset(); + _state->root_type = nullptr; ::orc::ReaderOptions options; options.setMemoryPool(*ExecEnv::GetInstance()->orc_memory_pool()); options.setReaderMetrics(&_state->reader_metrics); + if (serialized_file_tail != nullptr) { + options.setSerializedFileTail(*serialized_file_tail); + } OrcFileInputStreamOptions input_stream_options; const auto natural_read_size_mb = config::orc_natural_read_size_mb; diff --git a/be/src/format_v2/orc/orc_reader.h b/be/src/format_v2/orc/orc_reader.h index adba20fbce0b10..f0ffd7d762fda0 100644 --- a/be/src/format_v2/orc/orc_reader.h +++ b/be/src/format_v2/orc/orc_reader.h @@ -33,6 +33,10 @@ namespace cctz { class time_zone; } // namespace cctz +namespace doris { +class FileMetaCache; +} + namespace doris::format::orc { struct OrcReaderScanState; @@ -49,7 +53,8 @@ class OrcReader final : public format::FileReader { std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context = std::nullopt, - bool enable_mapping_timestamp_tz = false); + bool enable_mapping_timestamp_tz = false, FileMetaCache* file_meta_cache = nullptr, + bool enable_file_meta_memory_cache = true); ~OrcReader() override; static format::ColumnDefinition row_position_column_definition(); @@ -92,12 +97,23 @@ class OrcReader final : public format::FileReader { RuntimeProfile::Counter* read_row_groups = nullptr; // RowGroupsReadNum RuntimeProfile::Counter* lazy_read_filtered_rows = nullptr; RuntimeProfile::Counter* open_file_num = nullptr; + RuntimeProfile::Counter* file_footer_read_calls = nullptr; + RuntimeProfile::Counter* file_footer_hit_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_memory_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_miss_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_read_disk_cache_time = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache_time = nullptr; }; class OrcFilterImpl; void _init_profile() override; void _collect_profile() const; + void _update_file_meta_cache_profile() const; + Status _init_orc_reader(RuntimeState* state); + Status _create_orc_reader(const std::string* serialized_file_tail, RuntimeState* state); DataTypePtr _convert_to_doris_type(const ::orc::Type& type) const; DataTypePtr _convert_list_to_doris_type(const ::orc::Type& type) const; @@ -180,6 +196,8 @@ class OrcReader final : public format::FileReader { OrcProfile _orc_profile; // RuntimeProfile counters std::optional _global_rowid_context; bool _enable_mapping_timestamp_tz = false; + FileMetaCache* _file_meta_cache = nullptr; + bool _enable_file_meta_memory_cache = true; }; } // namespace doris::format::orc diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp index 52151c7942af2b..5cd6c2e61a10c8 100644 --- a/be/src/format_v2/parquet/parquet_file_context.cpp +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -32,9 +32,11 @@ #include "common/check.h" #include "common/config.h" +#include "common/logging.h" #include "io/cache/cached_remote_file_reader.h" #include "io/file_factory.h" #include "io/fs/buffered_reader.h" +#include "io/fs/file_meta_cache.h" #include "io/fs/file_reader.h" #include "io/fs/tracing_file_reader.h" #include "io/io_common.h" @@ -535,19 +537,93 @@ Status arrow_status_to_doris_status(const arrow::Status& status) { } Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, - bool enable_page_cache, - const io::FileDescription& file_description) { + bool enable_page_cache, const io::FileDescription& file_description, + FileMetaCache* file_meta_cache, + FileMetaCacheProfile* file_meta_cache_profile, + int64_t* file_footer_read_calls, + bool enable_file_meta_memory_cache) { DORIS_CHECK(input_file_reader != nullptr); + DORIS_CHECK(file_footer_read_calls != nullptr); auto page_cache_file_key = build_page_cache_file_key(*input_file_reader, file_description); - arrow_file = std::make_shared(std::move(input_file_reader), io_ctx, - enable_page_cache, - std::move(page_cache_file_key)); + const std::string file_meta_cache_key = + file_meta_cache != nullptr ? FileMetaCache::get_key(input_file_reader, file_description) + : std::string {}; + const int64_t file_size = file_description.file_size == -1 + ? static_cast(input_file_reader->size()) + : file_description.file_size; + std::shared_ptr<::parquet::FileMetaData> cached_metadata; + bool metadata_from_cache = false; + FileMetaCacheContext file_meta_cache_context { + .format = FileMetaCacheFormat::PARQUET_V2, + .key = file_meta_cache_key, + .modification_time = file_description.mtime, + .file_size = file_size, + .enable_memory_cache = enable_file_meta_memory_cache && file_meta_cache != nullptr && + file_meta_cache->enabled()}; + const std::string memory_cache_key = + file_meta_cache != nullptr + ? FileMetaCache::get_memory_cache_key(file_meta_cache_context) + : std::string {}; + const auto reader_properties = ::parquet::default_reader_properties(); try { - // TODO: Cache parquet metadata in file system layer to avoid repeated metadata read for same file. - this->file_reader = ::parquet::ParquetFileReader::Open( - arrow_file, ::parquet::default_reader_properties()); + if (file_meta_cache != nullptr) { + ObjLRUCache::CacheHandle meta_cache_handle; + std::string serialized_metadata; + const FileMetaCacheLookupResult lookup_result = + file_meta_cache->lookup(file_meta_cache_context, &meta_cache_handle, + &serialized_metadata, file_meta_cache_profile); + if (lookup_result.state == FileMetaCacheLookupState::MEMORY_HIT) { + const auto* cached_metadata_ptr = + meta_cache_handle.data>(); + DORIS_CHECK(cached_metadata_ptr != nullptr); + cached_metadata = *cached_metadata_ptr; + metadata_from_cache = true; + } else if (lookup_result.state == FileMetaCacheLookupState::PERSISTED_HIT) { + try { + uint32_t metadata_len = static_cast(serialized_metadata.size()); + cached_metadata = ::parquet::FileMetaData::Make( + serialized_metadata.data(), &metadata_len, reader_properties); + if (file_meta_cache_context.enable_memory_cache) { + auto cached_metadata_holder = + std::make_unique>( + cached_metadata); + ObjLRUCache::CacheHandle insert_handle; + file_meta_cache->insert(memory_cache_key, cached_metadata_holder, + &insert_handle); + } + metadata_from_cache = true; + } catch (const std::exception& e) { + VLOG_DEBUG << "ignore invalid parquet file meta persistent cache payload: " + << e.what(); + file_meta_cache->invalidate_persistent_cache(file_meta_cache_context); + cached_metadata.reset(); + } + } + } + arrow_file = std::make_shared(std::move(input_file_reader), io_ctx, + enable_page_cache, + std::move(page_cache_file_key)); + this->file_reader = + ::parquet::ParquetFileReader::Open(arrow_file, reader_properties, cached_metadata); metadata = this->file_reader->metadata(); schema = metadata != nullptr ? metadata->schema() : nullptr; + if (!metadata_from_cache) { + ++(*file_footer_read_calls); + if (file_meta_cache != nullptr && metadata != nullptr) { + auto cached_metadata_holder = + std::make_unique>(metadata); + ObjLRUCache::CacheHandle insert_handle; + if (FileMetaCache::is_persistent_cache_payload_size_allowed(metadata->size())) { + std::string serialized_metadata = metadata->SerializeToString(); + file_meta_cache->insert(file_meta_cache_context, cached_metadata_holder, + &insert_handle, serialized_metadata, + file_meta_cache_profile); + } else if (file_meta_cache_context.enable_memory_cache) { + file_meta_cache->insert(memory_cache_key, cached_metadata_holder, + &insert_handle); + } + } + } } catch (const ::parquet::ParquetException& e) { if (io_ctx != nullptr && io_ctx->should_stop && std::string_view(e.what()).find("stop") != std::string_view::npos) { diff --git a/be/src/format_v2/parquet/parquet_file_context.h b/be/src/format_v2/parquet/parquet_file_context.h index 0dca52244957d7..00e5277b8811bf 100644 --- a/be/src/format_v2/parquet/parquet_file_context.h +++ b/be/src/format_v2/parquet/parquet_file_context.h @@ -32,6 +32,8 @@ struct IOContext; } // namespace doris::io namespace doris { +class FileMetaCache; +struct FileMetaCacheProfile; class RuntimeProfile; } // namespace doris @@ -112,7 +114,9 @@ struct ParquetFileContext { const ::parquet::SchemaDescriptor* schema = nullptr; // physical leaf column schema Status open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, bool enable_page_cache, - const io::FileDescription& file_description); + const io::FileDescription& file_description, FileMetaCache* file_meta_cache, + FileMetaCacheProfile* file_meta_cache_profile, int64_t* file_footer_read_calls, + bool enable_file_meta_memory_cache = true); // Register file ranges that belong to selected Parquet column chunks. Arrow still owns page // decoding, so v2 caches the serialized bytes read inside these ranges and excludes // footer/metadata reads that happen before registration. diff --git a/be/src/format_v2/parquet/parquet_profile.cpp b/be/src/format_v2/parquet/parquet_profile.cpp index d41ff295ec4b3e..3ee99db7619824 100644 --- a/be/src/format_v2/parquet/parquet_profile.cpp +++ b/be/src/format_v2/parquet/parquet_profile.cpp @@ -103,6 +103,18 @@ void ParquetProfile::init(RuntimeProfile* profile) { ADD_CHILD_TIMER_WITH_LEVEL(profile, "RowGroupFilterTime", parquet_profile, 1); file_footer_read_calls = ADD_COUNTER_WITH_LEVEL(profile, "FileFooterReadCalls", TUnit::UNIT, 1); file_footer_hit_cache = ADD_COUNTER_WITH_LEVEL(profile, "FileFooterHitCache", TUnit::UNIT, 1); + file_footer_hit_memory_cache = + ADD_COUNTER_WITH_LEVEL(profile, "FileFooterHitMemoryCache", TUnit::UNIT, 1); + file_footer_hit_disk_cache = + ADD_COUNTER_WITH_LEVEL(profile, "FileFooterHitDiskCache", TUnit::UNIT, 1); + file_footer_miss_disk_cache = + ADD_COUNTER_WITH_LEVEL(profile, "FileFooterMissDiskCache", TUnit::UNIT, 1); + file_footer_write_disk_cache = + ADD_COUNTER_WITH_LEVEL(profile, "FileFooterWriteDiskCache", TUnit::UNIT, 1); + file_footer_read_disk_cache_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileFooterReadDiskCacheTime", parquet_profile, 1); + file_footer_write_disk_cache_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileFooterWriteDiskCacheTime", parquet_profile, 1); decompress_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecompressTime", parquet_profile, 1); decompress_cnt = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DecompressCount", TUnit::UNIT, parquet_profile, 1); diff --git a/be/src/format_v2/parquet/parquet_profile.h b/be/src/format_v2/parquet/parquet_profile.h index 27f9818f4a0dcd..8e5edba1b1ba60 100644 --- a/be/src/format_v2/parquet/parquet_profile.h +++ b/be/src/format_v2/parquet/parquet_profile.h @@ -116,6 +116,12 @@ struct ParquetProfile { RuntimeProfile::Counter* open_file_num = nullptr; RuntimeProfile::Counter* file_footer_read_calls = nullptr; RuntimeProfile::Counter* file_footer_hit_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_memory_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_miss_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_read_disk_cache_time = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache_time = nullptr; RuntimeProfile::Counter* row_group_filter_time = nullptr; RuntimeProfile::Counter* page_index_read_calls = nullptr; diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index 753b3628bfa19b..f7091eedfab59d 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -40,6 +40,7 @@ #include "format_v2/parquet/parquet_scan.h" #include "format_v2/parquet/parquet_statistics.h" #include "format_v2/parquet/reader/column_reader.h" +#include "io/fs/file_meta_cache.h" #include "io/io_common.h" #include "runtime/runtime_state.h" @@ -328,10 +329,13 @@ ParquetReader::ParquetReader(std::shared_ptr& system_p std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context, - bool enable_mapping_timestamp_tz) + bool enable_mapping_timestamp_tz, FileMetaCache* file_meta_cache, + bool enable_file_meta_memory_cache) : FileReader(system_properties, file_description, io_ctx, profile), _global_rowid_context(global_rowid_context), - _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz) {} + _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz), + _file_meta_cache(file_meta_cache), + _enable_file_meta_memory_cache(enable_file_meta_memory_cache) {} ParquetReader::~ParquetReader() = default; @@ -364,8 +368,36 @@ Status ParquetReader::init(RuntimeState* state) { _state->scheduler.set_merge_read_options(_profile, merge_read_slice_size); _state->scheduler.set_batch_size(_batch_size); // Open parquet file and parse metadata to get file schema. - RETURN_IF_ERROR(_state->file_context.open(_tracing_file_reader, _io_ctx.get(), - _state->enable_page_cache, *_file_description)); + FileMetaCacheProfile file_meta_cache_profile { + .hit_cache = &_reader_statistics.file_footer_hit_cache, + .hit_memory_cache = &_reader_statistics.file_footer_hit_memory_cache, + .hit_disk_cache = &_reader_statistics.file_footer_hit_disk_cache, + .miss_disk_cache = &_reader_statistics.file_footer_miss_disk_cache, + .write_disk_cache = &_reader_statistics.file_footer_write_disk_cache, + .read_disk_cache_time = &_reader_statistics.file_footer_read_disk_cache_time, + .write_disk_cache_time = &_reader_statistics.file_footer_write_disk_cache_time}; + RETURN_IF_ERROR(_state->file_context.open( + _tracing_file_reader, _io_ctx.get(), _state->enable_page_cache, *_file_description, + _file_meta_cache, &file_meta_cache_profile, &_reader_statistics.file_footer_read_calls, + _enable_file_meta_memory_cache)); + if (_profile != nullptr) { + COUNTER_UPDATE(_parquet_profile.file_footer_read_calls, + _reader_statistics.file_footer_read_calls); + COUNTER_UPDATE(_parquet_profile.file_footer_hit_cache, + _reader_statistics.file_footer_hit_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_hit_memory_cache, + _reader_statistics.file_footer_hit_memory_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_hit_disk_cache, + _reader_statistics.file_footer_hit_disk_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_miss_disk_cache, + _reader_statistics.file_footer_miss_disk_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_write_disk_cache, + _reader_statistics.file_footer_write_disk_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_read_disk_cache_time, + _reader_statistics.file_footer_read_disk_cache_time); + COUNTER_UPDATE(_parquet_profile.file_footer_write_disk_cache_time, + _reader_statistics.file_footer_write_disk_cache_time); + } // Build file schema from parquet metadata. // A file reader may expose raw file identifiers, such as Parquet field_id, through ColumnDefinition::identifier RETURN_IF_ERROR( diff --git a/be/src/format_v2/parquet/parquet_reader.h b/be/src/format_v2/parquet/parquet_reader.h index 6c8e88cc27a9b6..654b8850cd26f3 100644 --- a/be/src/format_v2/parquet/parquet_reader.h +++ b/be/src/format_v2/parquet/parquet_reader.h @@ -27,6 +27,7 @@ #include "format_v2/parquet/parquet_scan.h" namespace doris { +class FileMetaCache; namespace io { struct IOContext; } // namespace io @@ -46,7 +47,9 @@ class ParquetReader : public format::FileReader { std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context = std::nullopt, - bool enable_mapping_timestamp_tz = false); + bool enable_mapping_timestamp_tz = false, + FileMetaCache* file_meta_cache = nullptr, + bool enable_file_meta_memory_cache = true); ~ParquetReader() override; Status init(RuntimeState* state) override; @@ -89,6 +92,8 @@ class ParquetReader : public format::FileReader { std::optional _global_rowid_context; // global RowId context size_t _batch_size = ParquetScanScheduler::DEFAULT_READ_BATCH_SIZE; bool _enable_mapping_timestamp_tz = false; // whether UTC timestamps are mapped to TIMESTAMPTZ + FileMetaCache* _file_meta_cache = nullptr; + bool _enable_file_meta_memory_cache = true; }; } // namespace doris::format::parquet diff --git a/be/src/format_v2/table/hudi_reader.cpp b/be/src/format_v2/table/hudi_reader.cpp index 4294d51d043d9e..63473e81fe5fc3 100644 --- a/be/src/format_v2/table/hudi_reader.cpp +++ b/be/src/format_v2/table/hudi_reader.cpp @@ -146,6 +146,8 @@ Status HudiHybridReader::_init_child_reader(format::TableReader* reader, .scanner_profile = _scanner_profile, .push_down_agg_type = _push_down_agg_type, .condition_cache_digest = _condition_cache_digest, + .file_meta_cache = _file_meta_cache, + .enable_file_meta_memory_cache = _enable_file_meta_memory_cache, })); // Zero means no adaptive prediction has been produced yet. Preserve the child's normal // runtime default until FileScannerV2 supplies the first positive prediction. diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index b817a6d1eadbf5..ef18c0cc161b6e 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -648,11 +648,11 @@ Status IcebergTableReader::_create_delete_file_reader(const TIcebergDeleteFileDe if (delete_file.file_format == TFileFormatType::FORMAT_PARQUET) { *reader = std::make_unique( system_properties, file_description, io_ctx, _scanner_profile, std::nullopt, - enable_mapping_timestamp_tz); + enable_mapping_timestamp_tz, _file_meta_cache, _enable_file_meta_memory_cache); } else { - *reader = std::make_unique(system_properties, file_description, - io_ctx, _scanner_profile, std::nullopt, - enable_mapping_timestamp_tz); + *reader = std::make_unique( + system_properties, file_description, io_ctx, _scanner_profile, std::nullopt, + enable_mapping_timestamp_tz, _file_meta_cache, _enable_file_meta_memory_cache); } RETURN_IF_ERROR((*reader)->init(_runtime_state)); return Status::OK(); diff --git a/be/src/format_v2/table/paimon_reader.cpp b/be/src/format_v2/table/paimon_reader.cpp index 258bbb5c021dc0..e410af4c4b35a5 100644 --- a/be/src/format_v2/table/paimon_reader.cpp +++ b/be/src/format_v2/table/paimon_reader.cpp @@ -174,6 +174,8 @@ Status PaimonHybridReader::_init_child_reader(format::TableReader* reader, .scanner_profile = _scanner_profile, .push_down_agg_type = _push_down_agg_type, .condition_cache_digest = _condition_cache_digest, + .file_meta_cache = _file_meta_cache, + .enable_file_meta_memory_cache = _enable_file_meta_memory_cache, })); // Zero means no adaptive prediction has been produced yet. Preserve the child's normal // runtime default until FileScannerV2 supplies the first positive prediction. diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index f30a1d567cf562..ab12058c7ca2fd 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -525,6 +525,8 @@ Status TableReader::init(TableReadOptions&& options) { _io_ctx = options.io_ctx; _runtime_state = options.runtime_state; _scanner_profile = options.scanner_profile; + _file_meta_cache = options.file_meta_cache; + _enable_file_meta_memory_cache = options.enable_file_meta_memory_cache; _file_slot_descs = options.file_slot_descs; _push_down_agg_type = options.push_down_agg_type; _condition_cache_digest = options.condition_cache_digest; @@ -754,13 +756,15 @@ Status TableReader::create_file_reader(std::unique_ptr* reader) { if (_format == FileFormat::PARQUET) { *reader = std::make_unique( _system_properties, _current_task->data_file, _io_ctx, _scanner_profile, - _global_rowid_context, enable_mapping_timestamp_tz); + _global_rowid_context, enable_mapping_timestamp_tz, _file_meta_cache, + _enable_file_meta_memory_cache); return Status::OK(); } if (_format == FileFormat::ORC) { *reader = std::make_unique( _system_properties, _current_task->data_file, _io_ctx, _scanner_profile, - _global_rowid_context, enable_mapping_timestamp_tz); + _global_rowid_context, enable_mapping_timestamp_tz, _file_meta_cache, + _enable_file_meta_memory_cache); return Status::OK(); } if (_format == FileFormat::CSV) { diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index 8d99195c21da60..7ec0437b060759 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -69,6 +69,7 @@ namespace doris { class Block; +class FileMetaCache; struct DeleteFileDesc; class RuntimeState; } // namespace doris @@ -137,6 +138,8 @@ struct TableReadOptions { const TPushAggOp::type push_down_agg_type = TPushAggOp::type::NONE; // Digest of stable pushed-down predicates. A zero digest disables condition cache. uint64_t condition_cache_digest = 0; + FileMetaCache* file_meta_cache = nullptr; + bool enable_file_meta_memory_cache = true; }; struct SplitReadOptions { @@ -1591,6 +1594,8 @@ class TableReader { std::shared_ptr _io_ctx; RuntimeState* _runtime_state; RuntimeProfile* _scanner_profile; + FileMetaCache* _file_meta_cache = nullptr; + bool _enable_file_meta_memory_cache = true; const std::vector* _file_slot_descs = nullptr; FileFormat _format; TPushAggOp::type _push_down_agg_type = TPushAggOp::type::NONE; diff --git a/be/src/io/file_factory.cpp b/be/src/io/file_factory.cpp index 7dee26bf99e67d..ec2b90364adcfa 100644 --- a/be/src/io/file_factory.cpp +++ b/be/src/io/file_factory.cpp @@ -51,12 +51,29 @@ #include "runtime/runtime_state.h" #include "util/s3_uri.h" #include "util/s3_util.h" +#include "util/string_util.h" #include "util/uid_util.h" namespace doris { constexpr std::string_view RANDOM_CACHE_BASE_PATH = "random"; +namespace { + +void append_identity_property(const StringCaseMap& properties, const char* name, + std::string* identity) { + auto it = properties.find(name); + if (it == properties.end()) { + return; + } + identity->push_back('\0'); + identity->append(name); + identity->push_back('='); + identity->append(it->second); +} + +} // namespace + io::FileReaderOptions FileFactory::get_reader_options(const TQueryOptions& option, const io::FileDescription& fd) { io::FileReaderOptions opts { @@ -119,7 +136,7 @@ Result FileFactory::create_fs(const io::FSPropertiesRef& fs_ return io::S3FileSystem::create(std::move(s3_conf), io::FileSystem::TMP_FS_ID); } case TFileType::FILE_HDFS: { - std::string fs_name = _get_fs_name(file_description); + std::string fs_name = get_fs_name(file_description); return io::HdfsFileSystem::create(*fs_properties.properties, fs_name, io::FileSystem::TMP_FS_ID, nullptr); } @@ -137,7 +154,7 @@ Result FileFactory::create_fs(const io::FSPropertiesRef& fs_ } } -std::string FileFactory::_get_fs_name(const io::FileDescription& file_description) { +std::string FileFactory::get_fs_name(const io::FileDescription& file_description) { // If the destination path contains a schema, use the schema directly. // If not, use origin file_description.fs_name // Because the default fsname in file_description.fs_name maybe different from @@ -156,6 +173,30 @@ std::string FileFactory::_get_fs_name(const io::FileDescription& file_descriptio return fs_name; } +std::string FileFactory::get_file_system_identity(const io::FileSystemProperties& system_properties, + const io::FileDescription& file_description) { + switch (system_properties.system_type) { + case TFileType::FILE_HDFS: { + io::FileDescription identity_description = file_description; + if (identity_description.fs_name.empty()) { + identity_description.fs_name = system_properties.hdfs_params.fs_name; + } + return get_fs_name(identity_description); + } + case TFileType::FILE_S3: { + StringCaseMap properties(system_properties.properties.begin(), + system_properties.properties.end()); + std::string identity = "s3"; + append_identity_property(properties, "AWS_ENDPOINT", &identity); + append_identity_property(properties, "AWS_REGION", &identity); + append_identity_property(properties, "provider", &identity); + return identity; + } + default: + return file_description.fs_name; + } +} + Result FileFactory::create_file_writer( TFileType::type type, ExecEnv* env, const std::vector& broker_addresses, const std::map& properties, const std::string& path, @@ -238,15 +279,10 @@ Result FileFactory::_create_file_reader_internal( } case TFileType::FILE_HDFS: { std::shared_ptr handler; - // FIXME(plat1ko): Explain the difference between `system_properties.hdfs_params.fs_name` - // and `file_description.fs_name`, it's so confused. - const auto* fs_name = &file_description.fs_name; - if (fs_name->empty()) { - fs_name = &system_properties.hdfs_params.fs_name; - } + const std::string fs_name = get_file_system_identity(system_properties, file_description); RETURN_IF_ERROR_RESULT(ExecEnv::GetInstance()->hdfs_mgr()->get_or_create_fs( - system_properties.hdfs_params, *fs_name, &handler)); - return io::HdfsFileReader::create(file_description.path, handler->hdfs_fs, *fs_name, + system_properties.hdfs_params, fs_name, &handler)); + return io::HdfsFileReader::create(file_description.path, handler->hdfs_fs, fs_name, reader_options, profile) .and_then([&](auto&& reader) { return io::create_cached_file_reader(std::move(reader), reader_options); diff --git a/be/src/io/file_factory.h b/be/src/io/file_factory.h index 3ff7f3decaf84c..84c61ac60ad4b4 100644 --- a/be/src/io/file_factory.h +++ b/be/src/io/file_factory.h @@ -99,6 +99,11 @@ class FileFactory { static Result create_fs(const io::FSPropertiesRef& fs_properties, const io::FileDescription& file_description); + static std::string get_fs_name(const io::FileDescription& file_description); + + static std::string get_file_system_identity(const io::FileSystemProperties& system_properties, + const io::FileDescription& file_description); + /// Create FileWriter without FS static Result create_file_writer( TFileType::type type, ExecEnv* env, @@ -135,8 +140,6 @@ class FileFactory { } private: - static std::string _get_fs_name(const io::FileDescription& file_description); - /// Create FileReader without FS static Result _create_file_reader_internal( const io::FileSystemProperties& system_properties, diff --git a/be/src/io/fs/file_meta_cache.cpp b/be/src/io/fs/file_meta_cache.cpp index f97b2f80cd6ee6..34a3311d229713 100644 --- a/be/src/io/fs/file_meta_cache.cpp +++ b/be/src/io/fs/file_meta_cache.cpp @@ -17,27 +17,169 @@ #include "io/fs/file_meta_cache.h" +#include + +#include "common/config.h" +#include "common/logging.h" +#include "util/stopwatch.hpp" + namespace doris { +namespace { + +void update_profile_counter(int64_t* counter, int64_t value = 1) { + if (counter != nullptr) { + *counter += value; + } +} + +bool is_persistent_cache_identity_stable(const FileMetaCacheContext& context) { + return context.modification_time > 0; +} + +} // namespace + +FileMetaCache::FileMetaCache(int64_t capacity, + std::unique_ptr persistent_cache) + : _cache(capacity), _persistent_cache(std::move(persistent_cache)) {} std::string FileMetaCache::get_key(const std::string file_name, int64_t modification_time, int64_t file_size) { std::string meta_cache_key; - meta_cache_key.resize(file_name.size() + sizeof(int64_t)); + meta_cache_key.resize(file_name.size() + 2 * sizeof(int64_t)); memcpy(meta_cache_key.data(), file_name.data(), file_name.size()); - if (modification_time != 0) { - memcpy(meta_cache_key.data() + file_name.size(), &modification_time, sizeof(int64_t)); - } else { - memcpy(meta_cache_key.data() + file_name.size(), &file_size, sizeof(int64_t)); - } + memcpy(meta_cache_key.data() + file_name.size(), &modification_time, sizeof(int64_t)); + memcpy(meta_cache_key.data() + file_name.size() + sizeof(int64_t), &file_size, sizeof(int64_t)); return meta_cache_key; } std::string FileMetaCache::get_key(io::FileReaderSPtr file_reader, const io::FileDescription& _file_description) { + const std::string& file_path = file_reader->path().native(); + std::string file_identity; + if (_file_description.fs_name.empty()) { + file_identity = file_path; + } else { + file_identity.reserve(_file_description.fs_name.size() + 1 + file_path.size()); + file_identity.append(_file_description.fs_name); + file_identity.push_back('\0'); + file_identity.append(file_path); + } return FileMetaCache::get_key( - file_reader->path().native(), _file_description.mtime, + file_identity, _file_description.mtime, _file_description.file_size == -1 ? file_reader->size() : _file_description.file_size); } +std::string FileMetaCache::get_memory_cache_key(FileMetaCacheFormat format, std::string_view key) { + std::string memory_cache_key; + memory_cache_key.reserve(key.size() + sizeof(uint8_t)); + memory_cache_key.push_back(static_cast(format)); + memory_cache_key.append(key.data(), key.size()); + return memory_cache_key; +} + +bool FileMetaCache::is_persistent_cache_configured() { + return config::enable_external_file_meta_disk_cache; +} + +bool FileMetaCache::is_persistent_cache_enabled() { + return is_persistent_cache_configured() && + config::external_file_meta_disk_cache_max_entry_bytes > 0; +} + +bool FileMetaCache::is_persistent_cache_payload_size_allowed(uint64_t payload_size) { + const int64_t max_entry_bytes = config::external_file_meta_disk_cache_max_entry_bytes; + return config::enable_external_file_meta_disk_cache && max_entry_bytes > 0 && + payload_size != 0 && std::cmp_less_equal(payload_size, max_entry_bytes); +} + +FileMetaCacheLookupResult FileMetaCache::lookup(const FileMetaCacheContext& context, + ObjLRUCache::CacheHandle* handle, + std::string* serialized_meta, + FileMetaCacheProfile* profile) { + DCHECK(handle != nullptr); + DCHECK(serialized_meta != nullptr); + *handle = ObjLRUCache::CacheHandle(); + if (context.enable_memory_cache && lookup(get_memory_cache_key(context), handle)) { + serialized_meta->clear(); + if (profile != nullptr) { + update_profile_counter(profile->hit_cache); + update_profile_counter(profile->hit_memory_cache); + } + return {.state = FileMetaCacheLookupState::MEMORY_HIT}; + } + + FileMetaCacheLookupResult result; + int64_t persisted_read_time = 0; + if (lookup_persistent_cache(context, serialized_meta, &persisted_read_time)) { + result.state = FileMetaCacheLookupState::PERSISTED_HIT; + if (profile != nullptr) { + update_profile_counter(profile->hit_cache); + update_profile_counter(profile->hit_disk_cache); + update_profile_counter(profile->read_disk_cache_time, persisted_read_time); + } + } else if (is_persistent_cache_enabled() && is_persistent_cache_identity_stable(context) && + profile != nullptr) { + update_profile_counter(profile->miss_disk_cache); + update_profile_counter(profile->read_disk_cache_time, persisted_read_time); + } + return result; +} + +void FileMetaCache::invalidate_persistent_cache(const FileMetaCacheContext& context) { + if (_persistent_cache != nullptr) { + _persistent_cache->remove(context.format, context.key); + } +} + +bool FileMetaCache::lookup_persistent_cache(const FileMetaCacheContext& context, + std::string* payload, int64_t* read_time) { + DCHECK(payload != nullptr); + DCHECK(read_time != nullptr); + payload->clear(); + *read_time = 0; + if (!is_persistent_cache_enabled() || _persistent_cache == nullptr || + !is_persistent_cache_identity_stable(context)) { + return false; + } + + MonotonicStopWatch watch; + watch.start(); + Status status = _persistent_cache->read(context.format, context.key, context.modification_time, + context.file_size, payload); + *read_time = watch.elapsed_time(); + if (!status.ok()) { + payload->clear(); + VLOG_DEBUG << "lookup file meta persistent cache failed: " << status; + return false; + } + if (!is_persistent_cache_payload_size_allowed(static_cast(payload->size()))) { + payload->clear(); + _persistent_cache->remove(context.format, context.key); + return false; + } + return true; +} + +bool FileMetaCache::insert_persistent_cache(const FileMetaCacheContext& context, + std::string_view payload, int64_t* write_time) { + DCHECK(write_time != nullptr); + *write_time = 0; + if (!is_persistent_cache_payload_size_allowed(static_cast(payload.size())) || + _persistent_cache == nullptr || !is_persistent_cache_identity_stable(context)) { + return false; + } + + MonotonicStopWatch watch; + watch.start(); + Status status = _persistent_cache->write(context.format, context.key, context.modification_time, + context.file_size, payload); + *write_time = watch.elapsed_time(); + if (!status.ok()) { + VLOG_DEBUG << "insert file meta persistent cache failed: " << status; + return false; + } + return true; +} + } // namespace doris diff --git a/be/src/io/fs/file_meta_cache.h b/be/src/io/fs/file_meta_cache.h index 0c62c963ce122d..a7c93496dde442 100644 --- a/be/src/io/fs/file_meta_cache.h +++ b/be/src/io/fs/file_meta_cache.h @@ -17,18 +17,79 @@ #pragma once +#include +#include +#include +#include + +#include "common/status.h" #include "io/file_factory.h" #include "io/fs/file_reader_writer_fwd.h" #include "util/obj_lru_cache.h" namespace doris { +enum class FileMetaCacheFormat : uint8_t { + PARQUET = 1, + ORC = 2, + PARQUET_V2 = 3, + ORC_V2 = 4, +}; + +struct FileMetaCacheContext { + FileMetaCacheFormat format; + const std::string& key; + int64_t modification_time = 0; + int64_t file_size = 0; + bool enable_memory_cache = true; +}; + +enum class FileMetaCacheLookupState { + MEMORY_HIT, + PERSISTED_HIT, + MISS, +}; + +struct FileMetaCacheLookupResult { + FileMetaCacheLookupState state = FileMetaCacheLookupState::MISS; +}; + +struct FileMetaCacheInsertResult { + bool memory_inserted = false; + bool persisted_inserted = false; +}; + +struct FileMetaCacheProfile { + int64_t* hit_cache = nullptr; + int64_t* hit_memory_cache = nullptr; + int64_t* hit_disk_cache = nullptr; + int64_t* miss_disk_cache = nullptr; + int64_t* write_disk_cache = nullptr; + int64_t* read_disk_cache_time = nullptr; + int64_t* write_disk_cache_time = nullptr; +}; + +class FileMetaPersistentCache { +public: + virtual ~FileMetaPersistentCache() = default; + + virtual Status read(FileMetaCacheFormat format, const std::string& key, + int64_t modification_time, int64_t file_size, std::string* payload) = 0; + + virtual Status write(FileMetaCacheFormat format, const std::string& key, + int64_t modification_time, int64_t file_size, + std::string_view payload) = 0; + + virtual void remove(FileMetaCacheFormat format, const std::string& key) = 0; +}; + // A file meta cache depends on a LRU cache. // Such as parsed parquet footer. // The capacity will limit the number of cache entries in cache. class FileMetaCache { public: - FileMetaCache(int64_t capacity) : _cache(capacity) {} + explicit FileMetaCache(int64_t capacity, + std::unique_ptr persistent_cache = nullptr); FileMetaCache(const FileMetaCache&) = delete; const FileMetaCache& operator=(const FileMetaCache&) = delete; @@ -40,6 +101,14 @@ class FileMetaCache { static std::string get_key(io::FileReaderSPtr file_reader, const io::FileDescription& _file_description); + static std::string get_memory_cache_key(FileMetaCacheFormat format, std::string_view key); + static std::string get_memory_cache_key(const FileMetaCacheContext& context) { + return get_memory_cache_key(context.format, context.key); + } + + static bool is_persistent_cache_configured(); + static bool is_persistent_cache_enabled(); + static bool is_persistent_cache_payload_size_allowed(uint64_t payload_size); bool lookup(const std::string& key, ObjLRUCache::CacheHandle* handle) { return _cache.lookup({key}, handle); @@ -50,10 +119,56 @@ class FileMetaCache { _cache.insert({key}, value, handle); } + template + bool insert(const std::string& key, std::unique_ptr& value, + ObjLRUCache::CacheHandle* handle) { + DCHECK(value != nullptr); + if (!_cache.enabled()) { + return false; + } + _cache.insert({key}, value.release(), handle); + return true; + } + bool enabled() const { return _cache.enabled(); } + FileMetaCacheLookupResult lookup(const FileMetaCacheContext& context, + ObjLRUCache::CacheHandle* handle, std::string* serialized_meta, + FileMetaCacheProfile* profile = nullptr); + + void invalidate_persistent_cache(const FileMetaCacheContext& context); + + template + FileMetaCacheInsertResult insert(const FileMetaCacheContext& context, std::unique_ptr& value, + ObjLRUCache::CacheHandle* handle, + std::string_view serialized_meta, + FileMetaCacheProfile* profile = nullptr) { + FileMetaCacheInsertResult result; + int64_t persisted_write_time = 0; + result.persisted_inserted = + insert_persistent_cache(context, serialized_meta, &persisted_write_time); + if (result.persisted_inserted && profile != nullptr) { + if (profile->write_disk_cache != nullptr) { + ++(*profile->write_disk_cache); + } + if (profile->write_disk_cache_time != nullptr) { + *profile->write_disk_cache_time += persisted_write_time; + } + } + if (context.enable_memory_cache) { + result.memory_inserted = insert(get_memory_cache_key(context), value, handle); + } + return result; + } + private: + bool lookup_persistent_cache(const FileMetaCacheContext& context, std::string* payload, + int64_t* read_time); + bool insert_persistent_cache(const FileMetaCacheContext& context, std::string_view payload, + int64_t* write_time); + ObjLRUCache _cache; + std::unique_ptr _persistent_cache; }; } // namespace doris diff --git a/be/src/io/fs/file_meta_disk_cache.cpp b/be/src/io/fs/file_meta_disk_cache.cpp new file mode 100644 index 00000000000000..618e84f8658a8f --- /dev/null +++ b/be/src/io/fs/file_meta_disk_cache.cpp @@ -0,0 +1,371 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/fs/file_meta_disk_cache.h" + +#include +#include +#include + +#include +#include +#include +#include + +#include "common/logging.h" +#include "io/cache/block_file_cache.h" +#include "io/cache/block_file_cache_factory.h" +#include "io/cache/file_block.h" +#include "io/cache/file_cache_common.h" +#include "util/coding.h" +#include "util/slice.h" + +namespace doris { +namespace { + +constexpr uint8_t FILE_META_DISK_CACHE_VERSION = 1; +constexpr size_t FILE_META_DISK_CACHE_HEADER_SIZE_PREFIX = sizeof(uint32_t); +constexpr uint32_t FILE_META_DISK_CACHE_MAX_HEADER_SIZE = 1024; + +std::string_view format_name(FileMetaCacheFormat format) { + switch (format) { + case FileMetaCacheFormat::PARQUET: + return "parquet"; + case FileMetaCacheFormat::ORC: + return "orc"; + case FileMetaCacheFormat::PARQUET_V2: + return "parquet_v2"; + case FileMetaCacheFormat::ORC_V2: + return "orc_v2"; + } + DCHECK(false) << "unknown file meta cache format"; + return "unknown"; +} + +Status validate_format(uint32_t format) { + switch (static_cast(format)) { + case FileMetaCacheFormat::PARQUET: + case FileMetaCacheFormat::ORC: + case FileMetaCacheFormat::PARQUET_V2: + case FileMetaCacheFormat::ORC_V2: + return Status::OK(); + } + return Status::NotFound("file meta disk cache format mismatch"); +} + +Status parse_header(std::string_view serialized_header, + io::cache::FileMetaCacheEntryHeaderPB* parsed) { + DCHECK(parsed != nullptr); + DCHECK_LE(serialized_header.size(), FILE_META_DISK_CACHE_MAX_HEADER_SIZE); + if (!parsed->ParseFromArray(serialized_header.data(), + static_cast(serialized_header.size()))) { + return Status::NotFound("failed to parse file meta disk cache header"); + } + if (!parsed->has_version() || !parsed->has_format() || !parsed->has_file_size() || + !parsed->has_modification_time() || !parsed->has_payload_size() || + !parsed->has_checksum()) { + return Status::NotFound("file meta disk cache header is incomplete"); + } + if (parsed->version() != FILE_META_DISK_CACHE_VERSION) { + return Status::NotFound("file meta disk cache version mismatch"); + } + return validate_format(parsed->format()); +} + +Status build_cache_value(FileMetaCacheFormat format, int64_t modification_time, int64_t file_size, + std::string_view payload, std::string* value, size_t* header_end) { + DCHECK(value != nullptr); + DCHECK(header_end != nullptr); + io::cache::FileMetaCacheEntryHeaderPB header; + header.set_version(FILE_META_DISK_CACHE_VERSION); + header.set_format(static_cast(format)); + header.set_file_size(file_size); + header.set_modification_time(modification_time); + header.set_payload_size(payload.size()); + header.set_checksum(crc32c::Crc32c(payload.data(), payload.size())); + + std::string serialized_header; + if (!header.SerializeToString(&serialized_header)) { + return Status::InternalError("failed to serialize file meta disk cache header"); + } + DCHECK_LE(serialized_header.size(), FILE_META_DISK_CACHE_MAX_HEADER_SIZE); + value->clear(); + value->reserve(FILE_META_DISK_CACHE_HEADER_SIZE_PREFIX + serialized_header.size() + + payload.size()); + put_fixed32_le(value, static_cast(serialized_header.size())); + value->append(serialized_header); + *header_end = value->size(); + value->append(payload.data(), payload.size()); + return Status::OK(); +} + +io::CacheContext build_meta_cache_context() { + io::CacheContext context; + context.cache_type = io::FileCacheType::INDEX; + context.query_id = TUniqueId(); + context.expiration_time = 0; + context.is_cold_data = false; + context.is_warmup = false; + return context; +} + +Status read_cached_range(io::BlockFileCache* cache, const io::UInt128Wrapper& hash, size_t offset, + size_t size, const io::CacheContext& context, std::string* output, + std::vector* const holders) { + DCHECK(cache != nullptr); + DCHECK(output != nullptr); + DCHECK(holders != nullptr); + output->clear(); + if (size == 0) { + return Status::OK(); + } + + io::FileBlocks blocks; + bool fully_covered = false; + RETURN_IF_ERROR(cache->get_downloaded_blocks_if_fully_covered(hash, offset, size, context, + &blocks, &fully_covered)); + if (!fully_covered) { + return Status::NotFound("file meta disk cache range is not cached"); + } + holders->emplace_back(std::move(blocks)); + const auto& holder = holders->back(); + + output->resize(size); + size_t copied_size = 0; + const size_t requested_right = offset + size - 1; + for (const auto& block : holder.file_blocks) { + const auto& range = block->range(); + const size_t read_left = std::max(range.left, offset); + const size_t read_right = std::min(range.right, requested_right); + DCHECK_LE(read_left, read_right); + const size_t read_size = read_right - read_left + 1; + RETURN_IF_ERROR(block->read(Slice(output->data() + copied_size, read_size), + read_left - range.left)); + copied_size += read_size; + } + DCHECK_EQ(copied_size, size); + return Status::OK(); +} + +Status read_cache_entry(io::BlockFileCache* cache, const io::UInt128Wrapper& hash, + FileMetaCacheFormat format, int64_t modification_time, int64_t file_size, + std::string* payload) { + DCHECK(cache != nullptr); + DCHECK(payload != nullptr); + payload->clear(); + + io::ReadStatistics stats; + io::CacheContext context = build_meta_cache_context(); + context.stats = &stats; + std::vector holders; + + std::string header_size_prefix; + RETURN_IF_ERROR(read_cached_range(cache, hash, 0, FILE_META_DISK_CACHE_HEADER_SIZE_PREFIX, + context, &header_size_prefix, &holders)); + + const uint32_t header_size = + decode_fixed32_le(reinterpret_cast(header_size_prefix.data())); + if (header_size == 0 || header_size > FILE_META_DISK_CACHE_MAX_HEADER_SIZE) { + return Status::NotFound("file meta disk cache header size is invalid"); + } + + std::string serialized_header; + RETURN_IF_ERROR(read_cached_range(cache, hash, FILE_META_DISK_CACHE_HEADER_SIZE_PREFIX, + header_size, context, &serialized_header, &holders)); + + io::cache::FileMetaCacheEntryHeaderPB parsed; + RETURN_IF_ERROR(parse_header(serialized_header, &parsed)); + if (parsed.format() != static_cast(format) || + parsed.modification_time() != modification_time || parsed.file_size() != file_size || + !FileMetaCache::is_persistent_cache_payload_size_allowed(parsed.payload_size())) { + return Status::NotFound("file meta disk cache header mismatch"); + } + + const size_t payload_offset = FILE_META_DISK_CACHE_HEADER_SIZE_PREFIX + header_size; + RETURN_IF_ERROR(read_cached_range(cache, hash, payload_offset, parsed.payload_size(), context, + payload, &holders)); + const uint32_t checksum = crc32c::Crc32c(payload->data(), payload->size()); + if (checksum != parsed.checksum()) { + payload->clear(); + return Status::NotFound("file meta disk cache checksum mismatch"); + } + + return Status::OK(); +} + +} // namespace + +std::array FileMetaDiskCache::_entry_locks; + +std::string FileMetaDiskCache::get_key(FileMetaCacheFormat format, + std::string_view file_meta_cache_key) { + std::string key; + key.reserve(32 + file_meta_cache_key.size()); + key.append("file_meta_cache:v1:"); + key.append(format_name(format)); + key.push_back(':'); + key.append(file_meta_cache_key.data(), file_meta_cache_key.size()); + return key; +} + +Status FileMetaDiskCache::read(FileMetaCacheFormat format, const std::string& file_meta_cache_key, + int64_t modification_time, int64_t file_size, std::string* payload) { + payload->clear(); + const std::string cache_key = get_key(format, file_meta_cache_key); + const auto hash = io::BlockFileCache::hash(cache_key); + io::BlockFileCache* cache = get_cache(hash); + if (cache == nullptr) { + return Status::NotFound("file meta disk cache is not available"); + } + if (!cache->get_async_open_success()) { + return Status::NotFound("file meta disk cache is still restoring"); + } + + Status status; + { + std::shared_lock entry_lock(get_entry_lock(hash)); + status = read_cache_entry(cache, hash, format, modification_time, file_size, payload); + if (status.ok()) { + return status; + } + payload->clear(); + } + + // Revalidate after upgrading the entry lock so a writer that completed in between is preserved. + std::unique_lock entry_lock(get_entry_lock(hash)); + status = read_cache_entry(cache, hash, format, modification_time, file_size, payload); + if (status.ok()) { + return status; + } + payload->clear(); + cache->remove_if_cached(hash); + return status; +} + +Status FileMetaDiskCache::write(FileMetaCacheFormat format, const std::string& file_meta_cache_key, + int64_t modification_time, int64_t file_size, + std::string_view payload) { + if (!FileMetaCache::is_persistent_cache_payload_size_allowed( + static_cast(payload.size()))) { + return Status::NotFound("file meta disk cache payload size is not allowed"); + } + + const std::string cache_key = get_key(format, file_meta_cache_key); + const auto hash = io::BlockFileCache::hash(cache_key); + std::string value; + size_t header_end = 0; + RETURN_IF_ERROR( + build_cache_value(format, modification_time, file_size, payload, &value, &header_end)); + io::BlockFileCache* cache = get_cache(hash); + if (cache == nullptr) { + return Status::NotFound("file meta disk cache is not available"); + } + if (!cache->get_async_open_success()) { + return Status::NotFound("file meta disk cache is still restoring"); + } + std::unique_lock entry_lock(get_entry_lock(hash)); + + std::string existing_payload; + Status existing_status = + read_cache_entry(cache, hash, format, modification_time, file_size, &existing_payload); + if (existing_status.ok() && existing_payload == payload) { + return Status::OK(); + } + cache->remove_if_cached(hash); + + io::ReadStatistics stats; + io::CacheContext context = build_meta_cache_context(); + context.stats = &stats; + auto holder = cache->get_or_set(hash, 0, value.size(), context); + + // Claim the complete entry before appending so a rejected block cannot leave partial data. + io::FileBlocks blocks_to_write; + for (const auto& block : holder.file_blocks) { + auto state = block->state(); + if (state == io::FileBlock::State::DOWNLOADING && !block->is_downloader()) { + state = block->wait(); + } + if (state == io::FileBlock::State::DOWNLOADED) { + continue; + } + if (state != io::FileBlock::State::EMPTY) { + cache->remove_if_cached(hash); + return Status::NotFound("file meta disk cache block is not writable"); + } + if (block->get_or_set_downloader() != io::FileBlock::get_caller_id()) { + cache->remove_if_cached(hash); + return Status::NotFound("file meta disk cache block has another downloader"); + } + blocks_to_write.emplace_back(block); + } + + auto write_block = [&](const io::FileBlockSPtr& block) -> Status { + const auto& range = block->range(); + DCHECK_LT(range.right, value.size()); + Status status = block->append(Slice(value.data() + range.left, range.size())); + if (!status.ok()) { + cache->remove_if_cached(hash); + return status; + } + status = block->finalize(); + if (!status.ok()) { + cache->remove_if_cached(hash); + return status; + } + return Status::OK(); + }; + + for (const auto& block : blocks_to_write) { + if (block->range().left >= header_end) { + RETURN_IF_ERROR(write_block(block)); + } + } + // Publish the header in reverse block order so the length prefix at offset zero is last. + for (auto it = blocks_to_write.rbegin(); it != blocks_to_write.rend(); ++it) { + if ((*it)->range().left < header_end) { + RETURN_IF_ERROR(write_block(*it)); + } + } + return Status::OK(); +} + +void FileMetaDiskCache::remove(FileMetaCacheFormat format, const std::string& file_meta_cache_key) { + const std::string cache_key = get_key(format, file_meta_cache_key); + const auto hash = io::BlockFileCache::hash(cache_key); + io::BlockFileCache* cache = get_cache(hash); + if (cache != nullptr && cache->get_async_open_success()) { + std::unique_lock entry_lock(get_entry_lock(hash)); + cache->remove_if_cached(hash); + } +} + +std::shared_mutex& FileMetaDiskCache::get_entry_lock(const io::UInt128Wrapper& hash) { + return _entry_locks[io::KeyHash()(hash) % ENTRY_LOCK_SHARDS]; +} + +io::BlockFileCache* FileMetaDiskCache::get_cache(const io::UInt128Wrapper& hash) const { + if (_cache != nullptr) { + return _cache; + } + io::FileCacheFactory* factory = io::FileCacheFactory::instance(); + if (factory == nullptr || factory->get_cache_instance_size() == 0) { + return nullptr; + } + return factory->get_by_path(hash); +} + +} // namespace doris diff --git a/be/src/io/fs/file_meta_disk_cache.h b/be/src/io/fs/file_meta_disk_cache.h new file mode 100644 index 00000000000000..ae22c071243e85 --- /dev/null +++ b/be/src/io/fs/file_meta_disk_cache.h @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "io/fs/file_meta_cache.h" + +namespace doris { + +namespace io { +class BlockFileCache; +struct UInt128Wrapper; +} // namespace io + +class FileMetaDiskCache final : public FileMetaPersistentCache { +public: + explicit FileMetaDiskCache(io::BlockFileCache* cache = nullptr) : _cache(cache) {} + + Status read(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string* payload) override; + + Status write(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string_view payload) override; + + void remove(FileMetaCacheFormat format, const std::string& key) override; + +private: + static constexpr size_t ENTRY_LOCK_SHARDS = 64; + + static std::string get_key(FileMetaCacheFormat format, std::string_view file_meta_cache_key); + + std::shared_mutex& get_entry_lock(const io::UInt128Wrapper& hash); + + io::BlockFileCache* get_cache(const io::UInt128Wrapper& hash) const; + + io::BlockFileCache* _cache = nullptr; + static std::array _entry_locks; +}; + +} // namespace doris diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index c010c1deeba272..a5121d29975c9b 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -62,6 +62,7 @@ #include "io/cache/block_file_cache_factory.h" #include "io/cache/fs_file_cache_storage.h" #include "io/fs/file_meta_cache.h" +#include "io/fs/file_meta_disk_cache.h" #include "io/fs/local_file_reader.h" #include "load/channel/load_channel_mgr.h" #include "load/channel/load_stream_mgr.h" @@ -506,7 +507,9 @@ void ExecEnv::init_file_cache_factory(std::vector& cache_paths "= true"; exit(-1); } - return; + if (!FileMetaCache::is_persistent_cache_configured()) { + return; + } } if (config::file_cache_each_block_size > config::s3_write_buffer_size || config::s3_write_buffer_size % config::file_cache_each_block_size != 0) { @@ -676,7 +679,10 @@ Status ExecEnv::init_mem_env() { << config::file_cache_max_file_reader_cache_size; config::file_cache_max_file_reader_cache_size = block_file_cache_fd_cache_size; - _file_meta_cache = new FileMetaCache(config::max_external_file_meta_cache_num); + _file_meta_cache = new FileMetaCache(config::max_external_file_meta_cache_num, + FileMetaCache::is_persistent_cache_configured() + ? std::make_unique() + : nullptr); _lookup_connection_cache = LookupConnectionCache::create_global_instance(config::lookup_connection_cache_capacity); diff --git a/be/test/format/file_reader/file_meta_cache_test.cpp b/be/test/format/file_reader/file_meta_cache_test.cpp index 4f8f803bd3ef28..b1676d2db68a9b 100644 --- a/be/test/format/file_reader/file_meta_cache_test.cpp +++ b/be/test/format/file_reader/file_meta_cache_test.cpp @@ -17,23 +17,39 @@ #include "io/fs/file_meta_cache.h" +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/logging.h" #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "io/cache/block_file_cache.h" +#include "io/cache/file_block.h" +#include "io/cache/file_cache_common.h" +#include "io/cache/fs_file_cache_storage.h" +#include "io/fs/file_meta_disk_cache.h" #include "io/fs/file_reader.h" +#include "runtime/exec_env.h" +#include "util/coding.h" +#include "util/defer_op.h" namespace doris { namespace { class MockFileReader : public io::FileReader { public: - MockFileReader(const std::string& file_name, size_t size) - : _file_name(file_name), _size(size), _closed(false) {} + MockFileReader(const std::string& file_name, size_t size) : _path(file_name), _size(size) {} ~MockFileReader() override = default; - const io::Path& path() const override { - static io::Path p(_file_name); - return p; - } + const io::Path& path() const override { return _path; } size_t size() const override { return _size; } @@ -54,10 +70,161 @@ class MockFileReader : public io::FileReader { } private: - std::string _file_name; + io::Path _path; size_t _size; - bool _closed; + bool _closed = false; +}; + +class TestPersistentFileMetaCache final : public FileMetaPersistentCache { +public: + Status read(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string* payload) override { + ++read_count; + if (fail_read) { + return Status::IOError("injected persistent cache read failure"); + } + if (!has_value || format != stored_format || key != stored_key || + modification_time != stored_modification_time || file_size != stored_file_size) { + return Status::NotFound("persistent cache entry not found"); + } + *payload = stored_payload; + return Status::OK(); + } + + Status write(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string_view payload) override { + ++write_count; + if (fail_write) { + return Status::IOError("injected persistent cache write failure"); + } + stored_format = format; + stored_key = key; + stored_modification_time = modification_time; + stored_file_size = file_size; + stored_payload = payload; + has_value = true; + return Status::OK(); + } + + void remove(FileMetaCacheFormat format, const std::string& key) override { + ++remove_count; + if (has_value && format == stored_format && key == stored_key) { + has_value = false; + } + } + + bool fail_read = false; + bool fail_write = false; + bool has_value = false; + int read_count = 0; + int write_count = 0; + int remove_count = 0; + FileMetaCacheFormat stored_format = FileMetaCacheFormat::PARQUET; + std::string stored_key; + int64_t stored_modification_time = 0; + int64_t stored_file_size = 0; + std::string stored_payload; +}; + +io::FileCacheSettings create_file_meta_disk_cache_test_settings() { + io::FileCacheSettings settings; + settings.capacity = 1024 * 1024; + settings.max_file_block_size = 8; + settings.index_queue_size = 1024 * 1024; + settings.index_queue_elements = 1024; + settings.max_query_cache_size = 1024 * 1024; + settings.storage = "memory"; + return settings; +} + +constexpr uint32_t FILE_META_DISK_CACHE_TEST_VERSION = 1; + +std::string file_meta_disk_cache_key(FileMetaCacheFormat format, const std::string& meta_key) { + std::string key = "file_meta_cache:v1:"; + switch (format) { + case FileMetaCacheFormat::PARQUET: + key.append("parquet"); + break; + case FileMetaCacheFormat::ORC: + key.append("orc"); + break; + case FileMetaCacheFormat::PARQUET_V2: + key.append("parquet_v2"); + break; + case FileMetaCacheFormat::ORC_V2: + key.append("orc_v2"); + break; + } + key.push_back(':'); + key.append(meta_key); + return key; +} + +struct FileMetaDiskCacheTestValue { + std::string value; + size_t header_end = 0; }; + +FileMetaDiskCacheTestValue build_file_meta_disk_cache_value(FileMetaCacheFormat format, + int64_t modification_time, + int64_t file_size, + std::string_view payload) { + io::cache::FileMetaCacheEntryHeaderPB header; + header.set_version(FILE_META_DISK_CACHE_TEST_VERSION); + header.set_format(static_cast(format)); + header.set_file_size(file_size); + header.set_modification_time(modification_time); + header.set_payload_size(payload.size()); + header.set_checksum(crc32c::Crc32c(payload.data(), payload.size())); + + std::string serialized_header; + DORIS_CHECK(header.SerializeToString(&serialized_header)); + FileMetaDiskCacheTestValue encoded; + put_fixed32_le(&encoded.value, static_cast(serialized_header.size())); + encoded.value.append(serialized_header); + encoded.header_end = encoded.value.size(); + encoded.value.append(payload); + return encoded; +} + +io::CacheContext create_file_meta_disk_cache_context(io::ReadStatistics* stats) { + io::CacheContext context; + context.cache_type = io::FileCacheType::INDEX; + context.query_id = TUniqueId(); + context.expiration_time = 0; + context.is_cold_data = false; + context.is_warmup = false; + context.stats = stats; + return context; +} + +void write_raw_cache_value(io::BlockFileCache* cache, const io::UInt128Wrapper& hash, + std::string_view value) { + io::ReadStatistics stats; + io::CacheContext context = create_file_meta_disk_cache_context(&stats); + auto holder = cache->get_or_set(hash, 0, value.size(), context); + for (const auto& block : holder.file_blocks) { + ASSERT_EQ(block->get_or_set_downloader(), io::FileBlock::get_caller_id()); + ASSERT_TRUE(block->append(Slice(value.data() + block->range().left, block->range().size())) + .ok()); + ASSERT_TRUE(block->finalize().ok()); + } +} + +size_t downloaded_block_count(io::BlockFileCache* cache, const io::UInt128Wrapper& hash) { + auto blocks = cache->get_blocks_by_key(hash); + for (const auto& [_, block] : blocks) { + block->_owned_by_cached_reader = false; + } + return blocks.size(); +} + +bool wait_until_file_cache_ready(io::BlockFileCache* cache) { + for (int i = 0; i < 1000 && !cache->get_async_open_success(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + return cache->get_async_open_success(); +} } // anonymous namespace TEST(FileMetaCacheTest, KeyGenerationFromParams) { @@ -73,7 +240,11 @@ TEST(FileMetaCacheTest, KeyGenerationFromParams) { std::string key3 = FileMetaCache::get_key(file_name, mtime + 1, file_size); EXPECT_NE(key1, key3); - // mtime == 0, use file_size + // Same mtime with different size should produce different key + std::string key_with_different_size = FileMetaCache::get_key(file_name, mtime, file_size + 1); + EXPECT_NE(key1, key_with_different_size); + + // mtime == 0, still includes file_size std::string key4 = FileMetaCache::get_key(file_name, 0, file_size); std::string key5 = FileMetaCache::get_key(file_name, 0, file_size); EXPECT_EQ(key4, key5); @@ -108,7 +279,70 @@ TEST(FileMetaCacheTest, KeyGenerationFromFileReader) { std::string key2 = FileMetaCache::get_key(reader2, desc2); std::string expected_key2 = FileMetaCache::get_key(file_name, 0, 300); EXPECT_EQ(key2, expected_key2); + + desc1.fs_name = "hdfs://nameservice1"; + std::string file_identity = desc1.fs_name; + file_identity.push_back('\0'); + file_identity.append(file_name); + EXPECT_EQ(FileMetaCache::get_key(reader1, desc1), + FileMetaCache::get_key(file_identity, mtime, file_size)); + + io::FileDescription desc_with_other_fs = desc1; + desc_with_other_fs.fs_name = "hdfs://nameservice2"; + EXPECT_NE(FileMetaCache::get_key(reader1, desc1), + FileMetaCache::get_key(reader1, desc_with_other_fs)); } + +TEST(FileMetaCacheTest, HdfsFileCacheIdentityUsesEffectiveFileSystemName) { + io::FileSystemProperties properties; + properties.system_type = TFileType::FILE_HDFS; + properties.hdfs_params.__set_fs_name("hdfs://nameservice1"); + + io::FileDescription default_fs_file; + default_fs_file.path = "/warehouse/default/table/data.orc"; + EXPECT_EQ(FileFactory::get_file_system_identity(properties, default_fs_file), + "hdfs://nameservice1"); + + io::FileDescription uri_file; + uri_file.path = "hdfs://nameservice2/warehouse/default/table/data.orc"; + EXPECT_EQ(FileFactory::get_file_system_identity(properties, uri_file), "hdfs://nameservice2"); +} + +TEST(FileMetaCacheTest, S3FileCacheIdentityIncludesEndpoint) { + io::FileDescription file; + file.path = "s3://bucket/table/data.parquet"; + + io::FileSystemProperties first_properties; + first_properties.system_type = TFileType::FILE_S3; + first_properties.properties["AWS_ENDPOINT"] = "http://minio-a:9000"; + first_properties.properties["AWS_REGION"] = "us-east-1"; + + io::FileSystemProperties second_properties = first_properties; + second_properties.properties["AWS_ENDPOINT"] = "http://minio-b:9000"; + + EXPECT_NE(FileFactory::get_file_system_identity(first_properties, file), + FileFactory::get_file_system_identity(second_properties, file)); +} + +TEST(FileMetaCacheTest, ExternalFileMetaDiskCacheSwitchIsStartupOnly) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + + config::enable_external_file_meta_disk_cache = false; + Status status = config::set_config("enable_external_file_meta_disk_cache", "true"); + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(status.is() || status.is()) + << status; + EXPECT_FALSE(config::enable_external_file_meta_disk_cache); +} + +TEST(FileMetaCacheTest, ExternalFileMetaDiskCacheIsEnabledByDefault) { + EXPECT_TRUE(config::enable_external_file_meta_disk_cache); +} + TEST(FileMetaCacheTest, KeyContentVerification) { std::string file_name = "/path/to/file"; int64_t mtime = 0x0102030405060708; @@ -116,19 +350,27 @@ TEST(FileMetaCacheTest, KeyContentVerification) { std::string key_with_mtime = FileMetaCache::get_key(file_name, mtime, file_size); - ASSERT_EQ(key_with_mtime.size(), file_name.size() + sizeof(int64_t)); + ASSERT_EQ(key_with_mtime.size(), file_name.size() + 2 * sizeof(int64_t)); EXPECT_EQ(memcmp(key_with_mtime.data(), file_name.data(), file_name.size()), 0); int64_t extracted_mtime = 0; memcpy(&extracted_mtime, key_with_mtime.data() + file_name.size(), sizeof(int64_t)); EXPECT_EQ(extracted_mtime, mtime); + int64_t extracted_file_size = 0; + memcpy(&extracted_file_size, key_with_mtime.data() + file_name.size() + sizeof(int64_t), + sizeof(int64_t)); + EXPECT_EQ(extracted_file_size, file_size); std::string key_with_filesize = FileMetaCache::get_key(file_name, 0, file_size); - ASSERT_EQ(key_with_filesize.size(), file_name.size() + sizeof(int64_t)); + ASSERT_EQ(key_with_filesize.size(), file_name.size() + 2 * sizeof(int64_t)); EXPECT_EQ(memcmp(key_with_filesize.data(), file_name.data(), file_name.size()), 0); + int64_t extracted_zero_mtime = -1; + memcpy(&extracted_zero_mtime, key_with_filesize.data() + file_name.size(), sizeof(int64_t)); + EXPECT_EQ(extracted_zero_mtime, 0); int64_t extracted_filesize = 0; - memcpy(&extracted_filesize, key_with_filesize.data() + file_name.size(), sizeof(int64_t)); + memcpy(&extracted_filesize, key_with_filesize.data() + file_name.size() + sizeof(int64_t), + sizeof(int64_t)); EXPECT_EQ(extracted_filesize, file_size); } @@ -155,4 +397,886 @@ TEST(FileMetaCacheTest, InsertAndLookupWithIntValue) { EXPECT_EQ(*cached_val2, 12345); } -} // namespace doris \ No newline at end of file +TEST(FileMetaCacheTest, ContextLookupReportsMemoryHit) { + FileMetaCache cache(1024 * 1024); + const std::string meta_key = FileMetaCache::get_key("s3://bucket/memory.parquet", 123, 456); + const FileMetaCacheContext context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 123, + .file_size = 456}; + + auto value = std::make_unique("serialized footer payload"); + ObjLRUCache::CacheHandle insert_handle; + ASSERT_TRUE(cache.insert(FileMetaCache::get_memory_cache_key(context), value, &insert_handle)); + ASSERT_TRUE(insert_handle.valid()); + + int64_t hit_cache = 0; + int64_t hit_memory_cache = 0; + int64_t hit_disk_cache = 0; + int64_t miss_disk_cache = 0; + FileMetaCacheProfile profile {.hit_cache = &hit_cache, + .hit_memory_cache = &hit_memory_cache, + .hit_disk_cache = &hit_disk_cache, + .miss_disk_cache = &miss_disk_cache}; + + ObjLRUCache::CacheHandle lookup_handle; + std::string serialized_meta; + const FileMetaCacheLookupResult result = + cache.lookup(context, &lookup_handle, &serialized_meta, &profile); + + EXPECT_EQ(result.state, FileMetaCacheLookupState::MEMORY_HIT); + EXPECT_TRUE(lookup_handle.valid()); + EXPECT_TRUE(serialized_meta.empty()); + EXPECT_EQ(hit_cache, 1); + EXPECT_EQ(hit_memory_cache, 1); + EXPECT_EQ(hit_disk_cache, 0); + EXPECT_EQ(miss_disk_cache, 0); +} + +TEST(FileMetaCacheTest, ContextLookupClearsReusedMemoryHandle) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 1024; + + auto persistent_cache = std::make_unique(); + auto* persistent_cache_ptr = persistent_cache.get(); + FileMetaCache cache(1024, std::move(persistent_cache)); + + const std::string memory_key = + FileMetaCache::get_key("s3://bucket/memory-handle.parquet", 123, 456); + const FileMetaCacheContext memory_context {.format = FileMetaCacheFormat::PARQUET, + .key = memory_key, + .modification_time = 123, + .file_size = 456}; + auto memory_value = std::make_unique("parsed memory footer"); + ObjLRUCache::CacheHandle insert_handle; + ASSERT_TRUE( + cache.insert(memory_context, memory_value, &insert_handle, "serialized memory footer") + .memory_inserted); + + ObjLRUCache::CacheHandle reused_handle; + std::string serialized_meta; + ASSERT_EQ(cache.lookup(memory_context, &reused_handle, &serialized_meta).state, + FileMetaCacheLookupState::MEMORY_HIT); + ASSERT_TRUE(reused_handle.valid()); + + const std::string persistent_key = + FileMetaCache::get_key("s3://bucket/persistent-handle.parquet", 123, 456); + const FileMetaCacheContext persistent_context {.format = FileMetaCacheFormat::PARQUET, + .key = persistent_key, + .modification_time = 123, + .file_size = 456}; + persistent_cache_ptr->has_value = true; + persistent_cache_ptr->stored_format = persistent_context.format; + persistent_cache_ptr->stored_key = persistent_context.key; + persistent_cache_ptr->stored_modification_time = persistent_context.modification_time; + persistent_cache_ptr->stored_file_size = persistent_context.file_size; + persistent_cache_ptr->stored_payload = "serialized persistent footer"; + + EXPECT_EQ(cache.lookup(persistent_context, &reused_handle, &serialized_meta).state, + FileMetaCacheLookupState::PERSISTED_HIT); + EXPECT_FALSE(reused_handle.valid()); + + ASSERT_EQ(cache.lookup(memory_context, &reused_handle, &serialized_meta).state, + FileMetaCacheLookupState::MEMORY_HIT); + ASSERT_TRUE(reused_handle.valid()); + persistent_cache_ptr->has_value = false; + EXPECT_EQ(cache.lookup(persistent_context, &reused_handle, &serialized_meta).state, + FileMetaCacheLookupState::MISS); + EXPECT_FALSE(reused_handle.valid()); +} + +TEST(FileMetaCacheTest, ContextMemoryCacheKeyIncludesFormat) { + FileMetaCache cache(1024 * 1024); + const std::string meta_key = FileMetaCache::get_key("s3://bucket/same.parquet", 123, 456); + const FileMetaCacheContext parquet_context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 123, + .file_size = 456, + .enable_memory_cache = true}; + const FileMetaCacheContext parquet_v2_context {.format = FileMetaCacheFormat::PARQUET_V2, + .key = meta_key, + .modification_time = 123, + .file_size = 456, + .enable_memory_cache = true}; + + auto value = std::make_unique(7); + ObjLRUCache::CacheHandle insert_handle; + const FileMetaCacheInsertResult insert_result = + cache.insert(parquet_context, value, &insert_handle, "payload"); + ASSERT_TRUE(insert_result.memory_inserted); + + ObjLRUCache::CacheHandle lookup_handle; + std::string serialized_meta; + EXPECT_EQ(cache.lookup(parquet_v2_context, &lookup_handle, &serialized_meta).state, + FileMetaCacheLookupState::MISS); + EXPECT_EQ(cache.lookup(parquet_context, &lookup_handle, &serialized_meta).state, + FileMetaCacheLookupState::MEMORY_HIT); +} + +TEST(FileMetaCacheTest, PersistentCacheConfigurationIsIndependentFromRuntimeSizeGate) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 0; + EXPECT_TRUE(FileMetaCache::is_persistent_cache_configured()); + EXPECT_FALSE(FileMetaCache::is_persistent_cache_enabled()); + + config::external_file_meta_disk_cache_max_entry_bytes = 1024; + EXPECT_TRUE(FileMetaCache::is_persistent_cache_configured()); + EXPECT_TRUE(FileMetaCache::is_persistent_cache_enabled()); + + config::enable_external_file_meta_disk_cache = false; + EXPECT_FALSE(FileMetaCache::is_persistent_cache_configured()); + EXPECT_FALSE(FileMetaCache::is_persistent_cache_enabled()); +} + +TEST(FileMetaCacheDiskTest, OrcFormatsUseIndependentKeys) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = true; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + io::BlockFileCache block_cache("file_meta_disk_cache_orc_format_isolation_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/same.orc", 123, 456); + FileMetaDiskCache disk_cache(&block_cache); + ASSERT_TRUE(disk_cache + .write(FileMetaCacheFormat::ORC, meta_key, 123, 456, + "legacy orc serialized tail") + .ok()); + ASSERT_TRUE(disk_cache + .write(FileMetaCacheFormat::ORC_V2, meta_key, 123, 456, + "format v2 orc serialized tail") + .ok()); + + std::string legacy_payload; + ASSERT_TRUE( + disk_cache.read(FileMetaCacheFormat::ORC, meta_key, 123, 456, &legacy_payload).ok()); + EXPECT_EQ(legacy_payload, "legacy orc serialized tail"); + + std::string v2_payload; + ASSERT_TRUE(disk_cache.read(FileMetaCacheFormat::ORC_V2, meta_key, 123, 456, &v2_payload).ok()); + EXPECT_EQ(v2_payload, "format v2 orc serialized tail"); +} + +TEST(FileMetaCacheTest, ContextInsertCanSkipMemoryCacheWithoutPersistentStore) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = false; + + FileMetaCache cache(1024 * 1024); + const std::string meta_key = + FileMetaCache::get_key("s3://bucket/skip-memory.parquet", 123, 456); + const FileMetaCacheContext context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 123, + .file_size = 456, + .enable_memory_cache = false}; + auto value = std::make_unique("serialized footer payload"); + ObjLRUCache::CacheHandle handle; + + const FileMetaCacheInsertResult result = + cache.insert(context, value, &handle, std::string_view(*value)); + + EXPECT_FALSE(result.persisted_inserted); + EXPECT_FALSE(result.memory_inserted); + EXPECT_FALSE(handle.valid()); + EXPECT_NE(value, nullptr); + + ObjLRUCache::CacheHandle lookup_handle; + EXPECT_FALSE(cache.lookup(meta_key, &lookup_handle)); +} + +TEST(FileMetaCacheTest, PersistentCacheRoundTripUsesFileMetaBoundary) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 1024; + + auto persistent_cache = std::make_unique(); + auto* persistent_cache_ptr = persistent_cache.get(); + FileMetaCache cache(0, std::move(persistent_cache)); + const std::string meta_key = FileMetaCache::get_key("s3://bucket/persistent.parquet", 123, 456); + const FileMetaCacheContext context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 123, + .file_size = 456, + .enable_memory_cache = false}; + auto parsed_value = std::make_unique("parsed footer"); + ObjLRUCache::CacheHandle handle; + int64_t hit_cache = 0; + int64_t hit_disk_cache = 0; + int64_t miss_disk_cache = 0; + int64_t write_disk_cache = 0; + FileMetaCacheProfile profile {.hit_cache = &hit_cache, + .hit_disk_cache = &hit_disk_cache, + .miss_disk_cache = &miss_disk_cache, + .write_disk_cache = &write_disk_cache}; + + const FileMetaCacheInsertResult insert_result = + cache.insert(context, parsed_value, &handle, "serialized footer", &profile); + ASSERT_TRUE(insert_result.persisted_inserted); + EXPECT_FALSE(insert_result.memory_inserted); + EXPECT_NE(parsed_value, nullptr); + EXPECT_EQ(persistent_cache_ptr->write_count, 1); + EXPECT_EQ(write_disk_cache, 1); + + std::string serialized_meta; + const FileMetaCacheLookupResult lookup_result = + cache.lookup(context, &handle, &serialized_meta, &profile); + EXPECT_EQ(lookup_result.state, FileMetaCacheLookupState::PERSISTED_HIT); + EXPECT_EQ(serialized_meta, "serialized footer"); + EXPECT_EQ(persistent_cache_ptr->read_count, 1); + EXPECT_EQ(hit_cache, 1); + EXPECT_EQ(hit_disk_cache, 1); + EXPECT_EQ(miss_disk_cache, 0); + + cache.invalidate_persistent_cache(context); + EXPECT_EQ(persistent_cache_ptr->remove_count, 1); + EXPECT_FALSE(persistent_cache_ptr->has_value); + EXPECT_EQ(cache.lookup(context, &handle, &serialized_meta, &profile).state, + FileMetaCacheLookupState::MISS); + EXPECT_EQ(miss_disk_cache, 1); +} + +TEST(FileMetaCacheTest, PersistentCacheFailureFallsBackToMiss) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 1024; + + auto persistent_cache = std::make_unique(); + auto* persistent_cache_ptr = persistent_cache.get(); + persistent_cache_ptr->fail_read = true; + persistent_cache_ptr->fail_write = true; + FileMetaCache cache(0, std::move(persistent_cache)); + const std::string meta_key = FileMetaCache::get_key("s3://bucket/failure.orc", 123, 456); + const FileMetaCacheContext context {.format = FileMetaCacheFormat::ORC, + .key = meta_key, + .modification_time = 123, + .file_size = 456, + .enable_memory_cache = false}; + int64_t miss_disk_cache = 0; + int64_t write_disk_cache = 0; + FileMetaCacheProfile profile {.miss_disk_cache = &miss_disk_cache, + .write_disk_cache = &write_disk_cache}; + ObjLRUCache::CacheHandle handle; + std::string serialized_meta = "stale output"; + + EXPECT_EQ(cache.lookup(context, &handle, &serialized_meta, &profile).state, + FileMetaCacheLookupState::MISS); + EXPECT_TRUE(serialized_meta.empty()); + EXPECT_EQ(persistent_cache_ptr->read_count, 1); + EXPECT_EQ(miss_disk_cache, 1); + + auto parsed_value = std::make_unique("parsed footer"); + const FileMetaCacheInsertResult insert_result = + cache.insert(context, parsed_value, &handle, "serialized footer", &profile); + EXPECT_FALSE(insert_result.persisted_inserted); + EXPECT_FALSE(insert_result.memory_inserted); + EXPECT_NE(parsed_value, nullptr); + EXPECT_EQ(persistent_cache_ptr->write_count, 1); + EXPECT_EQ(write_disk_cache, 0); +} + +TEST(FileMetaCacheTest, OversizedPersistentReadFallsBackToMiss) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 4; + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/oversized.parquet", 123, 456); + const FileMetaCacheContext context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 123, + .file_size = 456, + .enable_memory_cache = false}; + auto persistent_cache = std::make_unique(); + auto* persistent_cache_ptr = persistent_cache.get(); + persistent_cache_ptr->has_value = true; + persistent_cache_ptr->stored_format = context.format; + persistent_cache_ptr->stored_key = context.key; + persistent_cache_ptr->stored_modification_time = context.modification_time; + persistent_cache_ptr->stored_file_size = context.file_size; + persistent_cache_ptr->stored_payload = "oversized payload"; + FileMetaCache cache(0, std::move(persistent_cache)); + + ObjLRUCache::CacheHandle handle; + std::string serialized_meta = "stale output"; + EXPECT_EQ(cache.lookup(context, &handle, &serialized_meta).state, + FileMetaCacheLookupState::MISS); + EXPECT_FALSE(handle.valid()); + EXPECT_TRUE(serialized_meta.empty()); + EXPECT_EQ(persistent_cache_ptr->read_count, 1); + EXPECT_EQ(persistent_cache_ptr->remove_count, 1); + EXPECT_FALSE(persistent_cache_ptr->has_value); +} + +TEST(FileMetaCacheDiskTest, PersistentCacheRequiresStableModificationTime) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = true; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + io::BlockFileCache block_cache("file_meta_disk_cache_unstable_identity_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/mtime-zero.parquet", 0, 456); + auto disk_cache = std::make_unique(&block_cache); + FileMetaCache cache(0, std::move(disk_cache)); + const FileMetaCacheContext context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 0, + .file_size = 456, + .enable_memory_cache = false}; + + auto value = std::make_unique("serialized footer payload"); + ObjLRUCache::CacheHandle handle; + const FileMetaCacheInsertResult insert_result = + cache.insert(context, value, &handle, std::string_view(*value)); + EXPECT_FALSE(insert_result.persisted_inserted); + EXPECT_FALSE(insert_result.memory_inserted); + + int64_t miss_disk_cache = 0; + FileMetaCacheProfile profile {.miss_disk_cache = &miss_disk_cache}; + std::string serialized_meta; + ObjLRUCache::CacheHandle lookup_handle; + EXPECT_EQ(cache.lookup(context, &lookup_handle, &serialized_meta, &profile).state, + FileMetaCacheLookupState::MISS); + EXPECT_EQ(miss_disk_cache, 0); +} + +TEST(FileMetaCacheDiskTest, PersistentCacheStoresPayloadBehindFileMetaInterface) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = true; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + io::BlockFileCache block_cache("file_meta_disk_cache_interface_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + auto disk_cache = std::make_unique(&block_cache); + FileMetaCache cache(1024 * 1024, std::move(disk_cache)); + const std::string meta_key = FileMetaCache::get_key("s3://bucket/interface.parquet", 123, 456); + const FileMetaCacheContext context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 123, + .file_size = 456, + .enable_memory_cache = false}; + const std::string payload = "serialized footer payload"; + auto value = std::make_unique(payload); + ObjLRUCache::CacheHandle handle; + + const FileMetaCacheInsertResult insert_result = cache.insert(context, value, &handle, payload); + ASSERT_TRUE(insert_result.persisted_inserted); + ASSERT_FALSE(insert_result.memory_inserted); + + ObjLRUCache::CacheHandle lookup_handle; + std::string output; + const FileMetaCacheLookupResult lookup_result = cache.lookup(context, &lookup_handle, &output); + EXPECT_EQ(lookup_result.state, FileMetaCacheLookupState::PERSISTED_HIT); + EXPECT_EQ(output, payload); + EXPECT_FALSE(lookup_handle.valid()); + + const FileMetaCacheContext stale_context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 124, + .file_size = 456, + .enable_memory_cache = false}; + output.clear(); + const FileMetaCacheLookupResult stale_lookup_result = + cache.lookup(stale_context, &lookup_handle, &output); + EXPECT_EQ(stale_lookup_result.state, FileMetaCacheLookupState::MISS); + EXPECT_TRUE(output.empty()); +} + +TEST(FileMetaCacheDiskTest, CapacityFailureDoesNotLeavePartiallyDownloadedBlocks) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 1024; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + settings.capacity = 64; + settings.index_queue_size = 64; + settings.index_queue_elements = 8; + settings.max_query_cache_size = 0; + io::BlockFileCache block_cache("file_meta_disk_cache_capacity_failure_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/too-large.parquet", 123, 456); + const auto hash = io::BlockFileCache::hash( + file_meta_disk_cache_key(FileMetaCacheFormat::PARQUET, meta_key)); + FileMetaDiskCache disk_cache(&block_cache); + + const Status status = disk_cache.write(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, + std::string(80, 'p')); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(downloaded_block_count(&block_cache, hash), 0); + EXPECT_EQ(block_cache.get_used_cache_size(io::FileCacheType::INDEX), 0); + EXPECT_EQ(block_cache.get_file_blocks_num(io::FileCacheType::INDEX), 0); +} + +TEST(FileMetaCacheDiskTest, ConcurrentWrappersSerializeReplacement) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 1024; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + io::BlockFileCache block_cache("file_meta_disk_cache_concurrent_wrappers_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/concurrent.parquet", 123, 456); + const auto hash = io::BlockFileCache::hash( + file_meta_disk_cache_key(FileMetaCacheFormat::PARQUET, meta_key)); + const std::string first_payload = "first serialized footer payload"; + const std::string second_payload = "second serialized footer payload with a different size"; + FileMetaDiskCache first_cache(&block_cache); + FileMetaDiskCache second_cache(&block_cache); + std::barrier start(3); + Status first_status; + Status second_status; + std::thread first_writer([&] { + start.arrive_and_wait(); + first_status = + first_cache.write(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, first_payload); + }); + std::thread second_writer([&] { + start.arrive_and_wait(); + second_status = second_cache.write(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, + second_payload); + }); + start.arrive_and_wait(); + first_writer.join(); + second_writer.join(); + + ASSERT_TRUE(first_status.ok()) << first_status; + ASSERT_TRUE(second_status.ok()) << second_status; + std::string output; + ASSERT_TRUE(first_cache.read(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, &output).ok()); + EXPECT_THAT(output, testing::AnyOf(first_payload, second_payload)); + const size_t expected_size = + build_file_meta_disk_cache_value(FileMetaCacheFormat::PARQUET, 123, 456, output) + .value.size(); + EXPECT_EQ(block_cache.get_used_cache_size(io::FileCacheType::INDEX), expected_size); + EXPECT_EQ(downloaded_block_count(&block_cache, hash), + (expected_size + settings.max_file_block_size - 1) / settings.max_file_block_size); +} + +// GTest assertion macros inflate complexity across the three sequential restart phases. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST(FileMetaCacheDiskTest, DiskEntrySurvivesBlockFileCacheReinitialization) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + const int64_t old_file_cache_background_ttl_gc_interval_ms = + config::file_cache_background_ttl_gc_interval_ms; + const int64_t old_file_cache_background_ttl_info_update_interval_ms = + config::file_cache_background_ttl_info_update_interval_ms; + Defer restore_config {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + config::file_cache_background_ttl_gc_interval_ms = + old_file_cache_background_ttl_gc_interval_ms; + config::file_cache_background_ttl_info_update_interval_ms = + old_file_cache_background_ttl_info_update_interval_ms; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 1024; + config::file_cache_background_ttl_gc_interval_ms = 20; + config::file_cache_background_ttl_info_update_interval_ms = 20; + + const std::filesystem::path cache_path = + std::filesystem::temp_directory_path() / + ("doris_file_meta_disk_cache_restart_" + std::to_string(getpid())); + std::error_code error; + std::filesystem::remove_all(cache_path, error); + ASSERT_TRUE(std::filesystem::create_directories(cache_path)); + Defer remove_cache_path {[&] { std::filesystem::remove_all(cache_path, error); }}; + + auto* exec_env = ExecEnv::GetInstance(); + const bool owns_fd_cache = exec_env->file_cache_open_fd_cache() == nullptr; + if (owns_fd_cache) { + exec_env->set_file_cache_open_fd_cache(std::make_unique()); + } + Defer release_fd_cache {[&] { + if (owns_fd_cache) { + exec_env->set_file_cache_open_fd_cache(nullptr); + } + }}; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + settings.storage = "disk"; + const std::string meta_key = FileMetaCache::get_key("s3://bucket/restart.parquet", 123, 456); + const std::string payload = "serialized footer persisted across cache initialization"; + const auto hash = io::BlockFileCache::hash( + file_meta_disk_cache_key(FileMetaCacheFormat::PARQUET, meta_key)); + + { + io::BlockFileCache block_cache(cache_path.string(), settings); + ASSERT_TRUE(block_cache.initialize().ok()); + ASSERT_TRUE(wait_until_file_cache_ready(&block_cache)); + FileMetaDiskCache disk_cache(&block_cache); + ASSERT_TRUE( + disk_cache.write(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, payload).ok()); + } + + { + io::BlockFileCache block_cache(cache_path.string(), settings); + FileMetaDiskCache disk_cache(&block_cache); + std::string output; + EXPECT_FALSE( + disk_cache.read(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, &output).ok()); + EXPECT_TRUE(output.empty()); + ASSERT_TRUE(block_cache.initialize().ok()); + ASSERT_TRUE(wait_until_file_cache_ready(&block_cache)); + ASSERT_TRUE( + disk_cache.read(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, &output).ok()); + EXPECT_EQ(output, payload); + disk_cache.remove(FileMetaCacheFormat::PARQUET, meta_key); + EXPECT_EQ(downloaded_block_count(&block_cache, hash), 0); + EXPECT_EQ(block_cache.get_used_cache_size(io::FileCacheType::INDEX), 0); + EXPECT_EQ(block_cache.get_file_blocks_num(io::FileCacheType::INDEX), 0); + } + + { + io::BlockFileCache block_cache(cache_path.string(), settings); + ASSERT_TRUE(block_cache.initialize().ok()); + ASSERT_TRUE(wait_until_file_cache_ready(&block_cache)); + FileMetaDiskCache disk_cache(&block_cache); + std::string output; + EXPECT_FALSE( + disk_cache.read(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, &output).ok()); + EXPECT_TRUE(output.empty()); + EXPECT_EQ(downloaded_block_count(&block_cache, hash), 0); + } +} + +TEST(FileMetaCacheDiskTest, InvalidProtobufHeaderReleasesCompleteEntry) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 1024; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + io::BlockFileCache block_cache("file_meta_disk_cache_invalid_protobuf_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/invalid.parquet", 123, 456); + const auto hash = io::BlockFileCache::hash( + file_meta_disk_cache_key(FileMetaCacheFormat::PARQUET, meta_key)); + std::string invalid_value; + put_fixed32_le(&invalid_value, 1); + invalid_value.push_back('\0'); + invalid_value.append("invalid protobuf payload"); + write_raw_cache_value(&block_cache, hash, invalid_value); + ASSERT_GT(downloaded_block_count(&block_cache, hash), 0); + + FileMetaDiskCache disk_cache(&block_cache); + std::string output; + EXPECT_FALSE(disk_cache.read(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, &output).ok()); + EXPECT_TRUE(output.empty()); + EXPECT_EQ(downloaded_block_count(&block_cache, hash), 0); + EXPECT_EQ(block_cache.get_used_cache_size(io::FileCacheType::INDEX), 0); + EXPECT_EQ(block_cache.get_file_blocks_num(io::FileCacheType::INDEX), 0); + + const std::string replacement_payload = "replacement payload"; + ASSERT_TRUE( + disk_cache.write(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, replacement_payload) + .ok()); + ASSERT_TRUE(disk_cache.read(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, &output).ok()); + EXPECT_EQ(output, replacement_payload); +} + +TEST(FileMetaCacheDiskTest, CorruptedPayloadReleasesCompleteEntry) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 1024; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + io::BlockFileCache block_cache("file_meta_disk_cache_corrupted_payload_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/corrupted.parquet", 123, 456); + const auto hash = io::BlockFileCache::hash( + file_meta_disk_cache_key(FileMetaCacheFormat::PARQUET, meta_key)); + FileMetaDiskCacheTestValue encoded = build_file_meta_disk_cache_value( + FileMetaCacheFormat::PARQUET, 123, 456, "serialized footer payload"); + encoded.value.back() ^= 1; + write_raw_cache_value(&block_cache, hash, encoded.value); + + FileMetaDiskCache disk_cache(&block_cache); + std::string output; + EXPECT_FALSE(disk_cache.read(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, &output).ok()); + EXPECT_TRUE(output.empty()); + EXPECT_EQ(downloaded_block_count(&block_cache, hash), 0); + EXPECT_EQ(block_cache.get_used_cache_size(io::FileCacheType::INDEX), 0); + EXPECT_EQ(block_cache.get_file_blocks_num(io::FileCacheType::INDEX), 0); +} + +TEST(FileMetaCacheDiskTest, IncompleteMultiBlockEntryIsFullyRemoved) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = true; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + io::BlockFileCache block_cache("file_meta_disk_cache_incomplete_entry_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/incomplete.parquet", 123, 456); + const std::string payload = "serialized footer payload spanning several blocks"; + const FileMetaDiskCacheTestValue encoded = + build_file_meta_disk_cache_value(FileMetaCacheFormat::PARQUET, 123, 456, payload); + const auto hash = io::BlockFileCache::hash( + file_meta_disk_cache_key(FileMetaCacheFormat::PARQUET, meta_key)); + + { + io::ReadStatistics stats; + io::CacheContext context = create_file_meta_disk_cache_context(&stats); + auto holder = block_cache.get_or_set(hash, 0, encoded.value.size(), context); + bool covered_header = false; + for (const auto& block : holder.file_blocks) { + if (block->range().left >= encoded.header_end) { + continue; + } + ASSERT_EQ(block->get_or_set_downloader(), io::FileBlock::get_caller_id()); + ASSERT_TRUE(block->append(Slice(encoded.value.data() + block->range().left, + block->range().size())) + .ok()); + ASSERT_TRUE(block->finalize().ok()); + covered_header = block->range().right + 1 >= encoded.header_end; + } + ASSERT_TRUE(covered_header); + } + + FileMetaDiskCache disk_cache(&block_cache); + std::string output; + const Status status = + disk_cache.read(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, &output); + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(output.empty()); + EXPECT_EQ(downloaded_block_count(&block_cache, hash), 0); + EXPECT_EQ(block_cache.get_used_cache_size(io::FileCacheType::INDEX), 0); + EXPECT_EQ(block_cache.get_file_blocks_num(io::FileCacheType::INDEX), 0); + + ASSERT_TRUE(disk_cache.write(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, payload).ok()); + ASSERT_TRUE(disk_cache.read(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, &output).ok()); + EXPECT_EQ(output, payload); +} + +TEST(FileMetaCacheDiskTest, PartialLruEvictionRemovesCompleteEntry) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 1024; + + const std::string payload = "serialized footer payload spanning several blocks"; + const FileMetaDiskCacheTestValue encoded = + build_file_meta_disk_cache_value(FileMetaCacheFormat::PARQUET, 123, 456, payload); + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + settings.capacity = encoded.value.size() + settings.max_file_block_size; + settings.index_queue_size = settings.capacity; + settings.max_query_cache_size = 0; + io::BlockFileCache block_cache("file_meta_disk_cache_partial_eviction_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/partial.parquet", 123, 456); + const auto hash = io::BlockFileCache::hash( + file_meta_disk_cache_key(FileMetaCacheFormat::PARQUET, meta_key)); + FileMetaDiskCache disk_cache(&block_cache); + ASSERT_TRUE(disk_cache.write(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, payload).ok()); + const size_t complete_block_count = downloaded_block_count(&block_cache, hash); + ASSERT_GT(complete_block_count, 1); + ASSERT_EQ(block_cache.get_used_cache_size(io::FileCacheType::INDEX), encoded.value.size()); + + const auto filler_hash = + io::BlockFileCache::hash("file_meta_disk_cache_partial_eviction_filler"); + const std::string filler(settings.max_file_block_size * 2, 'f'); + write_raw_cache_value(&block_cache, filler_hash, filler); + const size_t remaining_block_count = downloaded_block_count(&block_cache, hash); + ASSERT_GT(remaining_block_count, 0); + ASSERT_LT(remaining_block_count, complete_block_count); + + std::string output; + EXPECT_FALSE(disk_cache.read(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, &output).ok()); + EXPECT_TRUE(output.empty()); + EXPECT_EQ(downloaded_block_count(&block_cache, hash), 0); + EXPECT_EQ(block_cache.get_used_cache_size(io::FileCacheType::INDEX), filler.size()); + EXPECT_EQ(block_cache.get_file_blocks_num(io::FileCacheType::INDEX), 2); +} + +TEST(FileMetaCacheDiskTest, PersistentHitRefreshesLruThroughExistingReadPath) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const bool old_enable_file_cache_async_touch_on_get_or_set = + config::enable_file_cache_async_touch_on_get_or_set; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::enable_file_cache_async_touch_on_get_or_set = + old_enable_file_cache_async_touch_on_get_or_set; + }}; + config::enable_external_file_meta_disk_cache = true; + config::enable_file_cache_async_touch_on_get_or_set = false; + + const std::string payload = "serialized footer payload spanning several blocks"; + const FileMetaDiskCacheTestValue encoded = + build_file_meta_disk_cache_value(FileMetaCacheFormat::PARQUET, 123, 456, payload); + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + settings.capacity = encoded.value.size() * 2 + settings.max_file_block_size; + settings.index_queue_size = settings.capacity; + settings.max_query_cache_size = 0; + io::BlockFileCache block_cache("file_meta_disk_cache_lru_touch_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + const std::string hot_key = FileMetaCache::get_key("s3://bucket/hot.parquet", 123, 456); + const std::string cold_key = FileMetaCache::get_key("s3://bucket/cold.parquet", 123, 456); + const auto hot_hash = io::BlockFileCache::hash( + file_meta_disk_cache_key(FileMetaCacheFormat::PARQUET, hot_key)); + const auto cold_hash = io::BlockFileCache::hash( + file_meta_disk_cache_key(FileMetaCacheFormat::PARQUET, cold_key)); + FileMetaDiskCache disk_cache(&block_cache); + ASSERT_TRUE(disk_cache.write(FileMetaCacheFormat::PARQUET, hot_key, 123, 456, payload).ok()); + ASSERT_TRUE(disk_cache.write(FileMetaCacheFormat::PARQUET, cold_key, 123, 456, payload).ok()); + const size_t hot_block_count = downloaded_block_count(&block_cache, hot_hash); + const size_t cold_block_count = downloaded_block_count(&block_cache, cold_hash); + ASSERT_GT(hot_block_count, 1); + ASSERT_EQ(cold_block_count, hot_block_count); + + std::string output; + ASSERT_TRUE(disk_cache.read(FileMetaCacheFormat::PARQUET, hot_key, 123, 456, &output).ok()); + EXPECT_EQ(output, payload); + EXPECT_EQ(block_cache.need_update_lru_blocks_size_unsafe(), 0); + + const auto filler_hash = io::BlockFileCache::hash("file_meta_disk_cache_lru_touch_filler"); + const std::string filler(settings.max_file_block_size * 2, 'f'); + write_raw_cache_value(&block_cache, filler_hash, filler); + EXPECT_EQ(downloaded_block_count(&block_cache, hot_hash), hot_block_count); + EXPECT_LT(downloaded_block_count(&block_cache, cold_hash), cold_block_count); + + output.clear(); + EXPECT_TRUE(disk_cache.read(FileMetaCacheFormat::PARQUET, hot_key, 123, 456, &output).ok()); + EXPECT_EQ(output, payload); + output.clear(); + EXPECT_FALSE(disk_cache.read(FileMetaCacheFormat::PARQUET, cold_key, 123, 456, &output).ok()); + EXPECT_TRUE(output.empty()); + EXPECT_EQ(downloaded_block_count(&block_cache, cold_hash), 0); +} + +TEST(FileMetaCacheDiskTest, EmptyPayloadIsNotPersisted) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = true; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + io::BlockFileCache block_cache("file_meta_disk_cache_empty_payload_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/empty.parquet", 123, 456); + const auto hash = io::BlockFileCache::hash( + file_meta_disk_cache_key(FileMetaCacheFormat::PARQUET, meta_key)); + FileMetaDiskCache disk_cache(&block_cache); + + const Status status = disk_cache.write(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, ""); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(downloaded_block_count(&block_cache, hash), 0); + EXPECT_EQ(block_cache.get_used_cache_size(io::FileCacheType::INDEX), 0); + EXPECT_EQ(block_cache.get_file_blocks_num(io::FileCacheType::INDEX), 0); +} + +} // namespace doris diff --git a/be/test/format/table/iceberg/iceberg_reader_test.cpp b/be/test/format/table/iceberg/iceberg_reader_test.cpp index 6c504e0b49a46b..2f156c207f9dfc 100644 --- a/be/test/format/table/iceberg/iceberg_reader_test.cpp +++ b/be/test/format/table/iceberg/iceberg_reader_test.cpp @@ -1486,6 +1486,50 @@ TEST_F(IcebergReaderTest, v1_orc_partial_id_equality_delete_ignores_stale_field_ std::filesystem::remove_all(test_dir); } +TEST_F(IcebergReaderTest, position_delete_inherits_disabled_file_meta_memory_cache) { + RuntimeState runtime_state = RuntimeState(TQueryOptions(), TQueryGlobals()); + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + + TFileRangeDesc scan_range; + scan_range.__set_fs_name(""); + scan_range.__set_path("data.parquet"); + scan_range.__set_start_offset(0); + scan_range.__set_size(0); + + RuntimeProfile profile("test_profile"); + cctz::time_zone ctz; + TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + + IcebergParquetReader iceberg_reader(&kv_cache, &profile, scan_params, scan_range, 1024, &ctz, + &io_ctx, &runtime_state, cache.get()); + iceberg_reader.set_enable_file_meta_memory_cache(false); + + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(IcebergReaderMixin::POSITION_DELETE); + delete_file.__set_path(mixed_position_delete_file()); + ASSERT_TRUE(iceberg_reader + .TEST_position_delete_base("file:///tmp/non_matching_data.parquet", + {delete_file}) + .ok()); + + io::FileReaderSPtr delete_file_reader; + ASSERT_TRUE(io::global_local_filesystem() + ->open_file(mixed_position_delete_file(), &delete_file_reader) + .ok()); + io::FileDescription file_description; + file_description.mtime = 0; + file_description.file_size = -1; + const std::string meta_key = FileMetaCache::get_key(delete_file_reader, file_description); + const std::string memory_key = + FileMetaCache::get_memory_cache_key(FileMetaCacheFormat::PARQUET, meta_key); + ObjLRUCache::CacheHandle handle; + EXPECT_FALSE(cache->lookup(memory_key, &handle)); +} + // Test reading real Iceberg Orc file using IcebergTableReader TEST_F(IcebergReaderTest, read_iceberg_orc_file) { // Read only: name, profile.address.coordinates.lat, profile.address.coordinates.lng, profile.contact.email diff --git a/be/test/format_v2/orc/orc_reader_test.cpp b/be/test/format_v2/orc/orc_reader_test.cpp index b98e2bf14e6d0e..5d4dca5c96a463 100644 --- a/be/test/format_v2/orc/orc_reader_test.cpp +++ b/be/test/format_v2/orc/orc_reader_test.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -72,11 +73,13 @@ #include "format_v2/file_reader.h" #include "gen_cpp/Types_types.h" #include "io/fs/buffered_reader.h" +#include "io/fs/file_meta_cache.h" #include "io/io_common.h" #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" #include "storage/segment/condition_cache.h" #include "storage/utils.h" +#include "util/defer_op.h" #include "util/timezone_utils.h" namespace doris { @@ -4391,6 +4394,67 @@ format::LocalColumnIndex struct_child_projection(int32_t root_field_id, int32_t return projection; } +class TestPersistentFileMetaCache final : public FileMetaPersistentCache { +public: + Status read(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string* payload) override { + ++_read_count; + const auto it = _entries.find(entry_key(format, key)); + if (it == _entries.end()) { + return Status::NotFound("test file meta cache entry not found"); + } + if (it->second.modification_time != modification_time || + it->second.file_size != file_size) { + return Status::NotFound("test file meta cache entry stale"); + } + *payload = it->second.payload; + return Status::OK(); + } + + Status write(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string_view payload) override { + ++_write_count; + _entries[entry_key(format, key)] = Entry {.modification_time = modification_time, + .file_size = file_size, + .payload = std::string(payload)}; + return Status::OK(); + } + + void remove(FileMetaCacheFormat format, const std::string& key) override { + _entries.erase(entry_key(format, key)); + ++_remove_count; + } + + bool contains(FileMetaCacheFormat format, const std::string& key) const { + return _entries.contains(entry_key(format, key)); + } + + std::string payload(FileMetaCacheFormat format, const std::string& key) const { + const auto it = _entries.find(entry_key(format, key)); + return it == _entries.end() ? std::string {} : it->second.payload; + } + + int read_count() const { return _read_count; } + int write_count() const { return _write_count; } + int remove_count() const { return _remove_count; } + +private: + struct Entry { + int64_t modification_time = 0; + int64_t file_size = 0; + std::string payload; + }; + + static std::string entry_key(FileMetaCacheFormat format, const std::string& key) { + return std::to_string(static_cast(format)) + ":" + key; + } + + std::map _entries; + int _read_count = 0; + int _write_count = 0; + int _remove_count = 0; +}; + class NewOrcReaderTest : public testing::Test { protected: void SetUp() override { @@ -4405,14 +4469,18 @@ class NewOrcReaderTest : public testing::Test { std::unique_ptr create_reader( RuntimeProfile* profile = nullptr, - std::optional global_rowid_context = std::nullopt) const { + std::optional global_rowid_context = std::nullopt, + FileMetaCache* file_meta_cache = nullptr, bool enable_file_meta_memory_cache = true, + int64_t mtime = 123) const { auto system_properties = std::make_shared(); system_properties->system_type = TFileType::FILE_LOCAL; auto file_description = std::make_unique(); file_description->path = _file_path; file_description->file_size = static_cast(std::filesystem::file_size(_file_path)); - return std::make_unique(system_properties, file_description, - nullptr, profile, global_rowid_context); + file_description->mtime = mtime; + return std::make_unique( + system_properties, file_description, nullptr, profile, global_rowid_context, false, + file_meta_cache, enable_file_meta_memory_cache); } std::unique_ptr create_reader_for_path( @@ -4451,6 +4519,223 @@ class NewOrcReaderTest : public testing::Test { std::string _file_path; }; +TEST_F(NewOrcReaderTest, UsesFileMetaCacheForSerializedTail) { + FileMetaCache file_meta_cache(1024 * 1024); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + RuntimeProfile first_profile("new_orc_reader_file_meta_cache_first"); + auto first_reader = create_reader(&first_profile, std::nullopt, &file_meta_cache); + ASSERT_TRUE(first_reader->init(&state).ok()); + EXPECT_EQ(first_reader->reader_statistics().file_footer_read_calls, 1); + + const auto file_size = static_cast(std::filesystem::file_size(_file_path)); + const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 123, file_size); + const std::string memory_cache_key = + FileMetaCache::get_memory_cache_key(FileMetaCacheFormat::ORC_V2, file_meta_cache_key); + ObjLRUCache::CacheHandle handle; + EXPECT_TRUE(file_meta_cache.lookup(memory_cache_key, &handle)); + + RuntimeProfile second_profile("new_orc_reader_file_meta_cache_second"); + auto second_reader = create_reader(&second_profile, std::nullopt, &file_meta_cache); + ASSERT_TRUE(second_reader->init(&state).ok()); + EXPECT_EQ(second_reader->reader_statistics().file_footer_read_calls, 0); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_cache, 1); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_memory_cache, 1); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_disk_cache, 0); + EXPECT_EQ(second_profile.get_counter("FileFooterHitMemoryCache")->value(), 1); +} + +TEST_F(NewOrcReaderTest, ReportsPersistentFileMetaCacheProfile) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 64 * 1024 * 1024; + + auto persistent_cache = std::make_unique(); + TestPersistentFileMetaCache* persistent_cache_ptr = persistent_cache.get(); + FileMetaCache file_meta_cache(0, std::move(persistent_cache)); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + RuntimeProfile first_profile("new_orc_reader_persistent_file_meta_cache_first"); + auto first_reader = create_reader(&first_profile, std::nullopt, &file_meta_cache); + ASSERT_TRUE(first_reader->init(&state).ok()); + EXPECT_EQ(first_reader->reader_statistics().file_footer_read_calls, 1); + EXPECT_EQ(first_reader->reader_statistics().file_footer_miss_disk_cache, 1); + EXPECT_EQ(first_reader->reader_statistics().file_footer_write_disk_cache, 1); + + const auto file_size = static_cast(std::filesystem::file_size(_file_path)); + const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 123, file_size); + EXPECT_TRUE(persistent_cache_ptr->contains(FileMetaCacheFormat::ORC_V2, file_meta_cache_key)); + EXPECT_FALSE(persistent_cache_ptr->contains(FileMetaCacheFormat::ORC, file_meta_cache_key)); + + RuntimeProfile second_profile("new_orc_reader_persistent_file_meta_cache_second"); + auto second_reader = create_reader(&second_profile, std::nullopt, &file_meta_cache); + ASSERT_TRUE(second_reader->init(&state).ok()); + EXPECT_EQ(second_reader->reader_statistics().file_footer_read_calls, 0); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_cache, 1); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_memory_cache, 0); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_disk_cache, 1); + EXPECT_EQ(second_profile.get_counter("FileFooterHitDiskCache")->value(), 1); +} + +TEST_F(NewOrcReaderTest, InvalidPersistentFileMetaCachePayloadFallsBackToFile) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 64 * 1024 * 1024; + + auto persistent_cache = std::make_unique(); + TestPersistentFileMetaCache* persistent_cache_ptr = persistent_cache.get(); + FileMetaCache file_meta_cache(0, std::move(persistent_cache)); + const auto file_size = static_cast(std::filesystem::file_size(_file_path)); + const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 123, file_size); + ASSERT_TRUE(persistent_cache_ptr + ->write(FileMetaCacheFormat::ORC_V2, file_meta_cache_key, 123, file_size, + "not a serialized orc file tail") + .ok()); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto reader = create_reader(nullptr, std::nullopt, &file_meta_cache); + ASSERT_TRUE(reader->init(&state).ok()); + EXPECT_EQ(reader->reader_statistics().file_footer_hit_disk_cache, 1); + EXPECT_EQ(reader->reader_statistics().file_footer_read_calls, 1); + EXPECT_EQ(reader->reader_statistics().file_footer_write_disk_cache, 1); + EXPECT_EQ(persistent_cache_ptr->remove_count(), 1); + EXPECT_TRUE(persistent_cache_ptr->contains(FileMetaCacheFormat::ORC_V2, file_meta_cache_key)); + EXPECT_NE(persistent_cache_ptr->payload(FileMetaCacheFormat::ORC_V2, file_meta_cache_key), + "not a serialized orc file tail"); +} + +TEST_F(NewOrcReaderTest, InvalidPersistentPayloadDoesNotRewriteOversizedTail) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 7; + + auto persistent_cache = std::make_unique(); + TestPersistentFileMetaCache* persistent_cache_ptr = persistent_cache.get(); + FileMetaCache file_meta_cache(0, std::move(persistent_cache)); + const auto file_size = static_cast(std::filesystem::file_size(_file_path)); + const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 123, file_size); + ASSERT_TRUE(persistent_cache_ptr + ->write(FileMetaCacheFormat::ORC_V2, file_meta_cache_key, 123, file_size, + "invalid") + .ok()); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto reader = create_reader(nullptr, std::nullopt, &file_meta_cache, false); + ASSERT_TRUE(reader->init(&state).ok()); + EXPECT_EQ(reader->reader_statistics().file_footer_hit_disk_cache, 1); + EXPECT_EQ(reader->reader_statistics().file_footer_read_calls, 1); + EXPECT_EQ(reader->reader_statistics().file_footer_write_disk_cache, 0); + EXPECT_EQ(persistent_cache_ptr->write_count(), 1); + EXPECT_EQ(persistent_cache_ptr->remove_count(), 1); + EXPECT_FALSE(persistent_cache_ptr->contains(FileMetaCacheFormat::ORC_V2, file_meta_cache_key)); +} + +TEST_F(NewOrcReaderTest, PersistentFileMetaCacheCanSkipMemoryCache) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = true; + + FileMetaCache file_meta_cache(1024 * 1024, std::make_unique()); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto first_reader = create_reader(nullptr, std::nullopt, &file_meta_cache, false); + ASSERT_TRUE(first_reader->init(&state).ok()); + + const auto file_size = static_cast(std::filesystem::file_size(_file_path)); + const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 123, file_size); + const std::string memory_cache_key = + FileMetaCache::get_memory_cache_key(FileMetaCacheFormat::ORC_V2, file_meta_cache_key); + ObjLRUCache::CacheHandle handle; + EXPECT_FALSE(file_meta_cache.lookup(memory_cache_key, &handle)); + + auto second_reader = create_reader(nullptr, std::nullopt, &file_meta_cache, false); + ASSERT_TRUE(second_reader->init(&state).ok()); + EXPECT_EQ(second_reader->reader_statistics().file_footer_read_calls, 0); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_memory_cache, 0); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_disk_cache, 1); + EXPECT_FALSE(file_meta_cache.lookup(memory_cache_key, &handle)); +} + +TEST_F(NewOrcReaderTest, UnstableMtimeSkipsPersistentFileMetaCache) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = true; + + auto persistent_cache = std::make_unique(); + TestPersistentFileMetaCache* persistent_cache_ptr = persistent_cache.get(); + FileMetaCache file_meta_cache(0, std::move(persistent_cache)); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + auto first_reader = create_reader(nullptr, std::nullopt, &file_meta_cache, false, 0); + ASSERT_TRUE(first_reader->init(&state).ok()); + auto second_reader = create_reader(nullptr, std::nullopt, &file_meta_cache, false, 0); + ASSERT_TRUE(second_reader->init(&state).ok()); + + EXPECT_EQ(first_reader->reader_statistics().file_footer_read_calls, 1); + EXPECT_EQ(second_reader->reader_statistics().file_footer_read_calls, 1); + EXPECT_EQ(persistent_cache_ptr->read_count(), 0); + EXPECT_EQ(persistent_cache_ptr->write_count(), 0); +} + +TEST_F(NewOrcReaderTest, OversizedPersistentPayloadRemainsMemoryCacheEligible) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 1; + + auto persistent_cache = std::make_unique(); + TestPersistentFileMetaCache* persistent_cache_ptr = persistent_cache.get(); + FileMetaCache file_meta_cache(1024 * 1024, std::move(persistent_cache)); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + auto first_reader = create_reader(nullptr, std::nullopt, &file_meta_cache); + ASSERT_TRUE(first_reader->init(&state).ok()); + EXPECT_EQ(first_reader->reader_statistics().file_footer_read_calls, 1); + EXPECT_EQ(persistent_cache_ptr->write_count(), 0); + + auto second_reader = create_reader(nullptr, std::nullopt, &file_meta_cache); + ASSERT_TRUE(second_reader->init(&state).ok()); + EXPECT_EQ(second_reader->reader_statistics().file_footer_read_calls, 0); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_memory_cache, 1); + EXPECT_EQ(persistent_cache_ptr->write_count(), 0); +} + TEST_F(NewOrcReaderTest, AggregatePushdownReturnsCountFromFileMetadata) { auto reader = create_reader(); RuntimeState state {TQueryOptions(), TQueryGlobals()}; diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index 796123832c6f16..be81f14b41d18c 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -35,6 +35,7 @@ #include #include +#include "common/config.h" #include "core/assert_cast.h" #include "core/block/block.h" #include "core/column/column_nullable.h" @@ -60,12 +61,14 @@ #include "format_v2/parquet/reader/column_reader.h" #include "format_v2/table_reader.h" #include "gen_cpp/Types_types.h" +#include "io/fs/file_meta_cache.h" #include "io/io_common.h" #include "runtime/runtime_state.h" #include "storage/index/zone_map/zonemap_eval_context.h" #include "storage/index/zone_map/zonemap_filter_result.h" #include "storage/segment/condition_cache.h" #include "storage/utils.h" +#include "util/defer_op.h" namespace doris { namespace { @@ -1082,6 +1085,66 @@ class TestFileReader final : public format::FileReader { long io_context_use_count() const { return _io_ctx.use_count(); } }; +class TestPersistentFileMetaCache final : public FileMetaPersistentCache { +public: + Status read(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string* payload) override { + const auto it = _entries.find(entry_key(format, key)); + if (it == _entries.end()) { + return Status::NotFound("test file meta cache entry not found"); + } + if (it->second.modification_time != modification_time || + it->second.file_size != file_size) { + return Status::NotFound("test file meta cache entry stale"); + } + *payload = it->second.payload; + return Status::OK(); + } + + Status write(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string_view payload) override { + _entries[entry_key(format, key)] = Entry {.modification_time = modification_time, + .file_size = file_size, + .payload = std::string(payload)}; + return Status::OK(); + } + + void remove(FileMetaCacheFormat format, const std::string& key) override { + _entries.erase(entry_key(format, key)); + ++_remove_count; + } + + bool contains(FileMetaCacheFormat format, const std::string& key) const { + return _entries.contains(entry_key(format, key)); + } + + std::string payload(FileMetaCacheFormat format, const std::string& key) const { + const auto it = _entries.find(entry_key(format, key)); + return it == _entries.end() ? std::string {} : it->second.payload; + } + + int remove_count() const { return _remove_count; } + +private: + struct Entry { + int64_t modification_time = 0; + int64_t file_size = 0; + std::string payload; + }; + + static std::string entry_key(FileMetaCacheFormat format, const std::string& key) { + std::string result; + result.reserve(key.size() + 4); + result.append(std::to_string(static_cast(format))); + result.push_back(':'); + result.append(key); + return result; + } + + std::map _entries; + int _remove_count = 0; +}; + TEST(FileReaderTest, OpenStoresRequestAndCloseKeepsRequest) { auto system_properties = std::make_shared(); system_properties->system_type = TFileType::FILE_LOCAL; @@ -1136,18 +1199,21 @@ class NewParquetReaderTest : public testing::Test { RuntimeProfile* profile = nullptr, bool enable_mapping_timestamp_tz = false, std::shared_ptr io_ctx = nullptr, std::optional global_rowid_context = std::nullopt, - bool is_immutable = false) const { + bool is_immutable = false, FileMetaCache* file_meta_cache = nullptr, + bool enable_file_meta_memory_cache = true, int64_t mtime = 123) const { auto system_properties = std::make_shared(); system_properties->system_type = TFileType::FILE_LOCAL; auto file_description = std::make_unique(); file_description->path = _file_path; file_description->file_size = static_cast(std::filesystem::file_size(_file_path)); + file_description->mtime = mtime; file_description->range_start_offset = range_start_offset; file_description->range_size = range_size; file_description->is_immutable = is_immutable; return std::make_unique( system_properties, file_description, std::move(io_ctx), profile, - global_rowid_context, enable_mapping_timestamp_tz); + global_rowid_context, enable_mapping_timestamp_tz, file_meta_cache, + enable_file_meta_memory_cache); } std::filesystem::path _test_dir; @@ -1172,6 +1238,183 @@ TEST_F(NewParquetReaderTest, GetSchemaReturnsFileLocalColumns) { EXPECT_EQ(remove_nullable(schema[1].type)->get_primitive_type(), TYPE_STRING); } +TEST_F(NewParquetReaderTest, UsesFileMetaCacheForFooterMetadata) { + FileMetaCache file_meta_cache(1024 * 1024); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + RuntimeProfile first_profile("new_parquet_reader_file_meta_cache_first"); + auto first_reader = create_reader(0, -1, &first_profile, false, nullptr, std::nullopt, false, + &file_meta_cache); + ASSERT_TRUE(first_reader->init(&state).ok()); + ASSERT_NE(first_profile.get_counter("FileFooterReadCalls"), nullptr); + EXPECT_EQ(first_profile.get_counter("FileFooterReadCalls")->value(), 1); + + const auto file_size = static_cast(std::filesystem::file_size(_file_path)); + const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 123, file_size); + const std::string memory_cache_key = FileMetaCache::get_memory_cache_key( + FileMetaCacheFormat::PARQUET_V2, file_meta_cache_key); + ObjLRUCache::CacheHandle handle; + EXPECT_TRUE(file_meta_cache.lookup(memory_cache_key, &handle)); + EXPECT_TRUE(handle.valid()); + + RuntimeProfile second_profile("new_parquet_reader_file_meta_cache_second"); + auto second_reader = create_reader(0, -1, &second_profile, false, nullptr, std::nullopt, false, + &file_meta_cache); + ASSERT_TRUE(second_reader->init(&state).ok()); + ASSERT_NE(second_profile.get_counter("FileFooterReadCalls"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitMemoryCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitDiskCache"), nullptr); + EXPECT_EQ(second_profile.get_counter("FileFooterReadCalls")->value(), 0); + EXPECT_EQ(second_profile.get_counter("FileFooterHitCache")->value(), 1); + EXPECT_EQ(second_profile.get_counter("FileFooterHitMemoryCache")->value(), 1); + EXPECT_EQ(second_profile.get_counter("FileFooterHitDiskCache")->value(), 0); + std::vector schema; + ASSERT_TRUE(second_reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); +} + +TEST_F(NewParquetReaderTest, ReportsPersistentFileMetaCacheProfile) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 64 * 1024 * 1024; + + FileMetaCache file_meta_cache(0, std::make_unique()); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + RuntimeProfile first_profile("new_parquet_reader_persistent_file_meta_cache_first"); + auto first_reader = create_reader(0, -1, &first_profile, false, nullptr, std::nullopt, false, + &file_meta_cache); + ASSERT_TRUE(first_reader->init(&state).ok()); + ASSERT_NE(first_profile.get_counter("FileFooterReadCalls"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterHitCache"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterHitMemoryCache"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterHitDiskCache"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterMissDiskCache"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterWriteDiskCache"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterReadDiskCacheTime"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterWriteDiskCacheTime"), nullptr); + EXPECT_EQ(first_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(first_profile.get_counter("FileFooterHitCache")->value(), 0); + EXPECT_EQ(first_profile.get_counter("FileFooterHitMemoryCache")->value(), 0); + EXPECT_EQ(first_profile.get_counter("FileFooterHitDiskCache")->value(), 0); + EXPECT_EQ(first_profile.get_counter("FileFooterMissDiskCache")->value(), 1); + EXPECT_EQ(first_profile.get_counter("FileFooterWriteDiskCache")->value(), 1); + + RuntimeProfile second_profile("new_parquet_reader_persistent_file_meta_cache_second"); + auto second_reader = create_reader(0, -1, &second_profile, false, nullptr, std::nullopt, false, + &file_meta_cache); + ASSERT_TRUE(second_reader->init(&state).ok()); + ASSERT_NE(second_profile.get_counter("FileFooterReadCalls"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitMemoryCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitDiskCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterMissDiskCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterWriteDiskCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterReadDiskCacheTime"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterWriteDiskCacheTime"), nullptr); + EXPECT_EQ(second_profile.get_counter("FileFooterReadCalls")->value(), 0); + EXPECT_EQ(second_profile.get_counter("FileFooterHitCache")->value(), 1); + EXPECT_EQ(second_profile.get_counter("FileFooterHitMemoryCache")->value(), 0); + EXPECT_EQ(second_profile.get_counter("FileFooterHitDiskCache")->value(), 1); + EXPECT_EQ(second_profile.get_counter("FileFooterMissDiskCache")->value(), 0); + EXPECT_EQ(second_profile.get_counter("FileFooterWriteDiskCache")->value(), 0); +} + +TEST_F(NewParquetReaderTest, InvalidPersistentFileMetaCachePayloadFallsBackToFile) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 64 * 1024 * 1024; + + auto persistent_cache = std::make_unique(); + TestPersistentFileMetaCache* persistent_cache_ptr = persistent_cache.get(); + FileMetaCache file_meta_cache(0, std::move(persistent_cache)); + + const auto file_size = static_cast(std::filesystem::file_size(_file_path)); + const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 123, file_size); + ASSERT_TRUE(persistent_cache_ptr + ->write(FileMetaCacheFormat::PARQUET_V2, file_meta_cache_key, 123, + file_size, "not a serialized parquet footer") + .ok()); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("new_parquet_reader_invalid_persistent_file_meta_cache_payload"); + auto reader = + create_reader(0, -1, &profile, false, nullptr, std::nullopt, false, &file_meta_cache); + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + EXPECT_EQ(profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(profile.get_counter("FileFooterWriteDiskCache")->value(), 1); + EXPECT_EQ(persistent_cache_ptr->remove_count(), 1); + EXPECT_TRUE( + persistent_cache_ptr->contains(FileMetaCacheFormat::PARQUET_V2, file_meta_cache_key)); + EXPECT_NE(persistent_cache_ptr->payload(FileMetaCacheFormat::PARQUET_V2, file_meta_cache_key), + "not a serialized parquet footer"); +} + +TEST_F(NewParquetReaderTest, PersistentFileMetaCacheCanSkipMemoryCache) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 64 * 1024 * 1024; + + FileMetaCache file_meta_cache(1024 * 1024, std::make_unique()); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + RuntimeProfile first_profile("new_parquet_reader_skip_memory_file_meta_cache_first"); + auto first_reader = create_reader(0, -1, &first_profile, false, nullptr, std::nullopt, false, + &file_meta_cache, false); + ASSERT_TRUE(first_reader->init(&state).ok()); + const auto file_size = static_cast(std::filesystem::file_size(_file_path)); + const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 123, file_size); + const std::string memory_cache_key = FileMetaCache::get_memory_cache_key( + FileMetaCacheFormat::PARQUET_V2, file_meta_cache_key); + ObjLRUCache::CacheHandle handle; + EXPECT_FALSE(file_meta_cache.lookup(memory_cache_key, &handle)); + ASSERT_NE(first_profile.get_counter("FileFooterReadCalls"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterWriteDiskCache"), nullptr); + EXPECT_EQ(first_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(first_profile.get_counter("FileFooterWriteDiskCache")->value(), 1); + + RuntimeProfile second_profile("new_parquet_reader_skip_memory_file_meta_cache_second"); + auto second_reader = create_reader(0, -1, &second_profile, false, nullptr, std::nullopt, false, + &file_meta_cache, false); + ASSERT_TRUE(second_reader->init(&state).ok()); + ASSERT_NE(second_profile.get_counter("FileFooterReadCalls"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitMemoryCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitDiskCache"), nullptr); + EXPECT_EQ(second_profile.get_counter("FileFooterReadCalls")->value(), 0); + EXPECT_EQ(second_profile.get_counter("FileFooterHitMemoryCache")->value(), 0); + EXPECT_EQ(second_profile.get_counter("FileFooterHitDiskCache")->value(), 1); + EXPECT_FALSE(file_meta_cache.lookup(memory_cache_key, &handle)); +} + // Scenario: Parquet is columnar and supports predicate/non-predicate split, nested projection and // file-layer pruning hints. The reader declares those scan-request capabilities by choosing // ParquetColumnMapper itself. @@ -1511,7 +1754,8 @@ TEST_F(NewParquetReaderTest, UnknownMtimeSkipsPageCacheForMutableFile) { write_parquet_file(_file_path); RuntimeProfile profile("new_parquet_reader_mutable_unknown_mtime"); - auto reader = create_reader(0, -1, &profile); + auto reader = + create_reader(0, -1, &profile, false, nullptr, std::nullopt, false, nullptr, true, 0); TQueryOptions query_options; query_options.__set_enable_parquet_file_page_cache(true); RuntimeState state {query_options, TQueryGlobals()}; @@ -1542,7 +1786,8 @@ TEST_F(NewParquetReaderTest, UnknownMtimeUsesPageCacheForImmutableFile) { RuntimeProfile first_profile("new_parquet_reader_first_unknown_mtime"); { - auto reader = create_reader(0, -1, &first_profile, false, nullptr, std::nullopt, true); + auto reader = create_reader(0, -1, &first_profile, false, nullptr, std::nullopt, true, + nullptr, true, 0); TQueryOptions query_options; query_options.__set_enable_parquet_file_page_cache(true); RuntimeState state {query_options, TQueryGlobals()}; @@ -1568,7 +1813,8 @@ TEST_F(NewParquetReaderTest, UnknownMtimeUsesPageCacheForImmutableFile) { EXPECT_GT(first_profile.get_counter("PageCacheWriteCount")->value(), 0); RuntimeProfile second_profile("new_parquet_reader_second_unknown_mtime"); - auto reader = create_reader(0, -1, &second_profile, false, nullptr, std::nullopt, true); + auto reader = create_reader(0, -1, &second_profile, false, nullptr, std::nullopt, true, nullptr, + true, 0); TQueryOptions query_options; query_options.__set_enable_parquet_file_page_cache(true); RuntimeState state {query_options, TQueryGlobals()}; diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index 0831c98750ecbf..fdf25f2925b9ca 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -66,6 +66,8 @@ #include "gen_cpp/Exprs_types.h" #include "gen_cpp/ExternalTableSchema_types.h" #include "gen_cpp/PlanNodes_types.h" +#include "io/fs/file_meta_cache.h" +#include "io/fs/local_file_system.h" #include "io/io_common.h" #include "roaring/roaring64map.hh" #include "runtime/runtime_profile.h" @@ -2902,6 +2904,71 @@ TEST(IcebergV2ReaderTest, IcebergPositionDeleteCacheIsScopedByFileSystem) { std::filesystem::remove_all(test_dir); } +TEST(IcebergV2ReaderTest, IcebergDeleteReaderInheritsFileMetaMemoryCachePolicy) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_delete_file_meta_cache_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + write_position_delete_parquet_file(delete_file_path, {file_path}, {1}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache delete_file_cache(1); + + auto delete_footer_is_cached = [&](bool enable_file_meta_memory_cache) { + FileMetaCache file_meta_cache(1024); + doris::format::iceberg::IcebergTableReader reader; + EXPECT_TRUE( + reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + .file_meta_cache = &file_meta_cache, + .enable_file_meta_memory_cache = enable_file_meta_memory_cache, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &delete_file_cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_position_delete_file(delete_file_path)})); + EXPECT_TRUE(reader.prepare_split(split_options).ok()); + + io::FileReaderSPtr delete_file_reader; + EXPECT_TRUE(io::global_local_filesystem() + ->open_file(delete_file_path, &delete_file_reader) + .ok()); + io::FileDescription file_description; + file_description.mtime = 0; + file_description.file_size = -1; + const std::string meta_key = FileMetaCache::get_key(delete_file_reader, file_description); + const std::string memory_key = + FileMetaCache::get_memory_cache_key(FileMetaCacheFormat::PARQUET_V2, meta_key); + ObjLRUCache::CacheHandle handle; + const bool cached = file_meta_cache.lookup(memory_key, &handle); + EXPECT_TRUE(reader.close().ok()); + return cached; + }; + + EXPECT_TRUE(delete_footer_is_cached(true)); + EXPECT_FALSE(delete_footer_is_cached(false)); + std::filesystem::remove_all(test_dir); +} + TEST(IcebergV2ReaderTest, IcebergTableReaderAppliesNullablePositionDeleteFileWithoutNulls) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_iceberg_nullable_position_delete_file_test"; diff --git a/conf/be.conf b/conf/be.conf index e9024580a65930..2508e9ece2ef5d 100644 --- a/conf/be.conf +++ b/conf/be.conf @@ -67,6 +67,10 @@ ssl_private_key_path = "$DORIS_HOME/conf/key.pem" # Default dirs to put jdbc drivers,default value is ${DORIS_HOME}/jdbc_drivers # jdbc_drivers_dir = ${DORIS_HOME}/jdbc_drivers +# Enable disk cache for external file metadata, such as Parquet/ORC footer. +# Enabled by default. This is a startup-only config. Set it before BE starts and restart BE to take effect. +# enable_external_file_meta_disk_cache = true + # Advanced configurations # INFO, WARNING, ERROR, FATAL sys_log_level = INFO diff --git a/gensrc/proto/file_cache.proto b/gensrc/proto/file_cache.proto index d6c5c3c0e8c46b..ecf8e305ae144d 100644 --- a/gensrc/proto/file_cache.proto +++ b/gensrc/proto/file_cache.proto @@ -61,3 +61,11 @@ message BlockMetaPb { optional uint64 ttl = 3; } +message FileMetaCacheEntryHeaderPB { + optional uint32 version = 1; + optional uint32 format = 2; + optional int64 file_size = 3; + optional int64 modification_time = 4; + optional uint64 payload_size = 5; + optional fixed32 checksum = 6; +}