From de60c76b49bbf8683ffb4b9b3f3ec492f471e8da Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Mon, 14 Sep 2026 23:45:19 +0800 Subject: [PATCH 1/6] [python] Align native scan planning with Rust --- paimon-python/README.md | 50 +++ paimon-python/conftest.py | 3 +- .../pypaimon/common/options/core_options.py | 10 + .../pypaimon/read/interval_partition.py | 21 +- paimon-python/pypaimon/read/native_plan.py | 56 ++- paimon-python/pypaimon/read/read_builder.py | 5 +- .../pypaimon/read/scan_distribution.py | 70 ++++ .../scanner/append_table_split_generator.py | 128 +------ .../scanner/data_evolution_split_generator.py | 4 +- .../pypaimon/read/scanner/file_scanner.py | 48 +-- .../pypaimon/read/scanner/split_generator.py | 24 +- paimon-python/pypaimon/read/sliced_split.py | 6 + paimon-python/pypaimon/read/table_scan.py | 133 +++++-- .../pypaimon/table/file_store_table.py | 2 +- .../data_evolution_split_generator_test.py | 7 + .../tests/deletion_vector_path_test.py | 301 +++++++++++++++ .../pypaimon/tests/interval_partition_test.py | 103 +++++ .../tests/native_plan_capabilities_test.py | 352 +++++++++++++++++ .../tests/native_plan_distribution_test.py | 362 ++++++++++++++++++ .../tests/native_plan_incremental_test.py | 250 ++++++++++++ .../tests/native_plan_integration_test.py | 44 ++- .../pypaimon/tests/native_plan_test.py | 246 ++++++++++-- .../pypaimon/utils/file_store_path_factory.py | 57 ++- .../pypaimon/write/file_store_commit.py | 16 +- paimon-python/pypaimon/write/table_delete.py | 13 +- 25 files changed, 2040 insertions(+), 271 deletions(-) create mode 100644 paimon-python/pypaimon/read/scan_distribution.py create mode 100644 paimon-python/pypaimon/tests/deletion_vector_path_test.py create mode 100644 paimon-python/pypaimon/tests/interval_partition_test.py create mode 100644 paimon-python/pypaimon/tests/native_plan_capabilities_test.py create mode 100644 paimon-python/pypaimon/tests/native_plan_distribution_test.py create mode 100644 paimon-python/pypaimon/tests/native_plan_incremental_test.py diff --git a/paimon-python/README.md b/paimon-python/README.md index 7c2708bb5495..1730deedba99 100644 --- a/paimon-python/README.md +++ b/paimon-python/README.md @@ -31,6 +31,56 @@ pip3 install dist/*.tar.gz The command will install the package and core dependencies to your local Python environment. +# Native scan planning + +PyPaimon can plan splits with the optional `pypaimon-rust` package while retaining +the Python reader: + +```python +native_table = table.copy({"scan.native-plan.enabled": "true"}) +builder = native_table.new_read_builder() +plan = builder.new_scan().plan() +rows = builder.new_read().to_arrow(plan.splits()) +explanation = builder.explain() +print(explanation.native_planned) +``` + +The adapter checks the installed binding's capabilities and falls back to the +Python planner for unsupported scans. New bindings preserve `plan.snapshot_id` +even when pruning removes every split. Native explain output includes snapshot +and split metadata; native pruning counters are not exposed. + +Explicit row ranges on data-evolution tables require `ReadBuilder.with_row_ranges()`. +Watermark time travel requires Rust 0.4 or newer. Branch reads require the +branch-aware binding exposing `Table.branch()`, and the resolved branch is +checked before planning. Deletion-vector scans require `pypaimon-rust>=0.4.0`, +which includes schema-aware decoding of Python-written index manifests and +legacy bucket-index path compatibility. The reader honors explicit paths, then +bucket paths, and can read older Python files placed in `table/index`. +New Python writes honor `index-file-in-data-file-dir` and retain explicit paths +when Python and Java partition-directory formatting differs. Older releases +and prereleases before 0.4.0 use the Python planner for deletion vectors. +When using an unreleased 0.4.0 development wheel, rebuild it with these fixes; +package version checks cannot distinguish local builds with identical versions. + +Append scans support `with_shard()` and `with_slice()`; primary-key scans support +bucket-based `with_shard()`. Data-evolution position selection requires the +binding's `TableScan.with_row_position_slice()` and `with_row_position_shard()`. +Selection occurs before reader filtering and deletion vectors, so surviving row +counts can differ between shards. Limits are applied after shard/slice selection. + +Timestamp incremental scans require `ReadBuilder.new_incremental_scan()`. +Python resolves `(start_timestamp, end_timestamp]` to snapshot IDs; Rust combines +the selected APPEND deltas into one plan, including merging primary-key versions +across commits. Other commit kinds are excluded, and the ending snapshot supplies +snapshot metadata and deletion vectors even if it contributes no APPEND files. + +Chunk shuffle, query authorization, first-row merge, deletion-vector merge-on-read, +dynamic or cross-partition primary-key buckets, and scored or primary-key +global-index results still use the Python planner. Rust also rejects floating-point +partition-directory formatting; these scans fall back to Python. Native planning +remains optional and is disabled by default. + # Load LeRobot Dataset v3 Install the optional dependency, then import a local directory, FileIO URI, or diff --git a/paimon-python/conftest.py b/paimon-python/conftest.py index 4ea23aa02e36..b64f9d47c6d8 100644 --- a/paimon-python/conftest.py +++ b/paimon-python/conftest.py @@ -61,7 +61,8 @@ def enable_native_plan(request, monkeypatch): if (not _native_plan_enabled() or request.node.get_closest_marker("python_plan") is not None or request.path.name in ( - "native_plan_test.py", "native_plan_integration_test.py")): + "native_plan_test.py", "native_plan_integration_test.py", + "native_plan_capabilities_test.py")): yield return diff --git a/paimon-python/pypaimon/common/options/core_options.py b/paimon-python/pypaimon/common/options/core_options.py index 903697a94ff2..cd9913ff0ed7 100644 --- a/paimon-python/pypaimon/common/options/core_options.py +++ b/paimon-python/pypaimon/common/options/core_options.py @@ -638,6 +638,13 @@ class CoreOptions: .with_description("Whether to enable deletion vectors.") ) + INDEX_FILE_IN_DATA_FILE_DIR: ConfigOption[bool] = ( + ConfigOptions.key("index-file-in-data-file-dir") + .boolean_type() + .default_value(False) + .with_description("Whether to store bucket index files in the data file directory.") + ) + SCAN_NATIVE_PLAN_ENABLED: ConfigOption[bool] = ( ConfigOptions.key("scan.native-plan.enabled") .boolean_type() @@ -1484,6 +1491,9 @@ def global_index_column_update_action(self, default=None): def deletion_vectors_enabled(self, default=None): return self.options.get(CoreOptions.DELETION_VECTORS_ENABLED, default) + def index_file_in_data_file_dir(self, default=None): + return self.options.get(CoreOptions.INDEX_FILE_IN_DATA_FILE_DIR, default) + def native_plan_enabled(self, default=None): return self.options.get(CoreOptions.SCAN_NATIVE_PLAN_ENABLED, default) diff --git a/paimon-python/pypaimon/read/interval_partition.py b/paimon-python/pypaimon/read/interval_partition.py index ef2cffb875a8..5c40a53e3f10 100644 --- a/paimon-python/pypaimon/read/interval_partition.py +++ b/paimon-python/pypaimon/read/interval_partition.py @@ -16,6 +16,7 @@ # under the License. import heapq +import math from dataclasses import dataclass from functools import cmp_to_key from typing import Callable, List @@ -41,9 +42,20 @@ class IntervalPartition: def __init__(self, input_files: List[DataFileMeta]): self.files = input_files.copy() self.key_comparator = default_key_comparator - self.files.sort(key=cmp_to_key(self._compare_files)) + # Manifest FLOAT/DOUBLE values decode to Python float. Match the native + # planner's conservative fallback for NaN boundaries: all files share + # one section, but each file remains a separate merge input. + self.has_nan_key = any( + isinstance(value, float) and math.isnan(value) + for file in self.files + for key in (file.min_key, file.max_key) if key is not None + for value in key.values) + if not self.has_nan_key: + self.files.sort(key=cmp_to_key(self._compare_files)) def partition(self) -> List[List[SortedRun]]: + if self.has_nan_key: + return [[SortedRun(files=[file]) for file in self.files]] result = [] section: List[DataFileMeta] = [] bound = None @@ -116,6 +128,13 @@ def default_key_comparator(key1: GenericRow, key2: GenericRow) -> int: return -1 if val2 is None: return 1 + # Preserve Java's ordering of signed zeros in composite key bounds. + # NaN bounds take the conservative path before this comparator is used. + if (isinstance(val1, float) and isinstance(val2, float) + and val1 == 0.0 and val2 == 0.0): + sign1, sign2 = math.copysign(1.0, val1), math.copysign(1.0, val2) + if sign1 != sign2: + return -1 if sign1 < sign2 else 1 if val1 < val2: return -1 elif val1 > val2: diff --git a/paimon-python/pypaimon/read/native_plan.py b/paimon-python/pypaimon/read/native_plan.py index 3d3732a4f41e..b3f6ae9079cf 100644 --- a/paimon-python/pypaimon/read/native_plan.py +++ b/paimon-python/pypaimon/read/native_plan.py @@ -22,13 +22,15 @@ still applies them while reading, so pushdown remains an optimization. """ -import re from typing import List, Optional, Tuple +from packaging.version import InvalidVersion, Version + from pypaimon.common.options.config import CatalogOptions, OssOptions from pypaimon.common.options.core_options import CoreOptions from pypaimon.common.options.options_utils import OptionsUtils from pypaimon.common.predicate import Predicate +from pypaimon.read.plan import Plan from pypaimon.read.split import Split from pypaimon.read.split_serializer import deserialize_split_v1 @@ -49,6 +51,11 @@ def native_runtime_available() -> bool: def native_family_search_modes_available() -> bool: """Whether Rust supports family-specific global-index search modes.""" + return native_version_at_least(0, 4) + + +def native_version_at_least(major: int, minor: int, patch: int = 0) -> bool: + """Compare complete package versions, including patch and pre-release ordering.""" if not native_runtime_available(): return False try: @@ -56,11 +63,18 @@ def native_family_search_modes_available() -> bool: except ImportError: return False try: - rust_version = version('pypaimon-rust') - except PackageNotFoundError: + rust_version = Version(version('pypaimon-rust')) + except (PackageNotFoundError, InvalidVersion): + return False + return rust_version >= Version('%d.%d.%d' % (major, minor, patch)) + + +def native_method_available(type_name: str, method: str) -> bool: + try: + from pypaimon_rust import datafusion + return callable(getattr(getattr(datafusion, type_name, None), method, None)) + except ImportError: return False - match = re.match(r'^(\d+)\.(\d+)', rust_version) - return match is not None and tuple(map(int, match.groups())) >= (0, 4) def _partition_fields(table): @@ -134,6 +148,7 @@ def _read_options(table) -> dict: CoreOptions.SCAN_SNAPSHOT_ID, CoreOptions.SCAN_TAG_NAME, CoreOptions.SCAN_TIMESTAMP_MILLIS, + CoreOptions.SCAN_WATERMARK, CoreOptions.GLOBAL_INDEX_SEARCH_MODE, CoreOptions.SCALAR_INDEX_SEARCH_MODE, CoreOptions.VECTOR_INDEX_SEARCH_MODE, @@ -201,8 +216,11 @@ def native_plan( predicate: Optional[Predicate] = None, limit: Optional[int] = None, projection: Optional[List[str]] = None, - row_ranges: Optional[List[Tuple[int, int]]] = None) -> List[Split]: - """Plan with pypaimon_rust and return the decoded pypaimon splits. + row_ranges: Optional[List[Tuple[int, int]]] = None, + incremental_range: Optional[Tuple[int, int]] = None, + row_position_slice: Optional[Tuple[int, int]] = None, + row_position_shard: Optional[Tuple[int, int]] = None) -> Plan: + """Plan with pypaimon_rust, preserving snapshot metadata. Native conversion or planning failures are handled by TableScan, which falls back to the Python planner. @@ -213,6 +231,10 @@ def native_plan( from pypaimon_rust.datafusion import PaimonCatalog rt = PaimonCatalog(_catalog_options(table)).get_table(table.identifier.get_full_name()) + if table.current_branch() != 'main': + branch = getattr(rt, 'branch', None) + if not callable(branch) or branch() != table.current_branch(): + raise RuntimeError("Native table did not resolve the requested branch") builder = rt.new_read_builder(_read_options(table)) if projection is not None: builder = builder.with_projection(projection) @@ -222,10 +244,26 @@ def native_plan( builder = builder.with_limit(limit) if row_ranges is not None: builder = builder.with_row_ranges(row_ranges) - rust_splits = builder.new_scan().plan().splits() + scan = (builder.new_scan() if incremental_range is None + else builder.new_incremental_scan(*incremental_range)) + if row_position_slice is not None: + scan = scan.with_row_position_slice(*row_position_slice) + if row_position_shard is not None: + scan = scan.with_row_position_shard(*row_position_shard) + rust_plan = scan.plan() + rust_splits = rust_plan.splits() pfields = _partition_fields(table) # Trimmed primary keys decode per-file min/max keys (PK merge-on-read). kfields = table.trimmed_primary_keys_fields splits = [deserialize_split_v1(s.serialize(), pfields, kfields) for s in rust_splits] _restore_python_partition_paths(table, splits) - return splits + snapshot_id = getattr(rust_plan, 'snapshot_id', None) + if callable(snapshot_id): + snapshot_id = snapshot_id() + elif splits: + snapshot_id = splits[0].snapshot_id + else: + # Older bindings cannot distinguish an empty committed snapshot from a + # table without snapshots. Let the Python scanner recover the metadata. + raise RuntimeError("Native runtime cannot report an empty plan's snapshot") + return Plan(splits, snapshot_id=snapshot_id) diff --git a/paimon-python/pypaimon/read/read_builder.py b/paimon-python/pypaimon/read/read_builder.py index a531d5d2c83f..02e058687a48 100644 --- a/paimon-python/pypaimon/read/read_builder.py +++ b/paimon-python/pypaimon/read/read_builder.py @@ -208,7 +208,7 @@ def _resolve_dotted_paths(self, names: List[str]) -> List[List[int]]: return paths -def _build_explain_result(table, scan: TableScan, plan, stats: ScanStats, +def _build_explain_result(table, scan: TableScan, plan, stats: Optional[ScanStats], predicate, projection, limit, verbose: bool) -> ExplainResult: """Translate one (Plan, ScanStats) pair into an ExplainResult.""" splits: List[Split] = plan.splits() @@ -216,8 +216,7 @@ def _build_explain_result(table, scan: TableScan, plan, stats: ScanStats, table_schema = table.table_schema bucket_mode_str = _safe_bucket_mode(table) - # stats is None when planned natively (pypaimon_rust): no manifest pruning - # funnel is tracked, so the split-level signals below are all we can report. + # Native plans expose split metadata without Python pruning counters. native_planned = stats is None if native_planned: partition_pruning = bucket_pruning = file_skipping = None diff --git a/paimon-python/pypaimon/read/scan_distribution.py b/paimon-python/pypaimon/read/scan_distribution.py new file mode 100644 index 000000000000..416ae3ef14df --- /dev/null +++ b/paimon-python/pypaimon/read/scan_distribution.py @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Row-position selection shared by the Python and native planning adapters.""" + +from typing import List, Tuple + +from pypaimon.read.sliced_split import SlicedSplit +from pypaimon.read.split import Split + + +def validate_shard(index: int, count: int) -> None: + if not isinstance(count, int) or count <= 0: + raise ValueError("number_of_para_subtasks must be a positive integer") + if not isinstance(index, int) or index < 0 or index >= count: + raise ValueError("idx_of_this_subtask must be non-negative and less than number_of_para_subtasks") + + +def validate_slice(start: int, end: int) -> None: + if not isinstance(start, int) or not isinstance(end, int) or start < 0 or start >= end: + raise ValueError("start_pos must be non-negative and less than end_pos; both must be integers") + + +def shard_range(total: int, index: int, count: int) -> Tuple[int, int]: + base, remainder = divmod(total, count) + start = index * base + min(index, remainder) + return start, start + base + int(index < remainder) + + +def slice_append_splits(splits: List[Split], start: int, end: int) -> List[Split]: + """Select physical rows in [start, end), preserving file order and DVs. + + File statistics have already pruned the input; residual row predicates and + deletion vectors are applied by the reader after selecting these positions. + An end beyond the table is naturally clamped by each file's row count. + """ + if start >= end: + return [] + selected = [] + offset = 0 + for split in splits: + keep_ids, ranges = set(), {} + for file in split.files: + begin, stop = max(0, start - offset), min(file.row_count, end - offset) + offset += file.row_count + if begin < stop: + keep_ids.add(id(file)) + if begin != 0 or stop != file.row_count: + ranges[file.file_name] = (begin, stop) + if not keep_ids: + continue + kept = split.filter_file(lambda file: id(file) in keep_ids) + selected.append(SlicedSplit(kept, ranges) if ranges else kept) + if offset >= end: + break + return selected diff --git a/paimon-python/pypaimon/read/scanner/append_table_split_generator.py b/paimon-python/pypaimon/read/scanner/append_table_split_generator.py index 3387d305ba8a..44c984b580d1 100644 --- a/paimon-python/pypaimon/read/scanner/append_table_split_generator.py +++ b/paimon-python/pypaimon/read/scanner/append_table_split_generator.py @@ -16,13 +16,13 @@ # under the License. from collections import defaultdict -from typing import List, Dict, Tuple +from typing import List from pypaimon.manifest.schema.data_file_meta import DataFileMeta from pypaimon.manifest.schema.manifest_entry import ManifestEntry from pypaimon.read.scanner.split_generator import AbstractSplitGenerator from pypaimon.read.split import Split -from pypaimon.read.sliced_split import SlicedSplit +from pypaimon.read.scan_distribution import slice_append_splits class AppendTableSplitGenerator(AbstractSplitGenerator): @@ -35,20 +35,6 @@ def create_splits(self, file_entries: List[ManifestEntry]) -> List[Split]: for entry in file_entries: partitioned_files[(tuple(entry.partition.values), entry.bucket)].append(entry) - plan_start_pos = 0 - plan_end_pos = 0 - - if self.start_pos_of_this_subtask is not None: - # shard data range: [plan_start_pos, plan_end_pos) - partitioned_files, plan_start_pos, plan_end_pos = \ - self.__filter_by_slice( - partitioned_files, - self.start_pos_of_this_subtask, - self.end_pos_of_this_subtask - ) - elif self.idx_of_this_subtask is not None: - partitioned_files, plan_start_pos, plan_end_pos = self._filter_by_shard(partitioned_files) - def weight_func(f: DataFileMeta) -> int: return max(f.file_size, self.open_file_cost) @@ -66,108 +52,10 @@ def weight_func(f: DataFileMeta) -> int: packed_files, file_entries_list, False ) - if self.start_pos_of_this_subtask is not None or self.idx_of_this_subtask is not None: - splits = self._wrap_to_sliced_splits(splits, plan_start_pos, plan_end_pos) - + if self.idx_of_this_subtask is not None: + start, end = self._compute_shard_range(sum(split.row_count for split in splits)) + return slice_append_splits(splits, start, end) + if self.start_pos_of_this_subtask is not None: + return slice_append_splits( + splits, self.start_pos_of_this_subtask, self.end_pos_of_this_subtask) return splits - - def _wrap_to_sliced_splits(self, splits: List[Split], plan_start_pos: int, plan_end_pos: int) -> List[Split]: - sliced_splits = [] - file_end_pos = 0 # end row position of current file in all splits data - - for split in splits: - shard_file_idx_map = self.__compute_split_file_idx_map( - plan_start_pos, plan_end_pos, split, file_end_pos - ) - file_end_pos = shard_file_idx_map[self.NEXT_POS_KEY] - del shard_file_idx_map[self.NEXT_POS_KEY] - - if shard_file_idx_map: - sliced_splits.append(SlicedSplit(split, shard_file_idx_map)) - else: - sliced_splits.append(split) - - return sliced_splits - - @staticmethod - def __filter_by_slice( - partitioned_files: defaultdict, - start_pos: int, - end_pos: int - ) -> tuple: - plan_start_pos = 0 - plan_end_pos = 0 - entry_end_pos = 0 # end row position of current file in all data - splits_start_pos = 0 - filtered_partitioned_files = defaultdict(list) - - # Iterate through all file entries to find files that overlap with current shard range - for key, file_entries in partitioned_files.items(): - filtered_entries = [] - for entry in file_entries: - entry_begin_pos = entry_end_pos # Starting row position of current file in all data - entry_end_pos += entry.file.row_count # Update to row position after current file - - # If current file is completely after shard range, stop iteration - if entry_begin_pos >= end_pos: - break - # If current file is completely before shard range, skip it - if entry_end_pos <= start_pos: - continue - if entry_begin_pos <= start_pos < entry_end_pos: - splits_start_pos = entry_begin_pos - plan_start_pos = start_pos - entry_begin_pos - # If shard end position is within current file, record relative end position - if entry_begin_pos < end_pos <= entry_end_pos: - plan_end_pos = end_pos - splits_start_pos - # Add files that overlap with shard range to result - filtered_entries.append(entry) - if filtered_entries: - filtered_partitioned_files[key] = filtered_entries - - return filtered_partitioned_files, plan_start_pos, plan_end_pos - - def _filter_by_shard(self, partitioned_files: defaultdict) -> tuple: - """ - Filter file entries by shard. Only keep the files within the range, which means - that only the starting and ending files need to be further divided subsequently. - """ - # Calculate total rows - total_row = sum( - entry.file.row_count - for file_entries in partitioned_files.values() - for entry in file_entries - ) - - # Calculate shard range using shared helper - start_pos, end_pos = self._compute_shard_range(total_row) - - return self.__filter_by_slice(partitioned_files, start_pos, end_pos) - - @staticmethod - def __compute_split_file_idx_map( - plan_start_pos: int, - plan_end_pos: int, - split: Split, - file_end_pos: int - ) -> Dict[str, Tuple[int, int]]: - """ - Compute file index map for a split, determining which rows to read from each file. - - """ - shard_file_idx_map = {} - - for file in split.files: - file_begin_pos = file_end_pos # Starting row position of current file in all data - file_end_pos += file.row_count # Update to row position after current file - - # Use shared helper to compute file range - file_range = AppendTableSplitGenerator._compute_file_range( - plan_start_pos, plan_end_pos, file_begin_pos, file.row_count - ) - - if file_range is not None: - shard_file_idx_map[file.file_name] = file_range - - shard_file_idx_map[AppendTableSplitGenerator.NEXT_POS_KEY] = file_end_pos - return shard_file_idx_map diff --git a/paimon-python/pypaimon/read/scanner/data_evolution_split_generator.py b/paimon-python/pypaimon/read/scanner/data_evolution_split_generator.py index 5536026126e8..5f5a1dad52f6 100644 --- a/paimon-python/pypaimon/read/scanner/data_evolution_split_generator.py +++ b/paimon-python/pypaimon/read/scanner/data_evolution_split_generator.py @@ -107,7 +107,9 @@ def weight_func(file_list: List[DataFileMeta]) -> int: slice_row_ranges = Range.and_(slice_row_ranges, self.row_ranges) # Wrap splits with IndexedSplit for slice-based filtering or row_ranges - if slice_row_ranges: + if slice_row_ranges is not None: + if not slice_row_ranges: + return [] splits = self._wrap_to_indexed_splits(splits, slice_row_ranges) return splits diff --git a/paimon-python/pypaimon/read/scanner/file_scanner.py b/paimon-python/pypaimon/read/scanner/file_scanner.py index c63d3cf9a4ab..f8facf435b70 100755 --- a/paimon-python/pypaimon/read/scanner/file_scanner.py +++ b/paimon-python/pypaimon/read/scanner/file_scanner.py @@ -33,6 +33,7 @@ from pypaimon.manifest.simple_stats_evolutions import SimpleStatsEvolutions from pypaimon.schema.data_types import DataField from pypaimon.read.plan import Plan +from pypaimon.read.scan_distribution import validate_shard, validate_slice from pypaimon.read.push_down_utils import (_get_all_fields, exclude_predicate_with_fields, remove_row_id_filter, @@ -486,21 +487,25 @@ def _create_data_evolution_split_generator(self): None, ) - # Filter manifest files by row ranges if available - if row_ranges is not None: - manifest_files = _filter_manifest_files_by_row_ranges(manifest_files, row_ranges) + # Position selection counts the complete candidate row-id space. Early + # range pruning would renumber later files; the split generator applies + # the range intersection after assigning slice/shard positions. + positional = self.idx_of_this_subtask is not None or self.start_pos_of_this_subtask is not None + early_row_ranges = None if positional else row_ranges + if early_row_ranges is not None: + manifest_files = _filter_manifest_files_by_row_ranges(manifest_files, early_row_ranges) stats_predicate = getattr(self, 'predicate_for_stats', None) group_stats_enabled = stats_predicate is not None and score_getter is None entries = self.read_manifest_entries( manifest_files, - row_ranges=row_ranges, + row_ranges=early_row_ranges, keep_stats=group_stats_enabled, ) # Redundant when early_record_filter ran; kept for explain mode and as safety net. - if row_ranges is not None: - entries = _filter_manifest_entries_by_row_ranges(entries, row_ranges) + if early_row_ranges is not None: + entries = _filter_manifest_entries_by_row_ranges(entries, early_row_ranges) group_stats_filter = None if group_stats_enabled: @@ -632,19 +637,19 @@ def _filter(bucket: int, total_buckets: int) -> bool: return _filter def with_shard(self, idx_of_this_subtask: int, number_of_para_subtasks: int) -> 'FileScanner': - if idx_of_this_subtask >= number_of_para_subtasks: - raise ValueError("idx_of_this_subtask must be less than number_of_para_subtasks") + validate_shard(idx_of_this_subtask, number_of_para_subtasks) if self.start_pos_of_this_subtask is not None: - raise Exception("with_shard and with_slice cannot be used simultaneously") + raise ValueError("with_shard and with_slice cannot be used simultaneously") self.idx_of_this_subtask = idx_of_this_subtask self.number_of_para_subtasks = number_of_para_subtasks return self def with_slice(self, start_pos: int, end_pos: int) -> 'FileScanner': - if start_pos >= end_pos: - raise ValueError("start_pos must be less than end_pos") + validate_slice(start_pos, end_pos) + if self.table.is_primary_key_table: + raise NotImplementedError("Primary key tables do not support with_slice(); use with_shard instead") if self.idx_of_this_subtask is not None: - raise Exception("with_slice and with_shard cannot be used simultaneously") + raise ValueError("with_slice and with_shard cannot be used simultaneously") self.start_pos_of_this_subtask = start_pos self.end_pos_of_this_subtask = end_pos return self @@ -729,11 +734,12 @@ def _apply_push_down_limit(self, splits: List[DataSplit]) -> List[DataSplit]: limited_splits: List[DataSplit] = [] for split in splits: merged = split.merged_row_count() - if merged is not None: - limited_splits.append(split) - scanned_row_count += merged - if scanned_row_count >= self.limit: - return limited_splits + if merged is None: + return splits + limited_splits.append(split) + scanned_row_count += merged + if scanned_row_count >= self.limit: + return limited_splits return splits def _has_non_partition_filter(self) -> bool: @@ -944,7 +950,7 @@ def _scan_dv_index(self, snapshot, buckets: Set[tuple]) -> Dict[tuple, Dict[str, # Convert to deletion files deletion_files = self._to_deletion_files(entry) if deletion_files: - result[partition_bucket] = deletion_files + result.setdefault(partition_bucket, {}).update(deletion_files) return result @@ -960,10 +966,8 @@ def _to_deletion_files(self, index_entry) -> Dict[str, DeletionFile]: if not index_file.dv_ranges: return deletion_files - # Build deletion file path - # Format: manifest/index-manifest-{uuid} - index_path = self.table.table_path.rstrip('/') + '/index' - dv_file_path = f"{index_path}/{index_file.file_name}" + dv_file_path = self.table.path_factory().bucket_index_path( + tuple(index_entry.partition.values), index_entry.bucket, index_file, self.table.file_io) # Convert each DeletionVectorMeta to DeletionFile for data_file_name, dv_meta in index_file.dv_ranges.items(): diff --git a/paimon-python/pypaimon/read/scanner/split_generator.py b/paimon-python/pypaimon/read/scanner/split_generator.py index dfc25dfd05e1..4c61ebae22ce 100644 --- a/paimon-python/pypaimon/read/scanner/split_generator.py +++ b/paimon-python/pypaimon/read/scanner/split_generator.py @@ -21,6 +21,7 @@ from pypaimon.common.options.core_options import CoreOptions from pypaimon.manifest.schema.data_file_meta import DataFileMeta from pypaimon.manifest.schema.manifest_entry import ManifestEntry +from pypaimon.read.scan_distribution import shard_range, validate_shard, validate_slice from pypaimon.read.split import Split from pypaimon.read.split import DataSplit from pypaimon.table.row.generic_row import GenericRow @@ -57,8 +58,7 @@ def __init__( def with_shard(self, idx_of_this_subtask: int, number_of_para_subtasks: int): """Configure sharding for parallel processing.""" - if idx_of_this_subtask >= number_of_para_subtasks: - raise ValueError("idx_of_this_subtask must be less than number_of_para_subtasks") + validate_shard(idx_of_this_subtask, number_of_para_subtasks) if self.start_pos_of_this_subtask is not None: raise ValueError("with_shard and with_slice cannot be used simultaneously") self.idx_of_this_subtask = idx_of_this_subtask @@ -67,8 +67,7 @@ def with_shard(self, idx_of_this_subtask: int, number_of_para_subtasks: int): def with_slice(self, start_pos: int, end_pos: int): """Configure slice range for processing.""" - if start_pos >= end_pos: - raise ValueError("start_pos must be less than end_pos") + validate_slice(start_pos, end_pos) if self.idx_of_this_subtask is not None: raise ValueError("with_slice and with_shard cannot be used simultaneously") self.start_pos_of_this_subtask = start_pos @@ -193,22 +192,7 @@ def _compute_shard_range(self, total_row: int) -> Tuple[int, int]: Calculate start and end positions for this shard based on total rows. Uses balanced distribution to avoid last shard overload. """ - base_rows_per_shard = total_row // self.number_of_para_subtasks - remainder = total_row % self.number_of_para_subtasks - - # Each of the first 'remainder' shards gets one extra row - if self.idx_of_this_subtask < remainder: - num_row = base_rows_per_shard + 1 - start_pos = self.idx_of_this_subtask * (base_rows_per_shard + 1) - else: - num_row = base_rows_per_shard - start_pos = ( - remainder * (base_rows_per_shard + 1) + - (self.idx_of_this_subtask - remainder) * base_rows_per_shard - ) - - end_pos = start_pos + num_row - return start_pos, end_pos + return shard_range(total_row, self.idx_of_this_subtask, self.number_of_para_subtasks) @staticmethod def _compute_file_range( diff --git a/paimon-python/pypaimon/read/sliced_split.py b/paimon-python/pypaimon/read/sliced_split.py index 0bdf42ab5d95..137e8746283a 100644 --- a/paimon-python/pypaimon/read/sliced_split.py +++ b/paimon-python/pypaimon/read/sliced_split.py @@ -108,6 +108,12 @@ def merged_row_count(self): return self._exact_merged_row_count if not self._shard_file_idx_map: return self._data_split.merged_row_count() + + if (any(deletion is not None for deletion in self.data_deletion_files or []) + and any(self._get_sliced_file_row_count(file) != file.row_count + for file in self.files)): + # File-wide deletion counts cannot locate deletions inside a slice. + return None underlying_merged = self._data_split.merged_row_count() if underlying_merged is not None: diff --git a/paimon-python/pypaimon/read/table_scan.py b/paimon-python/pypaimon/read/table_scan.py index 884d33b3cb5e..ce56bc29eedf 100755 --- a/paimon-python/pypaimon/read/table_scan.py +++ b/paimon-python/pypaimon/read/table_scan.py @@ -42,12 +42,16 @@ } _NATIVE_FORWARDED_OPTIONS = frozenset({ CoreOptions.SCAN_NATIVE_PLAN_ENABLED.key(), + CoreOptions.SCAN_MODE.key(), + CoreOptions.INCREMENTAL_BETWEEN_TIMESTAMP.key(), CoreOptions.SOURCE_SPLIT_TARGET_SIZE.key(), CoreOptions.SOURCE_SPLIT_OPEN_FILE_COST.key(), CoreOptions.SCAN_SNAPSHOT_ID.key(), CoreOptions.SCAN_TAG_NAME.key(), CoreOptions.SCAN_TIMESTAMP.key(), CoreOptions.SCAN_TIMESTAMP_MILLIS.key(), + CoreOptions.SCAN_WATERMARK.key(), + CoreOptions.BRANCH.key(), }) | _NATIVE_SEARCH_MODE_OPTIONS _NATIVE_PLAN_INDEPENDENT_OPTIONS = frozenset({ CoreOptions.BLOB_AS_DESCRIPTOR.key(), @@ -59,6 +63,7 @@ CoreOptions.SCAN_TAG_NAME.key(), CoreOptions.SCAN_TIMESTAMP.key(), CoreOptions.SCAN_TIMESTAMP_MILLIS.key(), + CoreOptions.SCAN_WATERMARK.key(), }) @@ -85,8 +90,8 @@ def __init__( def plan(self) -> Plan: auth_result = self.__auth_query() - # Native planning covers only a plain full-snapshot scan and bypasses the - # auth-aware file scanner; fall back to the normal path otherwise. + # The native planner bypasses the auth-aware file scanner. Resolve auth + # before selecting a planning backend. if (auth_result is None and self.table.options.native_plan_enabled() and self._native_plan_supported()): native = self._try_native_plan() @@ -107,29 +112,51 @@ def _native_plan_supported(self) -> bool: def _native_plan_supported_impl(self) -> bool: """Fall back to the Python scanner for scans native can't carry: - shard/slice, chunk-shuffle, explicit row ranges, scored or primary-key + chunk-shuffle, scored or primary-key global-index results, first-row merge-engine (Rust drops L0), deletion - vectors, postpone bucket, + vector merge-on-read, postpone bucket, a primary-key table whose trimmed PK is empty (PK equals the partition - key; native may mark splits raw-convertible and skip merge), dynamic + key; Rust rejects this schema), dynamic bucket / cross-partition PK tables (unconfirmed Rust parity), a stale schema without time travel, removed copy() options which Rust cannot represent, unsupported time travel selectors, - query auth, non-main branch, incremental scans, a missing/old + query auth, a missing/old pypaimon-rust, or a catalog / identifier Rust cannot reconstruct. Keep this capability gate in sync when adding scan features.""" - from pypaimon.read.native_plan import native_runtime_available + from pypaimon.read.native_plan import ( + native_method_available, native_runtime_available, native_version_at_least, + ) if not native_runtime_available(): return False fs = self.file_scanner - if (getattr(fs, 'idx_of_this_subtask', None) is not None - or getattr(fs, 'start_pos_of_this_subtask', None) is not None - or getattr(fs, 'chunk_shuffle', None) is not None - or getattr(fs, '_row_ranges', None) is not None + if (getattr(fs, 'chunk_shuffle', None) is not None or not self._native_global_index_result_supported() - or getattr(fs, 'deletion_vectors_enabled', False) or getattr(fs, 'only_read_real_buckets', False)): return False + if getattr(fs, 'deletion_vectors_enabled', False): + # 0.4.0 includes Python-written DV decoding and legacy bucket paths. + if not native_version_at_least(0, 4, 0): + return False + # Python DV scans skip L0; Rust can include L0 when this option is + # enabled. Keep that mode on the Python planner until aligned. + table_options = self.table.options.options.to_map() + merge_on_read = table_options.get( + 'deletion-vectors.merge-on-read', False) + if str(merge_on_read).lower() == 'true': + return False + if getattr(fs, 'data_evolution', False): + if (getattr(fs, 'idx_of_this_subtask', None) is not None + and not native_method_available('TableScan', 'with_row_position_shard')): + return False + if (getattr(fs, 'start_pos_of_this_subtask', None) is not None + and not native_method_available('TableScan', 'with_row_position_slice')): + return False + if (getattr(fs, '_row_ranges', None) is not None + and not native_method_available('ReadBuilder', 'with_row_ranges')): + return False + if (self.table.current_branch() != 'main' + and not native_method_available('Table', 'branch')): + return False loader = getattr( getattr(self.table, 'catalog_environment', None), 'catalog_loader', @@ -152,10 +179,9 @@ def _native_plan_supported_impl(self) -> bool: if not database_name or database_name == UNKNOWN_DATABASE or '.' in database_name: return False if self.table.options.query_auth_enabled \ - or self.table.options.merge_engine() == 'first-row' \ - or self.table.current_branch() != 'main': + or self.table.options.merge_engine() == 'first-row': return False - # Empty trimmed PK (PK == partition key): native skips merge -> duplicate/stale rows. + # Rust rejects schemas whose primary keys are all partition keys. if getattr(self.table, 'is_primary_key_table', False) \ and not self.table.trimmed_primary_keys: return False @@ -164,6 +190,9 @@ def _native_plan_supported_impl(self) -> bool: if self.table.bucket_mode() in (BucketMode.HASH_DYNAMIC, BucketMode.CROSS_PARTITION): return False options = self.table.options.options + if (options.contains_key(CoreOptions.SCAN_WATERMARK.key()) + and not native_version_at_least(0, 4)): + return False if (any(options.contains_key(key) for key in _NATIVE_FAMILY_SEARCH_MODE_OPTIONS)): from pypaimon.read.native_plan import native_family_search_modes_available @@ -191,7 +220,8 @@ def _native_plan_supported_impl(self) -> bool: if any(options.contains_key(k) for k in unsupported_scan_keys) \ or options.contains_key('scan.version'): return False - return not options.contains(CoreOptions.INCREMENTAL_BETWEEN_TIMESTAMP) + return (not options.contains(CoreOptions.INCREMENTAL_BETWEEN_TIMESTAMP) + or native_method_available('ReadBuilder', 'new_incremental_scan')) def _native_global_index_result_supported(self) -> bool: result = self.file_scanner._global_index_result @@ -205,7 +235,10 @@ def _native_global_index_result_supported(self) -> bool: return (isinstance(result, GlobalIndexResult) and not isinstance(result, ScoredGlobalIndexResult)) - def _native_global_index_row_ranges(self) -> Optional[List[Tuple[int, int]]]: + def _native_row_ranges(self) -> Optional[List[Tuple[int, int]]]: + row_ranges = getattr(self.file_scanner, '_row_ranges', None) + if row_ranges is not None: + return [(range_.from_, range_.to) for range_ in row_ranges] result = self.file_scanner._global_index_result if result is None: return None @@ -216,13 +249,28 @@ def _try_native_plan(self) -> Optional[Plan]: """Plan via pypaimon_rust, then drop partitions the predicate rejects. Predicate and limit are pushed into Rust planning and are still enforced - by the reader. Empty unrestricted scans fall back to preserve snapshot - metadata; explicit empty row ranges are a terminal empty result. + by the reader. Snapshot metadata is preserved even when pruning removes + every split. """ from pypaimon.read.native_plan import native_plan try: - row_ranges = self._native_global_index_row_ranges() + fs = self.file_scanner + has_distribution = (fs.idx_of_this_subtask is not None + or fs.start_pos_of_this_subtask is not None) + extra_options = {} + if self.table.options.options.contains(CoreOptions.INCREMENTAL_BETWEEN_TIMESTAMP): + if self._incremental_snapshot_range is None: + return Plan([]) + extra_options['incremental_range'] = self._incremental_snapshot_range + if has_distribution and fs.data_evolution: + if fs.idx_of_this_subtask is not None: + extra_options['row_position_shard'] = ( + fs.idx_of_this_subtask, fs.number_of_para_subtasks) + else: + extra_options['row_position_slice'] = ( + fs.start_pos_of_this_subtask, fs.end_pos_of_this_subtask) + row_ranges = self._native_row_ranges() native_predicate = self.predicate if self.partition_predicate is not None: native_predicate = PredicateBuilder.and_predicates([ @@ -231,24 +279,37 @@ def _try_native_plan(self) -> Optional[Plan]: self.file_scanner.partition_key_predicate, ) if predicate is not None ]) - splits = native_plan( + plan = native_plan( self.table, predicate=native_predicate, - limit=self.limit, + limit=None if has_distribution else self.limit, projection=( [field.name for field in self._read_type] if self._read_type is not None else None), row_ranges=row_ranges, + **extra_options, ) - if not splits: - return Plan([]) if row_ranges is not None else None - snapshot_id = splits[0].snapshot_id + splits = plan.splits() partition_predicate = self.file_scanner.partition_key_predicate if partition_predicate is not None: splits = [s for s in splits if getattr(s, 'partition', None) is None or partition_predicate.test(s.partition)] - return Plan(splits, snapshot_id=snapshot_id) + if has_distribution: + if self.table.is_primary_key_table: + splits = [s for s in splits + if s.bucket % fs.number_of_para_subtasks == fs.idx_of_this_subtask] + elif not fs.data_evolution: + from pypaimon.read.scan_distribution import shard_range, slice_append_splits + if fs.idx_of_this_subtask is not None: + start, end = shard_range( + sum(s.row_count for s in splits), + fs.idx_of_this_subtask, fs.number_of_para_subtasks) + else: + start, end = fs.start_pos_of_this_subtask, fs.end_pos_of_this_subtask + splits = slice_append_splits(splits, start, end) + splits = fs._apply_push_down_limit(splits) + return Plan(splits, snapshot_id=plan.snapshot_id) except Exception as e: # Any native construction/planning/pruning failure -> fall back. logger.warning( @@ -267,9 +328,8 @@ def scan_with_stats(self) -> Tuple[Plan, Optional[ScanStats]]: """Run :meth:`plan` while recording manifest / pruning counters. Only used by :meth:`ReadBuilder.explain`; the regular read path - keeps going through :meth:`plan`. Native planning is not tracked, so - stats is None on the native path -- explain reflects the real plan and - marks the pruning funnel as untracked. + keeps going through :meth:`plan`. Native plans return stats=None; + explain reports their split metadata without pruning counters. """ auth_result = self.__auth_query() if (auth_result is None and self.table.options.native_plan_enabled() @@ -283,6 +343,7 @@ def scan_with_stats(self) -> Tuple[Plan, Optional[ScanStats]]: return wrap_plan_with_auth(auth_result, plan), stats def _create_file_scanner(self) -> FileScanner: + self._incremental_snapshot_range = None options = self.table.options.options snapshot_manager = self.table.snapshot_manager() manifest_list_manager = ManifestListManager(self.table) @@ -305,6 +366,12 @@ def _create_file_scanner(self) -> FileScanner: raise ValueError( "The incremental-between-timestamp must specific start(exclusive) and end timestamp. But is: " + options.get(CoreOptions.INCREMENTAL_BETWEEN_TIMESTAMP)) + start_timestamp = int(ts[0]) + end_timestamp = int(ts[1]) + if start_timestamp >= end_timestamp: + raise ValueError( + "Ending timestamp %s must be greater than starting timestamp %s." + % (end_timestamp, start_timestamp)) earliest_snapshot = snapshot_manager.try_get_earliest_snapshot() latest_snapshot = snapshot_manager.get_latest_snapshot() if earliest_snapshot is None or latest_snapshot is None: @@ -313,12 +380,7 @@ def _create_file_scanner(self) -> FileScanner: lambda: ([], None), partition_predicate=self.partition_predicate, ) - start_timestamp = int(ts[0]) - end_timestamp = int(ts[1]) - if start_timestamp >= end_timestamp: - raise ValueError( - "Ending timestamp %s should be >= starting timestamp %s." % (end_timestamp, start_timestamp)) - if (start_timestamp == end_timestamp or start_timestamp > latest_snapshot.time_millis + if (start_timestamp > latest_snapshot.time_millis or end_timestamp < earliest_snapshot.time_millis): return FileScanner( self.table, @@ -338,6 +400,7 @@ def _create_file_scanner(self) -> FileScanner: end_snapshot = snapshot_manager.earlier_or_equal_time_mills(end_timestamp) latest_snapshot = snapshot_manager.get_latest_snapshot() end_id = end_snapshot.id if end_snapshot else (latest_snapshot.id if latest_snapshot else -1) + self._incremental_snapshot_range = (start_id, end_id) def incremental_manifest(): snapshots_in_range = [] diff --git a/paimon-python/pypaimon/table/file_store_table.py b/paimon-python/pypaimon/table/file_store_table.py index d6700c57cbca..32f07d8534fb 100644 --- a/paimon-python/pypaimon/table/file_store_table.py +++ b/paimon-python/pypaimon/table/file_store_table.py @@ -394,7 +394,7 @@ def path_factory(self) -> 'FileStorePathFactory': external_paths=external_paths, external_path_strategy=self.options.data_file_external_paths_strategy(), external_path_weights=self.options.data_file_external_paths_weights(), - index_file_in_data_file_dir=False, + index_file_in_data_file_dir=self.options.index_file_in_data_file_dir(), global_index_external_path=self.options.global_index_external_path(), ) diff --git a/paimon-python/pypaimon/tests/data_evolution_split_generator_test.py b/paimon-python/pypaimon/tests/data_evolution_split_generator_test.py index e36f6c8bcba6..4d2c3fb118df 100644 --- a/paimon-python/pypaimon/tests/data_evolution_split_generator_test.py +++ b/paimon-python/pypaimon/tests/data_evolution_split_generator_test.py @@ -169,6 +169,13 @@ def test_preserves_manifest_order_within_row_id_group(self): [file.file_name for file in splits[0].files], ) + def test_disjoint_slice_and_row_ranges_are_terminal_empty(self): + generator = DataEvolutionSplitGenerator( + self._Table(), target_split_size=1024, open_file_cost=0, + row_ranges=[Range(7, 8)], + ).with_slice(0, 2) + self.assertEqual(generator.create_splits([self._entry('base.parquet', 1)]), []) + def test_slice_and_shard_preserve_blob_manifest_order(self): entries = [ self._entry('a.blob', 1), diff --git a/paimon-python/pypaimon/tests/deletion_vector_path_test.py b/paimon-python/pypaimon/tests/deletion_vector_path_test.py new file mode 100644 index 000000000000..006b21c36cbf --- /dev/null +++ b/paimon-python/pypaimon/tests/deletion_vector_path_test.py @@ -0,0 +1,301 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Deletion-vector locations must survive writes, further deletes and time travel.""" + +from pathlib import Path +from dataclasses import replace +from unittest.mock import patch + +import pyarrow as pa +import pytest + +from pypaimon import CatalogFactory, Schema +from pypaimon.manifest.index_manifest_file import IndexManifestFile +from pypaimon.read.native_plan import native_plan, native_version_at_least +from pypaimon.write.file_store_commit import _abort_commit_messages +from pypaimon.write.commit_message import CommitMessage +from pypaimon.write.table_delete import TableDeleteByRowId + + +_PLANNERS = ['python', pytest.param('native', marks=pytest.mark.skipif( + not native_version_at_least(0, 4, 0), + reason='pypaimon-rust>=0.4.0 required for native DV paths'))] + + +def _table(tmp_path, layout, first_partition='a', partition_type=None): + catalog = CatalogFactory.create({'warehouse': str(tmp_path / 'warehouse')}) + catalog.create_database('db', True) + options = { + 'bucket': '-1', 'data-evolution.enabled': 'true', 'row-tracking.enabled': 'true', + 'deletion-vectors.enabled': 'true', 'scan.native-plan.enabled': 'false', + 'index-file-in-data-file-dir': str(layout.startswith('bucket')).lower(), + } + if layout == 'bucket-external': + options.update({ + 'data-file.external-paths': (tmp_path / 'external-data').as_uri(), + 'data-file.external-paths.strategy': 'round-robin', + 'global-index.external-path': (tmp_path / 'unused-global-index').as_uri(), + }) + elif layout == 'global-external': + options['global-index.external-path'] = (tmp_path / 'external-index').as_uri() + second_partition = 'b' if partition_type is None else False if pa.types.is_boolean(partition_type) else 2.0 + schema = pa.schema([('p', pa.string() if partition_type is None else partition_type), ('k', pa.int64())]) + catalog.create_table('db.t', Schema.from_pyarrow_schema( + schema, partition_keys=['p'], options=options), False) + table = catalog.get_table('db.t') + builder = table.new_batch_write_builder() + writer, commit = builder.new_write(), builder.new_commit() + try: + writer.write_arrow(pa.Table.from_pydict({ + 'p': [first_partition, first_partition, second_partition, second_partition], 'k': [0, 1, 2, 3]}, schema)) + commit.commit(writer.prepare_commit()) + finally: + writer.close() + commit.close() + return table + + +def _delete(table, row_ids): + builder = table.new_batch_write_builder() + messages = builder.new_update().delete_by_row_id(row_ids) + commit = builder.new_commit() + try: + commit.commit(messages) + finally: + commit.close() + return messages + + +def _entries(table, snapshot_id): + snapshot = table.snapshot_manager().get_snapshot_by_id(snapshot_id) + return IndexManifestFile(table).read(snapshot.index_manifest) + + +def _read(table, planner, snapshot_id, expected): + table = table.copy({'scan.snapshot-id': str(snapshot_id)}) + builder = table.new_read_builder() + # Direct invocation makes a native planning error fail rather than fall back. + plan = native_plan(table) if planner == 'native' else builder.new_scan().plan() + assert plan.snapshot_id == snapshot_id + assert sorted(builder.new_read().to_arrow(plan.splits()).column('k').to_pylist()) == expected + return plan + + +@pytest.mark.parametrize('planner', _PLANNERS) +@pytest.mark.parametrize('layout', ['table', 'bucket', 'bucket-external', 'global-external']) +def test_delete_paths_preserve_repeated_deletes_and_historical_reads(tmp_path, planner, layout): + table = _table(tmp_path, layout) + _delete(table, [0, 2]) + entries = _entries(table, 2) + assert len(entries) == 2 + for entry in entries: + file = entry.index_file + bucket = table.path_factory().relative_bucket_path(tuple(entry.partition.values), entry.bucket) + if layout == 'bucket': + expected = Path(table.table_path) / bucket / file.file_name + elif layout == 'bucket-external': + expected = tmp_path / 'external-data' / bucket / file.file_name + elif layout == 'global-external': + expected = tmp_path / 'external-index' / file.file_name + else: + expected = Path(table.table_path) / 'index' / file.file_name + assert expected.is_file() + assert file.external_path == ('file://' + str(expected) if 'external' in layout else None) + # Obsolete locations with the same name must never shadow canonical or + # explicit paths. Invalid bytes make a wrong-path read fail observably. + if layout != 'table': + decoy = Path(table.table_path) / 'index' / file.file_name + decoy.parent.mkdir(parents=True, exist_ok=True) + decoy.write_bytes(b'not a deletion vector') + if layout == 'bucket-external': + decoy = Path(table.table_path) / bucket / file.file_name + decoy.parent.mkdir(parents=True, exist_ok=True) + decoy.write_bytes(b'not an external deletion vector') + plan = _read(table, planner, 2, [1, 3]) + assert all(table.file_io.exists(dv.dv_index_path) for split in plan.splits() + for dv in split.data_deletion_files if dv is not None) + _delete(table, [1]) + _read(table, planner, 3, [3]) + _read(table, planner, 2, [1, 3]) + _read(table, planner, 1, [0, 1, 2, 3]) + + +@pytest.mark.parametrize('planner', _PLANNERS) +def test_legacy_python_index_directory_remains_readable_and_new_deletes_use_bucket(tmp_path, planner): + table = _table(tmp_path, 'bucket') + # Old Python writers ignored the option and placed the index under table/index. + legacy_factory = table.path_factory() + legacy_factory.index_file_in_data_file_dir = False + with patch.object(table, 'path_factory', return_value=legacy_factory): + _delete(table, [0, 2]) + old_paths = [Path(table.table_path) / 'index' / entry.index_file.file_name for entry in _entries(table, 2)] + assert all(path.is_file() for path in old_paths) + _read(table, planner, 2, [1, 3]) + _delete(table, [1]) + _read(table, planner, 3, [3]) + _read(table, planner, 2, [1, 3]) + for entry in _entries(table, 3): + if entry.partition.values == ['a']: + assert (Path(table.path_factory().bucket_path(('a',), entry.bucket)) / + entry.index_file.file_name).is_file() + assert all(path.is_file() for path in old_paths) + + +@pytest.mark.parametrize('layout', ['bucket', 'bucket-external', 'global-external']) +def test_abort_removes_uncommitted_dv_from_its_actual_directory(tmp_path, layout): + table = _table(tmp_path, layout) + _delete(table, [0]) + builder = table.new_batch_write_builder() + messages = builder.new_update().delete_by_row_id([1]) + uncommitted = [entry for message in messages for entry in message.index_adds] + assert uncommitted + files_before = set(tmp_path.rglob('index-*')) + _abort_commit_messages(table, messages) + files_after = set(tmp_path.rglob('index-*')) + removed = {path.name for path in files_before - files_after} + assert removed == {entry.index_file.file_name for entry in uncommitted} + _read(table, 'python', 2, [1, 2, 3]) + + +@pytest.mark.parametrize('planner', _PLANNERS) +def test_missing_explicit_dv_is_not_replaced_by_a_local_copy(tmp_path, planner): + table = _table(tmp_path, 'bucket-external') + _delete(table, [0]) + entry = _entries(table, 2)[0] + file = entry.index_file + with table.file_io.new_input_stream(file.external_path) as stream: + data = stream.read() + factory = table.path_factory() + for directory in [factory.index_path(), factory.bucket_path(tuple(entry.partition.values), entry.bucket)]: + with table.file_io.new_output_stream(directory + '/' + file.file_name) as stream: + stream.write(data) + table.file_io.delete_quietly(file.external_path) + with pytest.raises(FileNotFoundError): + _read(table, planner, 2, [1, 2, 3]) + + +@pytest.mark.parametrize('planner', _PLANNERS) +@pytest.mark.parametrize('partition', ['a/b', 'a%2Fb', 'a=b', 'a#b', 'a b', '中文']) +def test_bucket_dv_in_partition_requiring_path_escaping(tmp_path, planner, partition): + table = _table(tmp_path, 'bucket', partition) + _delete(table, [0]) + file = _entries(table, 2)[0].index_file + if partition in ['a b', '中文']: + assert file.external_path is None + else: + assert file.external_path is not None + assert table.file_io.exists(file.external_path) + _read(table, planner, 2, [1, 2, 3]) + + +@pytest.mark.parametrize('planner', _PLANNERS) +@pytest.mark.parametrize('partition_type,value', [(pa.bool_(), True), (pa.float32(), 0.1), (pa.float64(), 0.1)], + ids=['BOOLEAN', 'FLOAT', 'DOUBLE']) +def test_bucket_dv_preserves_python_typed_partition_directory(tmp_path, planner, partition_type, value): + table = _table(tmp_path, 'bucket', value, partition_type) + _delete(table, [0]) + file = _entries(table, 2)[0].index_file + assert file.external_path is not None + assert table.file_io.exists(file.external_path) + effective_planner = planner + if planner == 'native' and pa.types.is_floating(partition_type): + # Rust intentionally rejects floating partition formatting until it can + # reproduce Java Float/Double.toString, including boundary values. + with pytest.raises(NotImplementedError, match='type is not supported as partition key'): + native_plan(table) + native_table = table.copy({'scan.native-plan.enabled': 'true'}) + scan = native_table.new_read_builder().new_scan() + with patch('pypaimon.read.native_plan.native_plan', wraps=native_plan) as native_call: + with patch.object(scan.file_scanner, 'scan', wraps=scan.file_scanner.scan) as fallback: + plan = scan.plan() + assert native_call.call_count == 1 + assert fallback.call_count == 1 + result = native_table.new_read_builder().new_read().to_arrow(plan.splits()) + assert sorted(result.column('k').to_pylist()) == [1, 2, 3] + effective_planner = 'python' + _read(table, effective_planner, 2, [1, 2, 3]) + _delete(table, [1]) + _read(table, effective_planner, 3, [2, 3]) + _read(table, effective_planner, 2, [1, 2, 3]) + + +@pytest.mark.parametrize('planner', _PLANNERS) +@pytest.mark.parametrize('value,partition_type,canonical_name', [('a/b', None, 'a%2Fb'), (True, pa.bool_(), 'true')]) +def test_java_canonical_bucket_dv_without_external_path(tmp_path, planner, value, partition_type, canonical_name): + table = _table(tmp_path, 'bucket', value, partition_type) + _delete(table, [0]) + old = _entries(table, 2)[0] + canonical_path = str(Path(table.table_path) / ('p=' + canonical_name) / ('bucket-' + str(old.bucket)) / + old.index_file.file_name) + # Model the persisted layout produced by Java: the bucket path is canonical + # and no explicit index location is needed in its manifest metadata. + with table.file_io.new_input_stream(old.index_file.external_path) as stream: + data = stream.read() + with table.file_io.new_output_stream(canonical_path) as stream: + stream.write(data) + canonical = replace(old, index_file=replace(old.index_file, external_path=None)) + commit = table.new_batch_write_builder().new_commit() + try: + commit.commit([CommitMessage( + partition=tuple(old.partition.values), bucket=old.bucket, new_files=[], + check_from_snapshot=2, index_adds=[canonical], index_deletes=[replace(old, kind=1)])]) + finally: + commit.close() + # Case-insensitive local filesystems consider p=True and p=true identical. + if not Path(old.index_file.external_path).samefile(canonical_path): + table.file_io.delete_quietly(old.index_file.external_path) + plan = _read(table, planner, 3, [1, 2, 3]) + paths = [dv.dv_index_path for split in plan.splits() + for dv in split.data_deletion_files or [] if dv is not None] + assert paths == [canonical_path] + _delete(table, [1]) + _read(table, planner, 4, [2, 3]) + _read(table, planner, 3, [1, 2, 3]) + + +@pytest.mark.parametrize('planner', _PLANNERS) +def test_multiple_dv_index_files_in_one_bucket_keep_all_deletions(tmp_path, planner): + table = _table(tmp_path, 'bucket') + builder = table.new_batch_write_builder() + writer, commit = builder.new_write(), builder.new_commit() + try: + writer.write_arrow(pa.table({'p': ['a', 'a'], 'k': [4, 5]})) + commit.commit(writer.prepare_commit()) + finally: + writer.close() + commit.close() + _delete(table, [0, 4]) + old = _entries(table, 3)[0] + deleter = TableDeleteByRowId(table) + _, vectors = deleter._read_existing_deletion_vectors(old.partition, old.bucket, 3) + assert len(vectors) == 2 + # Java rolls DV index files by target size, so one bucket can legitimately + # have more than one live DV index entry for different data files. + indexes = [deleter._write_deletion_vector_index(old.partition, old.bucket, {name: vector}) + for name, vector in vectors.items()] + commit = table.new_batch_write_builder().new_commit() + try: + commit.commit([CommitMessage( + partition=tuple(old.partition.values), bucket=old.bucket, new_files=[], + check_from_snapshot=3, index_adds=indexes, index_deletes=[replace(old, kind=1)])]) + finally: + commit.close() + _read(table, planner, 4, [1, 2, 3, 5]) + _delete(table, [1]) + _read(table, planner, 5, [2, 3, 5]) + _read(table, planner, 4, [1, 2, 3, 5]) diff --git a/paimon-python/pypaimon/tests/interval_partition_test.py b/paimon-python/pypaimon/tests/interval_partition_test.py new file mode 100644 index 000000000000..2043ff12ef22 --- /dev/null +++ b/paimon-python/pypaimon/tests/interval_partition_test.py @@ -0,0 +1,103 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Floating-point key bounds preserve overlapping split groups.""" + +from decimal import Decimal, InvalidOperation +from types import SimpleNamespace + +import pytest + +from pypaimon.common.options.core_options import CoreOptions +from pypaimon.common.options.options import Options +from pypaimon.manifest.schema.data_file_meta import DataFileMeta +from pypaimon.manifest.schema.manifest_entry import ManifestEntry +from pypaimon.manifest.schema.simple_stats import SimpleStats +from pypaimon.read.interval_partition import IntervalPartition, default_key_comparator +from pypaimon.read.scanner.primary_key_table_split_generator import PrimaryKeyTableSplitGenerator +from pypaimon.schema.data_types import AtomicType, DataField +from pypaimon.table.row.generic_row import GenericRow, GenericRowDeserializer, GenericRowSerializer + + +def _key_fields(type_name): + return [DataField(0, 'f', AtomicType(type_name)), DataField(1, 'i', AtomicType('INT'))] + + +def _file(name, minimum, maximum, fields): + def key(values): + row = GenericRow(list(values), fields) + return GenericRowDeserializer.from_bytes(GenericRowSerializer.to_bytes(row), fields) + + return DataFileMeta( + file_name=name, file_size=100, row_count=3, + min_key=key(minimum), max_key=key(maximum), + key_stats=SimpleStats.empty_stats(), value_stats=SimpleStats.empty_stats(), + min_sequence_number=0, max_sequence_number=0, schema_id=0, level=0, extra_files=[]) + + +@pytest.mark.parametrize('type_name', ['FLOAT', 'DOUBLE']) +@pytest.mark.parametrize('minimum,maximum,point', [ + ((1.0, 0), (float('nan'), 10), (2.0, 100)), + ((-0.0, 0), (0.0, 10), (-0.0, 100)), +]) +def test_floating_key_ranges_keep_versions_in_one_split(type_name, minimum, maximum, point): + fields = _key_fields(type_name) + files = [_file('broad', minimum, maximum, fields), _file('point', point, point, fields)] + sections = IntervalPartition(files).partition() + assert len(sections) == 1 + assert sorted([f.file_name for f in run.files] for run in sections[0]) == [['broad'], ['point']] + + table = SimpleNamespace(table_path='/tmp/interval-test', options=CoreOptions(Options({}))) + entries = [ManifestEntry(0, GenericRow([], []), 0, 1, file) for file in files] + splits = PrimaryKeyTableSplitGenerator(table, 1, 1).create_splits(entries) + assert len(splits) == 1 + assert sorted(file.file_name for file in splits[0].files) == ['broad', 'point'] + assert not splits[0].raw_convertible + + +@pytest.mark.parametrize('type_name', ['FLOAT', 'DOUBLE']) +def test_nan_boundary_keeps_disjoint_files_in_separate_runs(type_name): + fields = _key_fields(type_name) + files = [ + _file('finite', (1.0, 0), (2.0, 0), fields), + _file('nan', (float('nan'), 100), (float('nan'), 100), fields), + ] + sections = IntervalPartition(files).partition() + # NaN metadata uses the same conservative grouping as the native planner. + assert len(sections) == 1 + assert sorted([f.file_name for f in run.files] for run in sections[0]) == [['finite'], ['nan']] + + +def test_decimal_keys_keep_numeric_equality_and_invalid_nan_errors(): + fields = _key_fields('DECIMAL(10, 2)') + left = GenericRow([Decimal('-0'), 1], fields) + right = GenericRow([Decimal('0'), 1], fields) + assert default_key_comparator(left, right) == 0 + invalid = GenericRow([Decimal('NaN'), 1], fields) + with pytest.raises(InvalidOperation): + default_key_comparator(invalid, right) + + +@pytest.mark.parametrize('type_name', ['FLOAT', 'DOUBLE']) +def test_disjoint_finite_ranges_still_form_separate_sections(type_name): + fields = _key_fields(type_name) + sections = IntervalPartition([ + _file('high', (3.0, 0), (4.0, 0), fields), + _file('low', (1.0, 0), (2.0, 0), fields), + ]).partition() + assert [[f.file_name for run in section for f in run.files] for section in sections] == [ + ['low'], ['high']] diff --git a/paimon-python/pypaimon/tests/native_plan_capabilities_test.py b/paimon-python/pypaimon/tests/native_plan_capabilities_test.py new file mode 100644 index 000000000000..36e2eb641cc2 --- /dev/null +++ b/paimon-python/pypaimon/tests/native_plan_capabilities_test.py @@ -0,0 +1,352 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Native/Python planner parity on persisted snapshots and data files.""" + +import json +import tempfile +import unittest +from dataclasses import replace +from unittest.mock import patch + +import pyarrow as pa + +from pypaimon import CatalogFactory, Schema +from pypaimon.common.identifier import Identifier +from pypaimon.deletionvectors.bitmap_deletion_vector import BitmapDeletionVector +from pypaimon.manifest.index_manifest_file import IndexManifestFile +from pypaimon.read.native_plan import ( + native_method_available, + native_runtime_available, + native_version_at_least, +) +from pypaimon.schema.data_types import AtomicType +from pypaimon.schema.schema_change import SchemaChange +from pypaimon.table.row.generic_row import GenericRow +from pypaimon.utils.range import Range +from pypaimon.write.commit_message import CommitMessage +from pypaimon.write.table_delete import TableDeleteByRowId + + +@unittest.skipUnless(native_runtime_available(), + "pypaimon_rust split-planning API not installed") +class NativePlanCapabilitiesTest(unittest.TestCase): + + def setUp(self): + warehouse = tempfile.TemporaryDirectory(prefix='native_capabilities_') + self.addCleanup(warehouse.cleanup) + self.catalog = CatalogFactory.create({'warehouse': warehouse.name}) + self.catalog.create_database('default', True) + self.schema = pa.schema([('k', pa.int64()), ('v', pa.string())]) + + def _create(self, name, options=None, schema=None, primary_keys=None): + identifier = 'default.' + name + self.catalog.create_table(identifier, Schema.from_pyarrow_schema( + self.schema if schema is None else schema, + options=options, primary_keys=primary_keys), False) + return self.catalog.get_table(identifier) + + def _write(self, table, rows, schema=None): + builder = table.new_batch_write_builder() + writer, commit = builder.new_write(), builder.new_commit() + try: + writer.write_arrow(pa.Table.from_pylist( + rows, schema=self.schema if schema is None else schema)) + commit.commit(writer.prepare_commit()) + finally: + writer.close() + commit.close() + + @staticmethod + def _de_options(): + return { + 'data-evolution.enabled': 'true', + 'row-tracking.enabled': 'true', + } + + @staticmethod + def _delete(table, row_ids): + builder = table.new_batch_write_builder() + messages = builder.new_update().delete_by_row_id(row_ids) + commit = builder.new_commit() + try: + commit.commit(messages) + finally: + commit.close() + + @staticmethod + def _set_watermark(table, snapshot_id, watermark): + # PyPaimon's batch writer can inherit a watermark but cannot set one. + # Add the field to an otherwise real committed snapshot, as Flink does. + path = table.snapshot_manager().get_snapshot_path(snapshot_id) + snapshot = json.loads(table.file_io.read_file_utf8(path)) + snapshot['watermark'] = watermark + table.file_io.write_file(path, json.dumps(snapshot), overwrite=True) + + @staticmethod + def _split_metadata(plan): + result = {} + for split in plan.splits(): + ranges = getattr(split, 'row_ranges', lambda: None)() + ranges = None if ranges is None else [ + (range_.from_, range_.to) for range_ in ranges] + deletions = split.data_deletion_files or [None] * len(split.files) + for data_file, deletion in zip(split.files, deletions): + result[data_file.file_name] = (ranges, deletion) + return result + + def _assert_parity(self, table, expected_rows, snapshot_id, + row_ranges=None, predicate=None, projection=None): + plans = [] + for native in (False, True): + read_table = table.copy({ + 'scan.native-plan.enabled': str(native).lower()}) + builder = read_table.new_read_builder() + if predicate is not None: + builder.with_filter(predicate) + if projection is not None: + builder.with_projection(projection) + scan = builder.new_scan() + if row_ranges is not None: + scan.with_row_ranges(row_ranges) + if native: + # Make an implicit Python fallback fail the test, including + # empty plans for which a row comparison alone proves nothing. + with patch.object(scan.file_scanner, 'scan', side_effect=AssertionError( + 'native planner fell back')): + plan = scan.plan() + else: + plan = scan.plan() + rows = builder.new_read().to_arrow(plan.splits()).to_pylist() + self.assertEqual(sorted(rows, key=lambda row: row['k']), expected_rows) + self.assertEqual(plan.snapshot_id, snapshot_id) + plans.append(plan) + self.assertEqual(self._split_metadata(plans[0]), + self._split_metadata(plans[1])) + return plans[1] + + @unittest.skipUnless( + native_method_available('ReadBuilder', 'with_row_ranges') + and native_method_available('Plan', 'snapshot_id'), + 'pypaimon_rust row ranges and plan snapshot metadata required') + def test_explicit_row_ranges_preserve_selection_and_snapshot(self): + table = self._create('ranges', self._de_options()) + rows = [{'k': k, 'v': 'v%d' % k} for k in range(6)] + self._write(table, rows[:3]) + self._write(table, rows[3:]) + cases = [ + ([], []), + ([Range(20, 25)], []), + ([Range(0, 0), Range(2, 4)], [rows[k] for k in (0, 2, 3, 4)]), + ([Range(3, 4), Range(1, 3)], rows[1:5]), + ] + for ranges, expected in cases: + with self.subTest(ranges=ranges): + self._assert_parity(table, expected, 2, row_ranges=ranges) + + @unittest.skipUnless(native_method_available('ReadBuilder', 'with_row_ranges'), + 'pypaimon_rust row-range API required') + def test_explicit_row_ranges_combine_with_filter_and_projection(self): + table = self._create('filtered_ranges', self._de_options()) + self._write(table, [{'k': k, 'v': 'v%d' % k} for k in range(6)]) + predicate = table.new_read_builder().new_predicate_builder().greater_than('k', 2) + self._assert_parity( + table, [{'k': 3}, {'k': 4}], 1, + row_ranges=[Range(1, 4)], predicate=predicate, projection=['k']) + self._assert_parity( + table, [], 1, row_ranges=[Range(0, 1)], predicate=predicate) + + @unittest.skipUnless(native_version_at_least(0, 4), + 'pypaimon_rust 0.4 watermark support required') + def test_watermark_selects_first_matching_snapshot(self): + table = self._create('watermarks') + rows = [{'k': k, 'v': 'v%d' % k} for k in range(4)] + for row in rows: + self._write(table, [row]) + for snapshot_id, watermark in ((2, 100), (3, 100), (4, 200)): + self._set_watermark(table, snapshot_id, watermark) + for watermark, snapshot_id in ((0, 2), (100, 2), (101, 4), (200, 4)): + with self.subTest(watermark=watermark): + self._assert_parity( + table.copy({'scan.watermark': str(watermark)}), + rows[:snapshot_id], snapshot_id) + + @unittest.skipUnless(native_version_at_least(0, 4), + 'pypaimon_rust 0.4 watermark support required') + def test_watermark_uses_historical_schema(self): + table = self._create('watermark_schema') + self._write(table, [{'k': 1, 'v': 'old'}]) + self._set_watermark(table, 1, 100) + self.catalog.alter_table(table.identifier, [ + SchemaChange.add_column('added', AtomicType('STRING'))]) + table = self.catalog.get_table(table.identifier) + self._write(table, [{'k': 2, 'v': 'new', 'added': 'new field'}], + self.schema.append(pa.field('added', pa.string()))) + self._set_watermark(table, 2, 200) + + historical = table.copy({'scan.watermark': '100'}) + self.assertEqual(historical.field_names, ['k', 'v']) + self._assert_parity(historical, [{'k': 1, 'v': 'old'}], 1) + + @unittest.skipUnless(native_version_at_least(0, 4), + 'pypaimon_rust 0.4 watermark support required') + def test_unmatched_watermark_fails_in_both_planners(self): + from pypaimon_rust.datafusion import PaimonCatalog + + table = self._create('unmatched_watermark') + self._write(table, [{'k': 1, 'v': 'a'}]) + rust_table = PaimonCatalog({ + 'warehouse': self.catalog.warehouse, + }).get_table(table.identifier.get_full_name()) + for watermark in (None, 100): + with self.subTest(stored_watermark=watermark): + self._set_watermark(table, 1, watermark) + with self.assertRaisesRegex(ValueError, 'watermark'): + table.copy({ + 'scan.watermark': '101', + 'scan.native-plan.enabled': 'false', + }).new_read_builder().new_scan().plan() + with self.assertRaisesRegex(Exception, '(?i)watermark'): + rust_table.new_read_builder({ + 'scan.watermark': '101'}).new_scan().plan() + + @unittest.skipUnless(native_method_available('Table', 'branch'), + 'pypaimon_rust branch API required') + def test_branch_does_not_read_subsequent_main_commits(self): + table = self._create('branched') + self._write(table, [{'k': 1, 'v': 'shared'}]) + table.create_tag('base') + self.catalog.create_branch(table.identifier, 'test', tag_name='base') + branch_id = Identifier('default', 'branched', branch='test') + branch = self.catalog.get_table(branch_id) + self._write(branch, [{'k': 2, 'v': 'branch'}]) + self._write(table, [{'k': 3, 'v': 'main'}]) + self._write(table, [{'k': 4, 'v': 'later main'}]) + + self._assert_parity(branch, [ + {'k': 1, 'v': 'shared'}, {'k': 2, 'v': 'branch'}], 2) + self._assert_parity(table, [ + {'k': 1, 'v': 'shared'}, {'k': 3, 'v': 'main'}, + {'k': 4, 'v': 'later main'}], 3) + self._assert_parity(branch.copy({'scan.snapshot-id': '1'}), + [{'k': 1, 'v': 'shared'}], 1) + + @unittest.skipUnless(native_version_at_least(0, 4, 0), + 'pypaimon-rust>=0.4.0 required for native DV scans') + def test_deletion_vectors_preserve_deletes_and_historical_snapshots(self): + options = self._de_options() + options['deletion-vectors.enabled'] = 'true' + table = self._create('deletions', options) + rows = [{'k': k, 'v': 'v%d' % k} for k in range(6)] + self._write(table, rows) + self._delete(table, [1, 4]) + self._delete(table, [2]) + + current = self._assert_parity( + table, [rows[k] for k in (0, 3, 5)], 3) + deletion_files = [deletion + for split in current.splits() + for deletion in (split.data_deletion_files or []) + if deletion is not None] + self.assertEqual([deletion.cardinality for deletion in deletion_files], [3]) + self._assert_parity(table.copy({'scan.snapshot-id': '1'}), rows, 1) + self._assert_parity( + table.copy({'scan.snapshot-id': '2'}), + [rows[k] for k in (0, 2, 3, 5)], 2) + self._assert_parity( + table, [rows[k] for k in (0, 3)], 3, row_ranges=[Range(0, 4)]) + + @unittest.skipUnless(native_version_at_least(0, 4, 0), + 'pypaimon-rust>=0.4.0 required for native DV scans') + def test_primary_key_deletion_vectors_preserve_compacted_rows(self): + table = self._create('pk_deletions', { + 'bucket': '1', 'deletion-vectors.enabled': 'true', + }, primary_keys=['k']) + rows = [{'k': k, 'v': 'v%d' % k} for k in range(4)] + builder = table.new_batch_write_builder() + writer, commit = builder.new_write(), builder.new_commit() + try: + writer.write_arrow(pa.Table.from_pylist(rows, schema=self.schema)) + messages = writer.prepare_commit() + # A single sorted PK run needs no rewrite during compaction; + # promote its metadata because DV scans intentionally skip L0. + for message in messages: + message.new_files = [replace(file, level=1) + for file in message.new_files] + commit.commit(messages) + finally: + writer.close() + commit.close() + + self._assert_parity(table, rows, 1) + data_file = messages[0].new_files[0] + vector = BitmapDeletionVector() + vector.delete(1) + vector.delete(3) + # Python's delete API is DE-only. Use its production bitmap writer to + # create the same index payload for this compacted primary-key fixture. + index_entry = TableDeleteByRowId(table)._write_deletion_vector_index( + GenericRow([], []), 0, {data_file.file_name: vector}) + commit = table.new_batch_write_builder().new_commit() + try: + commit.commit([CommitMessage( + partition=(), bucket=0, new_files=[], index_adds=[index_entry])]) + finally: + commit.close() + + plan = self._assert_parity(table, [rows[0], rows[2]], 2) + self.assertEqual(plan.splits()[0].data_deletion_files[0].cardinality, 2) + self._assert_parity(table.copy({'scan.snapshot-id': '1'}), rows, 1) + + @unittest.skipUnless(native_version_at_least(0, 4, 0), + 'pypaimon-rust>=0.4.0 required for native DV scans') + def test_external_deletion_vector_path_is_preserved(self): + options = self._de_options() + options['deletion-vectors.enabled'] = 'true' + table = self._create('external_deletions', options) + rows = [{'k': k, 'v': 'v%d' % k} for k in range(4)] + self._write(table, rows) + self._delete(table, [1, 3]) + snapshot = table.snapshot_manager().get_latest_snapshot() + original = IndexManifestFile(table).read(snapshot.index_manifest)[0] + external_directory = tempfile.TemporaryDirectory(prefix='external_dv_') + self.addCleanup(external_directory.cleanup) + external_path = external_directory.name + '/deletions.bin' + table.file_io.copy_file( + table.path_factory().index_path() + '/' + original.index_file.file_name, + external_path) + external = replace(original, index_file=replace( + original.index_file, + file_name='external-' + original.index_file.file_name, + external_path=external_path)) + commit = table.new_batch_write_builder().new_commit() + try: + commit.commit([CommitMessage( + partition=(), bucket=0, new_files=[], index_adds=[external], + index_deletes=[replace(original, kind=1)])]) + finally: + commit.close() + + plan = self._assert_parity(table, [rows[0], rows[2]], 3) + self.assertEqual(plan.splits()[0].data_deletion_files[0].dv_index_path, + external_path) + self._assert_parity(table.copy({'scan.snapshot-id': '2'}), + [rows[0], rows[2]], 2) + + +if __name__ == '__main__': + unittest.main() diff --git a/paimon-python/pypaimon/tests/native_plan_distribution_test.py b/paimon-python/pypaimon/tests/native_plan_distribution_test.py new file mode 100644 index 000000000000..824faab67db9 --- /dev/null +++ b/paimon-python/pypaimon/tests/native_plan_distribution_test.py @@ -0,0 +1,362 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Distributed native planning must preserve assignment before reader limits.""" + +import tempfile +import unittest +from contextlib import ExitStack +from unittest.mock import patch + +import pyarrow as pa + +from pypaimon import CatalogFactory, Schema +from pypaimon.deletionvectors.bitmap_deletion_vector import BitmapDeletionVector +from pypaimon.read.native_plan import ( + native_method_available, + native_version_at_least, + native_runtime_available, +) +from pypaimon.read.sliced_split import SlicedSplit +from pypaimon.table.row.generic_row import GenericRow +from pypaimon.utils.range import Range +from pypaimon.write.commit_message import CommitMessage +from pypaimon.write.table_delete import TableDeleteByRowId + + +class _DistributionFixture: + + def setUp(self): + warehouse = tempfile.TemporaryDirectory(prefix='native_distribution_') + self.addCleanup(warehouse.cleanup) + self.catalog = CatalogFactory.create({'warehouse': warehouse.name}) + self.catalog.create_database('default', True) + self.schema = pa.schema([('k', pa.int64()), ('v', pa.string())]) + + def _create(self, name, options=None, schema=None, partition_keys=None, + primary_keys=None): + self.catalog.create_table('default.' + name, Schema.from_pyarrow_schema( + self.schema if schema is None else schema, + options=options, partition_keys=partition_keys, + primary_keys=primary_keys), False) + return self.catalog.get_table('default.' + name) + + def _write(self, table, rows, schema=None): + builder = table.new_batch_write_builder() + writer, commit = builder.new_write(), builder.new_commit() + try: + writer.write_arrow(pa.Table.from_pylist( + rows, schema=self.schema if schema is None else schema)) + commit.commit(writer.prepare_commit()) + finally: + writer.close() + commit.close() + + @staticmethod + def _de_options(**extra): + options = { + 'data-evolution.enabled': 'true', + 'row-tracking.enabled': 'true', + 'source.split.target-size': '1b', + } + options.update(extra) + return options + + @staticmethod + def _commit_update(table, operation): + builder = table.new_batch_write_builder() + messages = operation(builder.new_update()) + commit = builder.new_commit() + try: + commit.commit(messages) + finally: + commit.close() + + def _read(self, table, native, shard=None, slice_=None, row_ranges=None, + limit=None, predicate=None, projection=None): + builder = table.copy({ + 'scan.native-plan.enabled': str(native).lower(), + }).new_read_builder() + if limit is not None: + builder.with_limit(limit) + if predicate is not None: + builder.with_filter(predicate) + if projection is not None: + builder.with_projection(projection) + scan = builder.new_scan() + if shard is not None: + scan.with_shard(*shard) + if slice_ is not None: + scan.with_slice(*slice_) + if row_ranges is not None: + scan.with_row_ranges(row_ranges) + guard = patch.object(scan.file_scanner, 'scan', side_effect=AssertionError( + 'distributed native plan fell back to Python')) if native else ExitStack() + with guard: + plan = scan.plan() + return plan, builder.new_read().to_arrow(plan.splits(), parallelism=1).to_pylist() + + def _assert_parity(self, table, expected, snapshot_id, ordered=True, **options): + plans = [] + for native in (False, True): + plan, rows = self._read(table, native, **options) + self.assertEqual(plan.snapshot_id, snapshot_id) + if ordered: + self.assertEqual(rows, expected) + else: + self.assertCountEqual(rows, expected) + plans.append(plan) + return plans[1] + + def _append_dv_table(self, name, deleted_positions): + table = self._create(name, { + 'deletion-vectors.enabled': 'true', + 'source.split.target-size': '1b', + }) + rows = [{'k': k, 'v': str(k)} for k in range(9)] + self._write(table, rows[:6]) + self._write(table, rows[6:]) + plan, _ = self._read(table, False) + first_file = plan.splits()[0].files[0] + vector = BitmapDeletionVector() + for position in deleted_positions: + vector.delete(position) + entry = TableDeleteByRowId(table)._write_deletion_vector_index( + GenericRow([], []), 0, {first_file.file_name: vector}) + commit = table.new_batch_write_builder().new_commit() + try: + commit.commit([CommitMessage( + partition=(), bucket=0, new_files=[], index_adds=[entry])]) + finally: + commit.close() + return table, rows + + +@unittest.skipUnless(native_runtime_available(), + 'pypaimon_rust split-planning API required') +class NativePlanDistributionTest(_DistributionFixture, unittest.TestCase): + + def test_append_shards_preserve_partition_file_order_and_cover_once(self): + schema = self.schema.append(pa.field('p', pa.string())) + table = self._create('append_shards', + {'source.split.target-size': '1b'}, schema, ['p']) + batches = [ + [{'k': k, 'v': str(k), 'p': p} for k in keys] + for p, keys in [('a', range(0, 3)), ('b', range(3, 5)), + ('a', range(5, 8)), ('b', range(8, 11))]] + for rows in batches: + self._write(table, rows, schema) + expected_order = batches[0] + batches[2] + batches[1] + batches[3] + for count in (1, 3, 13): + collected = [] + for shard in range(count): + start = shard * (11 // count) + min(shard, 11 % count) + end = start + 11 // count + (shard < 11 % count) + expected = expected_order[start:end] + with self.subTest(count=count, shard=shard): + self._assert_parity(table, expected, 4, shard=(shard, count)) + collected.extend(expected) + self.assertEqual(collected, expected_order) + + def test_append_slices_cross_files_and_clip_end(self): + table = self._create('append_slices', {'source.split.target-size': '1b'}) + rows = [{'k': k, 'v': str(k)} for k in range(9)] + for start in range(0, 9, 3): + self._write(table, rows[start:start + 3]) + for start, end in ((0, 1), (2, 7), (7, 100), (20, 22)): + with self.subTest(start=start, end=end): + self._assert_parity(table, rows[start:end], 3, slice_=(start, end)) + + def test_append_limit_is_applied_after_shard_or_slice(self): + table = self._create('append_limit', {'source.split.target-size': '1b'}) + rows = [{'k': k, 'v': str(k)} for k in range(12)] + for start in range(0, 12, 3): + self._write(table, rows[start:start + 3]) + self._assert_parity(table, rows[4:6], 4, shard=(1, 3), limit=2) + self._assert_parity(table, rows[7:9], 4, slice_=(7, 11), limit=2) + self._assert_parity(table, [], 4, shard=(1, 3), limit=0) + + @unittest.skipUnless( + native_version_at_least(0, 4, 0), + 'pypaimon-rust>=0.4.0 required for native DV scans') + def test_append_dv_slice_limit_uses_actual_surviving_positions(self): + for deleted, expected_key in (((2, 3, 4, 5), 6), ((3, 4, 5), 2)): + with self.subTest(deleted=deleted): + table, rows = self._append_dv_table('append_dv_' + str(expected_key), deleted) + self._assert_parity(table, [rows[expected_key]], 3, + slice_=(2, 9), limit=1) + + def test_primary_key_shards_keep_buckets_and_all_versions_together(self): + table = self._create('pk_shards', { + 'bucket': '4', 'source.split.target-size': '1b', + }, primary_keys=['k']) + rows = [{'k': k, 'v': str(k)} for k in range(24)] + self._write(table, rows) + updates = [{'k': k, 'v': 'updated'} for k in (1, 4, 8, 12, 20)] + self._write(table, updates) + latest = {row['k']: row for row in rows + updates} + covered = [] + for shard in range(6): + normal, expected = self._read(table, False, shard=(shard, 6)) + native = self._assert_parity( + table, expected, 2, ordered=False, shard=(shard, 6)) + for plan in (normal, native): + self.assertTrue(all(split.bucket % 6 == shard for split in plan.splits())) + covered.extend(expected) + if expected: + # A nonzero shard must still see its rows with a small limit; + # applying the limit globally first would silently starve it. + _, limited = self._read(table, True, shard=(shard, 6), limit=1) + self.assertEqual(len(limited), 1) + self.assertIn(limited[0], expected) + self.assertCountEqual(covered, list(latest.values())) + self.assertEqual(len({row['k'] for row in covered}), len(covered)) + + @unittest.skipUnless(native_method_available('TableScan', 'with_row_position_slice'), + 'pypaimon_rust row-position selection API required') + def test_data_evolution_slice_positions_skip_row_id_gaps(self): + schema = self.schema.append(pa.field('p', pa.string())) + table = self._create('de_gaps', self._de_options(), schema, ['p']) + rows = [{'k': k, 'v': str(k), 'p': str(k // 3)} for k in range(9)] + for start in range(0, 9, 3): + self._write(table, rows[start:start + 3], schema) + predicate = table.new_read_builder().new_predicate_builder().equal('p', '1') + self._commit_update(table, lambda update: update.delete_by_predicate(predicate)) + surviving = rows[:3] + rows[6:] + self._assert_parity(table, surviving[2:5], 4, slice_=(2, 5)) + self._assert_parity(table, surviving[3:], 4, shard=(1, 2)) + self._assert_parity(table, [], 4, slice_=(10, 12)) + + @unittest.skipUnless(native_method_available('TableScan', 'with_row_position_shard'), + 'pypaimon_rust row-position selection API required') + def test_data_evolution_sharding_precedes_group_stats_pruning(self): + table = self._create('de_filter', self._de_options()) + rows = [{'k': k, 'v': str(k)} for k in range(6)] + self._write(table, rows[:3]) + self._write(table, rows[3:]) + predicate = table.new_read_builder().new_predicate_builder().greater_or_equal('k', 3) + self._assert_parity(table, [], 2, shard=(0, 2), predicate=predicate) + self._assert_parity(table, rows[3:], 2, shard=(1, 2), predicate=predicate) + self._assert_parity(table, rows[3:4], 2, shard=(1, 2), predicate=predicate, limit=1) + + @unittest.skipUnless(native_method_available('TableScan', 'with_row_position_slice'), + 'pypaimon_rust row-position selection API required') + def test_data_evolution_slice_intersects_explicit_ranges(self): + table = self._create('de_intersection', self._de_options()) + rows = [{'k': k, 'v': str(k)} for k in range(6)] + self._write(table, rows[:3]) + self._write(table, rows[3:]) + self._assert_parity(table, [], 2, slice_=(0, 2), row_ranges=[Range(4, 4)]) + self._assert_parity(table, rows[4:5], 2, + slice_=(3, 6), row_ranges=[Range(4, 4)]) + self._assert_parity(table, rows[4:5], 2, + shard=(1, 2), row_ranges=[Range(4, 4)]) + self._assert_parity(table, [], 2, shard=(0, 2), row_ranges=[]) + + @unittest.skipUnless(native_method_available('TableScan', 'with_row_position_shard'), + 'pypaimon_rust row-position selection API required') + def test_data_evolution_shards_count_positions_before_deletions(self): + options = self._de_options(**{'deletion-vectors.enabled': 'true'}) + table = self._create('de_dv_shards', options) + rows = [{'k': k, 'v': str(k)} for k in range(8)] + self._write(table, rows[:4]) + self._write(table, rows[4:]) + self._commit_update(table, lambda update: update.delete_by_row_id([1, 2, 7])) + for shard, keys in enumerate(((0,), (3, 4, 5), (6,))): + with self.subTest(shard=shard): + self._assert_parity(table, [rows[k] for k in keys], 3, + shard=(shard, 3)) + self._assert_parity(table, [rows[3], rows[4]], 3, + slice_=(1, 7), limit=2) + self._assert_parity(table.copy({'scan.snapshot-id': '2'}), rows[:3], 2, + shard=(0, 3)) + + @unittest.skipUnless( + native_method_available('Plan', 'snapshot_id') + and native_method_available('TableScan', 'with_row_position_shard'), + 'pypaimon_rust row-position selection and snapshot metadata required') + def test_empty_distributed_plans_keep_snapshot_metadata(self): + for de in (False, True): + table = self._create('empty_' + str(de), self._de_options() if de else None) + self._assert_parity(table, [], None, shard=(2, 3)) + self._assert_parity(table, [], None, slice_=(2, 3)) + self._write(table, [{'k': 1, 'v': 'a'}]) + self._assert_parity(table, [], 1, shard=(2, 3)) + + @unittest.skipUnless(native_method_available('TableScan', 'with_row_position_shard'), + 'pypaimon_rust row-position selection API required') + def test_data_evolution_updates_and_projection_do_not_multiply_positions(self): + schema = self.schema.append(pa.field('payload', pa.large_binary())) + table = self._create('de_projected_updates', self._de_options(), schema) + rows = [{'k': k, 'v': str(k), 'payload': str(k).encode()} for k in range(8)] + self._write(table, rows[:4], schema) + self._write(table, rows[4:], schema) + self._commit_update(table, lambda update: update.with_update_type(['v']) + .update_by_arrow_with_row_id(pa.table({ + '_ROW_ID': pa.array([5], type=pa.int64()), + 'v': ['updated'], + }))) + projected = [{'k': row['k'], 'v': row['v']} for row in rows] + projected[5]['v'] = 'updated' + self._assert_parity(table, projected[:4], 3, shard=(0, 2), projection=['k', 'v']) + self._assert_parity(table, projected[4:], 3, shard=(1, 2), projection=['k', 'v']) + self._assert_parity(table, projected[2:6], 3, slice_=(2, 6), projection=['k', 'v']) + self._assert_parity(table, projected[2:3], 3, slice_=(2, 6), + projection=['k', 'v'], limit=1) + + def test_invalid_distribution_parameters_fail_before_planning(self): + table = self._create('invalid') + self._write(table, [{'k': 1, 'v': 'a'}]) + for native in (False, True): + table_copy = table.copy({'scan.native-plan.enabled': str(native).lower()}) + for shard in ((-1, 2), (0, 0), (0, -1), (2, 2), (0.5, 2), (0, 2.5)): + with self.subTest(native=native, shard=shard), self.assertRaises(ValueError): + table_copy.new_read_builder().new_scan().with_shard(*shard) + for bounds in ((-1, 2), (0, 0), (2, 1), (0.5, 2), (0, 2.5)): + with self.subTest(native=native, bounds=bounds), self.assertRaises(ValueError): + table_copy.new_read_builder().new_scan().with_slice(*bounds) + with self.assertRaisesRegex(ValueError, 'simultaneously'): + table_copy.new_read_builder().new_scan().with_shard(0, 2).with_slice(0, 1) + with self.assertRaisesRegex(ValueError, 'simultaneously'): + table_copy.new_read_builder().new_scan().with_slice(0, 1).with_shard(0, 2) + + def test_primary_key_slice_remains_unsupported(self): + table = self._create('pk_slice', {'bucket': '1'}, primary_keys=['k']) + self._write(table, [{'k': 1, 'v': 'a'}]) + for native in (False, True): + with self.assertRaisesRegex(NotImplementedError, 'Primary key'): + self._read(table, native, slice_=(0, 1)) + + +class SlicedDeletionVectorLimitTest(_DistributionFixture, unittest.TestCase): + """Reader correctness also runs when the optional Rust package is absent.""" + + def test_partial_dv_split_cannot_be_estimated_or_dropped_by_limit(self): + for deleted, expected_key in (((2, 3, 4, 5), 6), ((3, 4, 5), 2)): + table, rows = self._append_dv_table('python_dv_' + str(expected_key), deleted) + plan, _ = self._read(table, False, slice_=(2, 9)) + sliced = plan.splits()[0] + self.assertIsNone(sliced.merged_row_count()) + exact = SlicedSplit(sliced.data_split(), sliced.shard_file_idx_map(), + exact_merged_row_count=int(expected_key == 2)) + self.assertEqual(exact.merged_row_count(), int(expected_key == 2)) + _, actual = self._read(table, False, slice_=(2, 9), limit=1) + self.assertEqual(actual, [rows[expected_key]]) + + +if __name__ == '__main__': + unittest.main() diff --git a/paimon-python/pypaimon/tests/native_plan_incremental_test.py b/paimon-python/pypaimon/tests/native_plan_incremental_test.py new file mode 100644 index 000000000000..74bd7a0f1ce9 --- /dev/null +++ b/paimon-python/pypaimon/tests/native_plan_incremental_test.py @@ -0,0 +1,250 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Compare committed timestamp windows, including cross-snapshot PK merges.""" + +import json +from contextlib import ExitStack +from unittest.mock import patch + +import pyarrow as pa +import pytest + +from pypaimon import CatalogFactory, Schema +from pypaimon.common.identifier import Identifier +from pypaimon.read.native_plan import native_method_available +from pypaimon.utils.range import Range + + +@pytest.fixture(params=[False, pytest.param( + True, marks=pytest.mark.skipif( + not native_method_available('ReadBuilder', 'new_incremental_scan'), + reason='pypaimon_rust combined incremental planning API required'))], + ids=['python', 'native']) +def native(request): + return request.param + + +@pytest.fixture +def catalog(tmp_path): + result = CatalogFactory.create({'warehouse': str(tmp_path)}) + result.create_database('default', True) + return result + + +SCHEMA = pa.schema([('k', pa.int64()), ('v', pa.string())]) + + +def _table(catalog, name, primary_key=False, options=None): + catalog.create_table('default.' + name, Schema.from_pyarrow_schema( + SCHEMA, primary_keys=['k'] if primary_key else None, + options=options), False) + return catalog.get_table('default.' + name) + + +def _set_time(table, timestamp): + manager = table.snapshot_manager() + snapshot = manager.get_latest_snapshot() + path = manager.get_snapshot_path(snapshot.id) + with open(path) as reader: + data = json.load(reader) + data['timeMillis'] = timestamp + with open(path, 'w') as writer: + json.dump(data, writer) + return snapshot.id + + +def _write(table, timestamp, rows, overwrite=False): + builder = table.new_batch_write_builder() + if overwrite: + builder.overwrite() + writer, commit = builder.new_write(), builder.new_commit() + try: + writer.write_arrow(pa.Table.from_pydict({ + key: [row[key] for row in rows] for key in SCHEMA.names}, schema=SCHEMA)) + commit.commit(writer.prepare_commit()) + finally: + writer.close() + commit.close() + return _set_time(table, timestamp) + + +def _read(table, native, window, predicate=None, limit=None, shard=None, + slice_=None, row_ranges=None, with_stats=False): + builder = table.copy({ + 'scan.native-plan.enabled': str(native).lower(), + 'scan.mode': 'incremental', + 'incremental-between-timestamp': '%s,%s' % window, + }).new_read_builder() + if predicate is not None: + builder.with_filter(predicate) + if limit is not None: + builder.with_limit(limit) + scan = builder.new_scan() + if shard is not None: + scan.with_shard(*shard) + if slice_ is not None: + scan.with_slice(*slice_) + if row_ranges is not None: + scan.with_row_ranges(row_ranges) + with ExitStack() as stack: + if native: + for method in ('scan', 'scan_with_stats'): + stack.enter_context(patch.object( + scan.file_scanner, method, side_effect=AssertionError( + 'incremental native plan fell back to Python'))) + plan = scan.scan_with_stats()[0] if with_stats else scan.plan() + result = builder.new_read().to_arrow(plan.splits(), parallelism=1).to_pydict() + return plan, [dict(zip(result, row)) for row in zip(*result.values())] + + +@pytest.fixture +def history(catalog): + table = _table(catalog, 'history', True, {'bucket': '1'}) + _write(table, 100, [{'k': 1, 'v': 'base'}, {'k': 2, 'v': 'base'}]) + _write(table, 200, [{'k': 1, 'v': 'intermediate'}, {'k': 3, 'v': 'third'}]) + _write(table, 200, [{'k': 1, 'v': 'latest'}, {'k': 4, 'v': 'fourth'}]) + _write(table, 300, [{'k': 9, 'v': 'overwrite'}], overwrite=True) + assert table.snapshot_manager().get_latest_snapshot().commit_kind == 'OVERWRITE' + _write(table, 400, [{'k': 5, 'v': 'fifth'}]) + return table + + +@pytest.mark.parametrize('window,snapshot_id,expected', [ + ((100, 200), 3, [(1, 'latest'), (3, 'third'), (4, 'fourth')]), + ((200, 300), 4, []), + ((100, 300), 4, [(1, 'latest'), (3, 'third'), (4, 'fourth')]), + ((100, 150), 1, []), + ((500, 600), None, []), + ((0, 50), None, []), + ((0, 100), 1, [(1, 'base'), (2, 'base')]), + ((200, 400), 5, [(5, 'fifth')]), +]) +def test_timestamp_windows_merge_appends_and_preserve_end_snapshot( + native, history, window, snapshot_id, expected): + plan, rows = _read(history, native, window) + assert plan.snapshot_id == snapshot_id + assert sorted((row['k'], row['v']) for row in rows) == expected + if native: + assert all(split.snapshot_id == snapshot_id for split in plan.splits()) + + +def test_predicate_does_not_resurrect_an_earlier_version(native, history): + predicate = history.new_read_builder().new_predicate_builder().equal('v', 'intermediate') + plan, rows = _read(history, native, (100, 200), predicate=predicate, with_stats=True) + assert plan.snapshot_id == 3 + assert rows == [] + + +def test_append_distribution_precedes_limit_in_incremental_window(native, catalog): + table = _table(catalog, 'append', options={'source.split.target-size': '1b'}) + rows = [{'k': key, 'v': str(key)} for key in range(12)] + for offset in range(0, 12, 3): + _write(table, (offset // 3 + 1) * 100, rows[offset:offset + 3]) + for options in ({'shard': (1, 3)}, {'slice_': (3, 8)}): + plan, actual = _read(table, native, (100, 400), limit=2, **options) + assert plan.snapshot_id == 4 + assert actual == rows[6:8] + + +def test_primary_key_shards_merge_all_selected_commits(native, catalog): + table = _table(catalog, 'pk_shards', True, {'bucket': '4'}) + rows = [{'k': key, 'v': 'base'} for key in range(20)] + _write(table, 100, rows) + _write(table, 200, [dict(row, v='old') for row in rows]) + _write(table, 300, [dict(row, v='new') for row in rows]) + result = [] + for shard in range(4): + plan, actual = _read(table, native, (100, 300), shard=(shard, 4)) + assert plan.snapshot_id == 3 + assert all(split.bucket % 4 == shard for split in plan.splits()) + result.extend(actual) + if actual: + _, limited = _read(table, native, (100, 300), shard=(shard, 4), limit=1) + assert len(limited) == 1 + assert limited[0] in actual + assert sorted((row['k'], row['v']) for row in result) == [ + (key, 'new') for key in range(20)] + + +def test_data_evolution_positions_intersect_incremental_row_ranges(native, catalog): + table = _table(catalog, 'de', options={ + 'data-evolution.enabled': 'true', 'row-tracking.enabled': 'true', + 'source.split.target-size': '1b', + }) + rows = [{'k': key, 'v': str(key)} for key in range(9)] + for offset in range(0, 9, 3): + _write(table, (offset // 3 + 1) * 100, rows[offset:offset + 3]) + plan, actual = _read(table, native, (100, 300), slice_=(1, 5), + row_ranges=[Range(5, 7)], limit=2) + assert plan.snapshot_id == 3 + assert actual == rows[5:7] + for selection in ({'slice_': (3, 6)}, {'shard': (1, 2)}): + plan, actual = _read(table, native, (100, 300), + row_ranges=[Range(7, 7)], **selection) + assert plan.snapshot_id == 3 + assert actual == rows[7:8] + _, actual = _read(table, native, (100, 300), slice_=(0, 1), + row_ranges=[Range(5, 7)]) + assert actual == [] + + +def test_incremental_branch_uses_its_own_snapshot_history(native, catalog): + table = _table(catalog, 'branch') + _write(table, 100, [{'k': 1, 'v': 'base'}]) + table.create_tag('base') + catalog.create_branch(table.identifier, 'test', tag_name='base') + branch = catalog.get_table(Identifier('default', 'branch', branch='test')) + _write(branch, 200, [{'k': 2, 'v': 'branch'}]) + _write(table, 300, [{'k': 3, 'v': 'main'}]) + plan, actual = _read(branch, native, (100, 300)) + assert plan.snapshot_id == 2 + assert actual == [{'k': 2, 'v': 'branch'}] + + +def test_incremental_uses_deletion_vectors_from_window_end(native, catalog): + table = _table(catalog, 'deletions', options={ + 'data-evolution.enabled': 'true', 'row-tracking.enabled': 'true', + 'deletion-vectors.enabled': 'true', 'index-file-in-data-file-dir': 'true', + }) + rows = [{'k': key, 'v': str(key)} for key in range(6)] + _write(table, 100, rows[:3]) + _write(table, 200, rows[3:]) + builder = table.new_batch_write_builder() + messages = builder.new_update().delete_by_row_id([1, 4]) + commit = builder.new_commit() + try: + commit.commit(messages) + finally: + commit.close() + assert _set_time(table, 300) == 3 + + historical, actual = _read(table, native, (100, 200)) + assert historical.snapshot_id == 2 + assert actual == rows[3:] + current, actual = _read(table, native, (100, 300)) + assert current.snapshot_id == 3 + assert actual == [rows[3], rows[5]] + _, actual = _read(table, native, (100, 300), slice_=(1, 3), limit=1) + assert actual == [rows[5]] + + +@pytest.mark.parametrize('window', ['100,100', '200,100', '100', 'one,200']) +def test_invalid_timestamp_window_is_rejected_even_for_empty_tables(catalog, window): + table = _table(catalog, 'invalid') + with pytest.raises(ValueError): + table.copy({'incremental-between-timestamp': window}).new_read_builder().new_scan() diff --git a/paimon-python/pypaimon/tests/native_plan_integration_test.py b/paimon-python/pypaimon/tests/native_plan_integration_test.py index 7c0112d59616..5ed13200a99a 100644 --- a/paimon-python/pypaimon/tests/native_plan_integration_test.py +++ b/paimon-python/pypaimon/tests/native_plan_integration_test.py @@ -23,7 +23,9 @@ from pypaimon import CatalogFactory, Schema from pypaimon.globalindex.global_index_result import GlobalIndexResult -from pypaimon.read.native_plan import native_family_search_modes_available +from pypaimon.read.native_plan import ( + native_family_search_modes_available, native_method_available, +) from pypaimon.table.row.blob import BlobDescriptor from pypaimon.utils.range import Range @@ -96,7 +98,7 @@ def test_primary_key_matches_normal_plan(self): self._assert_matches('pk_t') def test_pk_equal_to_partition_key_falls_back(self): - # Empty trimmed PK: native would skip merge and return duplicates -> must fall back. + # Rust rejects empty trimmed PK schemas; Python must retain its supported behavior. self.cat.create_table('default.pkpart_t', Schema.from_pyarrow_schema( self.schema, partition_keys=['k'], primary_keys=['k'], options={'bucket': '1'}), False) self._write('pkpart_t', [{'k': 1, 'v': 'a1'}, {'k': 2, 'v': 'b1'}]) @@ -433,17 +435,43 @@ def test_explain_reflects_native_plan(self): self.assertEqual(native.snapshot_id, normal.snapshot_id) self.assertIn('native', str(native)) # render shows the Planner line - def test_empty_table_explain_reflects_python_fallback(self): + @unittest.skipUnless(native_method_available('Plan', 'snapshot_id'), + "pypaimon_rust snapshot metadata API not installed") + def test_native_explain_reports_split_metadata_without_pruning_counters(self): + self.cat.create_table( + 'default.explain_metadata_t', Schema.from_pyarrow_schema( + self.schema, options={'metadata.stats-mode': 'full'}), False) + self._write('explain_metadata_t', [{'k': 1, 'v': 'a'}]) + self._write('explain_metadata_t', [{'k': 8, 'v': 'b'}]) + table = self.cat.get_table('default.explain_metadata_t').copy( + {'scan.native-plan.enabled': 'true'}) + builder = table.new_read_builder() + builder.with_filter(builder.new_predicate_builder().equal('k', 8)) + result = builder.explain() + self.assertTrue(result.native_planned) + self.assertEqual(result.snapshot_id, 2) + self.assertEqual(result.split_count, 1) + self.assertEqual(result.file_count, 1) + self.assertIn('pruning not tracked', str(result)) + self.assertIsNone(result.file_skipping) + builder.with_filter(builder.new_predicate_builder().equal('k', 99)) + empty = builder.explain() + self.assertTrue(empty.native_planned) + self.assertEqual(empty.snapshot_id, 2) + self.assertEqual(empty.split_count, 0) + + def test_empty_table_explain_preserves_native_metadata(self): self.cat.create_table( 'default.empty_t', Schema.from_pyarrow_schema(self.schema), False) normal = self.cat.get_table('default.empty_t').new_read_builder().explain() - fallback = self.cat.get_table('default.empty_t').copy( + native = self.cat.get_table('default.empty_t').copy( {'scan.native-plan.enabled': 'true'}).new_read_builder().explain() - self.assertFalse(fallback.native_planned) - self.assertEqual(fallback.snapshot_id, normal.snapshot_id) - self.assertEqual(fallback.split_count, 0) - self.assertNotIn('Planner:', str(fallback)) + self.assertEqual(native.native_planned, + native_method_available('Plan', 'snapshot_id')) + self.assertEqual(native.snapshot_id, normal.snapshot_id) + self.assertEqual(native.split_count, 0) + self.assertEqual('Planner:' in str(native), native.native_planned) if __name__ == '__main__': diff --git a/paimon-python/pypaimon/tests/native_plan_test.py b/paimon-python/pypaimon/tests/native_plan_test.py index acd624a6a73c..d363e8a1edfd 100644 --- a/paimon-python/pypaimon/tests/native_plan_test.py +++ b/paimon-python/pypaimon/tests/native_plan_test.py @@ -17,7 +17,7 @@ import sys import unittest -from types import ModuleType +from types import ModuleType, SimpleNamespace from unittest.mock import Mock, patch from pypaimon.catalog.catalog_context import CatalogContext @@ -37,8 +37,9 @@ _restore_python_partition_paths, native_family_search_modes_available, native_plan, + native_version_at_least, ) -from pypaimon.read.scan_stats import ScanStats +from pypaimon.read.plan import Plan from pypaimon.read.table_scan import TableScan from pypaimon.table.bucket_mode import BucketMode from pypaimon.utils.range import Range @@ -98,6 +99,10 @@ def setUp(self): {'pypaimon_rust': fake_mod, 'pypaimon_rust.datafusion': fake_df}) patcher.start() self.addCleanup(patcher.stop) + if sys.version_info >= (3, 8): + version_patcher = patch('importlib.metadata.version', return_value='0.3.0') + version_patcher.start() + self.addCleanup(version_patcher.stop) def test_switch_defaults_off(self): self.assertFalse(CoreOptions(Options({})).native_plan_enabled()) @@ -121,7 +126,7 @@ def test_plan_routes_to_native_and_prunes_partitions(self): fs = Mock(partition_key_predicate=pred) scan = _scan(native_enabled=True, file_scanner=fs) - with patch('pypaimon.read.native_plan.native_plan', return_value=[keep, drop]) as np: + with patch('pypaimon.read.native_plan.native_plan', return_value=Plan([keep, drop], 1)) as np: plan = scan.plan() np.assert_called_once_with( @@ -138,7 +143,7 @@ def test_plan_falls_back_when_partition_prune_raises(self): fs.scan.return_value = sentinel scan = _scan(native_enabled=True, file_scanner=fs) split = Mock(partition=Mock(values=[2026, 7]), snapshot_id=1) - with patch('pypaimon.read.native_plan.native_plan', return_value=[split]): + with patch('pypaimon.read.native_plan.native_plan', return_value=Plan([split], 1)): self.assertIs(scan.plan(), sentinel) fs.scan.assert_called_once_with() @@ -146,7 +151,7 @@ def test_plan_native_no_partition_predicate_keeps_all(self): splits = [Mock(partition=Mock(values=[1])), Mock(partition=Mock(values=[2]))] fs = Mock(partition_key_predicate=None) scan = _scan(native_enabled=True, file_scanner=fs) - with patch('pypaimon.read.native_plan.native_plan', return_value=splits): + with patch('pypaimon.read.native_plan.native_plan', return_value=Plan(splits, 1)): self.assertEqual(scan.plan().splits(), splits) def test_plan_forwards_filter_limit_partition_and_time_travel(self): @@ -168,7 +173,7 @@ def test_plan_forwards_filter_limit_partition_and_time_travel(self): scan.table.schema_manager.latest.return_value.id = 3 split = Mock(partition=Mock(values=['2026-08-02']), snapshot_id=3) - with patch('pypaimon.read.native_plan.native_plan', return_value=[split]) as np: + with patch('pypaimon.read.native_plan.native_plan', return_value=Plan([split], 3)) as np: plan = scan.plan() self.assertEqual(plan.snapshot_id, 3) @@ -188,7 +193,7 @@ def test_plan_forwards_global_index_row_ranges(self): Range(1, 2), Range(5, 5)]) split = Mock(partition=Mock(values=[]), snapshot_id=3) - with patch('pypaimon.read.native_plan.native_plan', return_value=[split]) as np: + with patch('pypaimon.read.native_plan.native_plan', return_value=Plan([split], 3)) as np: plan = scan.plan() np.assert_called_once_with( @@ -207,7 +212,7 @@ def test_empty_global_index_result_does_not_fall_back(self): fs.data_evolution = True fs._global_index_result = GlobalIndexResult.create_empty() - with patch('pypaimon.read.native_plan.native_plan', return_value=[]) as np: + with patch('pypaimon.read.native_plan.native_plan', return_value=Plan([], 1)) as np: plan = scan.plan() np.assert_called_once_with( @@ -256,7 +261,7 @@ def test_global_index_row_ranges_require_data_evolution_append_table(self): fs.scan.assert_called_once_with() def test_plan_falls_back_when_scan_is_not_plain(self): - # Native planning does not carry shard/slice, explicit row ranges, + # Older native bindings cannot carry DE shard/slice or explicit row ranges, # arbitrary global-index results, or incremental scans. def check(setup): fs = Mock(partition_key_predicate=None) @@ -269,8 +274,10 @@ def check(setup): np.assert_not_called() fs.scan.assert_called_once_with() - check(lambda s, fs: setattr(fs, 'idx_of_this_subtask', 0)) - check(lambda s, fs: setattr(fs, 'start_pos_of_this_subtask', 0)) + check(lambda s, fs: (setattr(fs, 'data_evolution', True), + setattr(fs, 'idx_of_this_subtask', 0))) + check(lambda s, fs: (setattr(fs, 'data_evolution', True), + setattr(fs, 'start_pos_of_this_subtask', 0))) check(lambda s, fs: setattr(fs, 'chunk_shuffle', (1, 100))) check(lambda s, fs: setattr(fs, '_global_index_result', object())) check(lambda s, fs: setattr(fs, '_row_ranges', [object()])) @@ -305,15 +312,17 @@ def check(setup): check(lambda s, fs: s.table.options.options.contains.__setattr__( 'return_value', True)) # incremental - def test_plan_native_empty_falls_back(self): - # Empty native result -> fall back for an atomic snapshot id. - fs = Mock(partition_key_predicate=None) - sentinel = object() - fs.scan.return_value = sentinel - scan = _scan(native_enabled=True, file_scanner=fs) - with patch('pypaimon.read.native_plan.native_plan', return_value=[]): - self.assertIs(scan.plan(), sentinel) - fs.scan.assert_called_once_with() + def test_plan_native_empty_preserves_snapshot_without_fallback(self): + for snapshot_id in (None, 7): + with self.subTest(snapshot_id=snapshot_id): + fs = Mock(partition_key_predicate=None) + scan = _scan(native_enabled=True, file_scanner=fs) + with patch('pypaimon.read.native_plan.native_plan', + return_value=Plan([], snapshot_id)): + result = scan.plan() + self.assertEqual(result.splits(), []) + self.assertEqual(result.snapshot_id, snapshot_id) + fs.scan.assert_not_called() def test_plan_falls_back_when_rust_unavailable(self): # scan.native-plan.enabled but pypaimon-rust missing/old -> fall back, @@ -347,7 +356,7 @@ def test_family_search_modes_require_rust_0_4(self): 'native_family_search_modes_available', return_value=available), patch( 'pypaimon.read.native_plan.native_plan', - return_value=[split]) as native: + return_value=Plan([split], 1)) as native: plan = scan.plan() if expect_native: @@ -385,7 +394,7 @@ def test_dynamic_read_option_uses_native_plan(self): with patch( 'pypaimon.read.native_plan.native_plan', - return_value=[split]) as native: + return_value=Plan([split], 1)) as native: self.assertEqual(scan.plan().splits(), [split]) native.assert_called_once() @@ -454,22 +463,18 @@ def load(self): np.assert_not_called() fs.scan.assert_called_once_with() - def test_scan_with_stats_native_empty_uses_fallback_stats(self): + def test_scan_with_stats_preserves_native_empty_snapshot(self): fs = Mock(partition_key_predicate=None) - fallback_plan = object() - fallback_stats = ScanStats(manifest_files_total=7) - fs.scan_with_stats.return_value = (fallback_plan, fallback_stats) scan = _scan(native_enabled=True, file_scanner=fs) - - with patch('pypaimon.read.native_plan.native_plan', return_value=[]) as np: + native = Plan([], 7) + with patch('pypaimon.read.native_plan.native_plan', return_value=native) as np: plan, stats = scan.scan_with_stats() - - self.assertIs(plan, fallback_plan) - self.assertIs(stats, fallback_stats) + self.assertEqual(plan.snapshot_id, 7) + self.assertIsNone(stats) np.assert_called_once_with( scan.table, predicate=None, limit=None, projection=None, row_ranges=None) - fs.scan_with_stats.assert_called_once_with() + fs.scan_with_stats.assert_not_called() fs.scan.assert_not_called() def test_catalog_options_are_normalized_for_rust(self): @@ -589,7 +594,7 @@ def test_family_search_mode_version_gate(self): cases = { '0.3.0': False, '0.4.0': True, - '0.4.0.dev20260808': True, + '0.4.0.dev20260808': False, '1.0.0': True, } for version, expected in cases.items(): @@ -682,6 +687,7 @@ def test_native_plan_threads_trimmed_keys_to_deserializer(self): # deserializer so per-file min/max keys are decoded for merge-on-read. kfields = [object()] table = Mock(trimmed_primary_keys_fields=kfields) + table.current_branch.return_value = 'main' table.table_schema = Mock(fields=[], partition_keys=[]) table.partition_keys = [] table.options.source_split_target_size.return_value = 1024 @@ -694,6 +700,7 @@ def test_native_plan_threads_trimmed_keys_to_deserializer(self): builder = rt.new_read_builder.return_value builder.with_row_ranges.return_value = builder builder.new_scan.return_value.plan.return_value.splits.return_value = [split] + builder.new_scan.return_value.plan.return_value.snapshot_id.return_value = 3 catalog = Mock() catalog.get_table.return_value = rt @@ -710,7 +717,8 @@ def test_native_plan_threads_trimmed_keys_to_deserializer(self): return_value='decoded') as des: result = native_plan(table, row_ranges=[(1, 2)]) - self.assertEqual(result, ['decoded']) + self.assertEqual(result.splits(), ['decoded']) + self.assertEqual(result.snapshot_id, 3) rt.new_read_builder.assert_called_once_with({ CoreOptions.SOURCE_SPLIT_TARGET_SIZE.key(): '1024', CoreOptions.SOURCE_SPLIT_OPEN_FILE_COST.key(): '128', @@ -718,6 +726,176 @@ def test_native_plan_threads_trimmed_keys_to_deserializer(self): builder.with_row_ranges.assert_called_once_with([(1, 2)]) des.assert_called_once_with(b'bytes', [], kfields) + def test_native_plan_empty_snapshot_and_legacy_runtime(self): + for snapshot_id, legacy in ((None, False), (7, False), (None, True)): + with self.subTest(snapshot_id=snapshot_id, legacy=legacy): + table = _scan(True, Mock()).table + table.table_schema = Mock(fields=[], partition_keys=[]) + rust_plan = SimpleNamespace(splits=lambda: []) + if not legacy: + rust_plan.snapshot_id = lambda: snapshot_id + scan = SimpleNamespace(plan=lambda: rust_plan) + rt = Mock() + rt.new_read_builder.return_value.new_scan.return_value = scan + with patch('pypaimon_rust.datafusion.PaimonCatalog') as catalog: + catalog.return_value.get_table.return_value = rt + if legacy: + with self.assertRaisesRegex(RuntimeError, "empty plan's snapshot"): + native_plan(table) + else: + plan = native_plan(table) + self.assertEqual(plan.snapshot_id, snapshot_id) + self.assertEqual(plan.splits(), []) + + def test_native_plan_branch_resolution(self): + table = _scan(True, Mock()).table + table.table_schema = Mock(fields=[], partition_keys=[]) + table.current_branch.return_value = 'b1' + rust_plan = SimpleNamespace(splits=lambda: [], snapshot_id=lambda: 7) + scan = Mock() + scan.plan.return_value = rust_plan + rt = Mock() + rt.branch.return_value = 'b1' + rt.new_read_builder.return_value.new_scan.return_value = scan + with patch('pypaimon_rust.datafusion.PaimonCatalog') as catalog: + catalog.return_value.get_table.return_value = rt + plan = native_plan(table) + self.assertEqual(plan.snapshot_id, 7) + scan.plan.assert_called_once_with() + rt.branch.return_value = 'main' + with self.assertRaisesRegex(RuntimeError, 'requested branch'): + native_plan(table) + + def test_explicit_row_ranges_require_runtime_api(self): + for available in (False, True): + for ranges in ([], [Range(1, 2), Range(5, 8)]): + with self.subTest(available=available, ranges=ranges): + fs = Mock(partition_key_predicate=None) + scan = _scan(True, fs) + fs._row_ranges = ranges + fs.scan.return_value = fallback = object() + with patch('pypaimon.read.native_plan.native_method_available', + return_value=available), patch( + 'pypaimon.read.native_plan.native_plan', + return_value=Plan([], 3)) as native: + result = scan.plan() + if available: + self.assertEqual(result.snapshot_id, 3) + self.assertEqual(native.call_args[1]['row_ranges'], + [(r.from_, r.to) for r in ranges]) + fs.scan.assert_not_called() + else: + self.assertIs(result, fallback) + native.assert_not_called() + + def test_watermark_forwarding_requires_current_runtime(self): + for available in (False, True): + with self.subTest(available=available): + fs = Mock(partition_key_predicate=None) + scan = _scan(True, fs) + scan.table.options.options = Options({'scan.watermark': '200'}) + scan.table._applied_dynamic_options = {'scan.watermark': '200'} + scan.table.schema_manager.latest.return_value.id = 2 + fs.scan.return_value = fallback = object() + self.assertEqual(_read_options(scan.table)['scan.watermark'], '200') + with patch('pypaimon.read.native_plan.native_version_at_least', + return_value=available), patch( + 'pypaimon.read.native_plan.native_plan', + return_value=Plan([], 1)) as native: + result = scan.plan() + if available: + self.assertEqual(result.snapshot_id, 1) + fs.scan.assert_not_called() + else: + self.assertIs(result, fallback) + native.assert_not_called() + + @unittest.skipIf(sys.version_info < (3, 8), + "importlib.metadata requires Python 3.8") + def test_deletion_vectors_require_fixed_version_and_matching_l0_semantics(self): + for version in ('0.3.0', '0.4.0rc1', '0.4.0', '0.4.1'): + for bucket_local in (False, True): + for merge_on_read in (False, True): + with self.subTest(version=version, bucket_local=bucket_local, + merge_on_read=merge_on_read): + fs = Mock(partition_key_predicate=None) + scan = _scan(True, fs) + fs.deletion_vectors_enabled = True + scan.table.options.options = Options({ + 'index-file-in-data-file-dir': str(bucket_local).lower(), + 'deletion-vectors.merge-on-read': str(merge_on_read).lower(), + }) + fs.scan.return_value = fallback = object() + with patch('importlib.metadata.version', return_value=version), patch( + 'pypaimon.read.native_plan.native_plan', + return_value=Plan([], 1)) as native: + result = scan.plan() + if version in ('0.4.0', '0.4.1') and not merge_on_read: + self.assertEqual(result.snapshot_id, 1) + fs.scan.assert_not_called() + else: + self.assertIs(result, fallback) + native.assert_not_called() + + @unittest.skipIf(sys.version_info < (3, 8), + "importlib.metadata requires Python 3.8") + def test_runtime_version_comparison_preserves_patch_and_release_order(self): + cases = [ + ('0.3.99', (0, 4, 0), False), + ('0.4.0.dev1', (0, 4, 0), False), + ('0.4.0rc1', (0, 4, 0), False), + ('0.4.0', (0, 4, 0), True), + ('0.4.0+local', (0, 4, 0), True), + ('0.4.0.post1', (0, 4, 0), True), + ('0.4.0', (0, 4, 1), False), + ('0.4.1', (0, 4, 1), True), + ('1.0.0', (0, 4, 0), True), + ('unknown', (0, 4, 0), False), + ] + for version, minimum, expected in cases: + with self.subTest(version=version, minimum=minimum), patch( + 'importlib.metadata.version', return_value=version): + self.assertEqual(native_version_at_least(*minimum), expected) + + def test_incremental_range_is_forwarded_without_timestamp_reinterpretation(self): + fs = Mock(partition_key_predicate=None) + scan = _scan(True, fs) + scan.table.options.options = Options({'incremental-between-timestamp': '100,200'}) + scan.table._applied_dynamic_options = {'incremental-between-timestamp': '100,200'} + scan._incremental_snapshot_range = (2, 4) + with patch('pypaimon.read.native_plan.native_method_available', return_value=True), patch( + 'pypaimon.read.native_plan.native_plan', return_value=Plan([], 4)) as native: + self.assertEqual(scan.plan().snapshot_id, 4) + self.assertEqual(native.call_args[1]['incremental_range'], (2, 4)) + fs.scan.assert_not_called() + + def test_incremental_window_outside_snapshots_is_terminal_empty(self): + fs = Mock(partition_key_predicate=None) + scan = _scan(True, fs) + scan.table.options.options = Options({'incremental-between-timestamp': '100,200'}) + scan._incremental_snapshot_range = None + with patch('pypaimon.read.native_plan.native_method_available', return_value=True), patch( + 'pypaimon.read.native_plan.native_plan') as native: + plan = scan.plan() + self.assertEqual(plan.splits(), []) + self.assertIsNone(plan.snapshot_id) + native.assert_not_called() + fs.scan.assert_not_called() + + def test_primary_key_shard_defers_limit_until_after_bucket_selection(self): + fs = Mock(partition_key_predicate=None) + scan = _scan(True, fs) + scan.table.is_primary_key_table = True + scan.limit = 1 + fs.idx_of_this_subtask, fs.number_of_para_subtasks = 1, 2 + splits = [Mock(bucket=0), Mock(bucket=1)] + fs._apply_push_down_limit.side_effect = lambda selected: selected + with patch('pypaimon.read.native_plan.native_plan', return_value=Plan(splits, 3)) as native: + self.assertEqual(scan.plan().splits(), [splits[1]]) + self.assertIsNone(native.call_args[1]['limit']) + fs._apply_push_down_limit.assert_called_once_with([splits[1]]) + fs.scan.assert_not_called() + def test_native_plan_requires_split_api(self): # An intermediate pypaimon-rust missing either get_table or Split.serialize # must raise a clear error, not an AttributeError mid-plan. diff --git a/paimon-python/pypaimon/utils/file_store_path_factory.py b/paimon-python/pypaimon/utils/file_store_path_factory.py index a98ddf00ce35..413bc463163b 100644 --- a/paimon-python/pypaimon/utils/file_store_path_factory.py +++ b/paimon-python/pypaimon/utils/file_store_path_factory.py @@ -28,6 +28,14 @@ def _is_null_or_whitespace_only(value) -> bool: return len(s) == 0 or s.isspace() +def _escape_partition_component(value: str) -> str: + # Java PartitionPathUtils.CHAR_TO_ESCAPE (spaces and Unicode stay as-is). + escape_chars = "\"#%'*/:=?\\{}[]^" + return ''.join('%{:02X}'.format(ord(char)) + if ord(char) < 32 or ord(char) == 127 or char in escape_chars else char + for char in value) + + class FileStorePathFactory: MANIFEST_PATH = "manifest" MANIFEST_PREFIX = "manifest-" @@ -98,7 +106,7 @@ def data_file_path(self) -> str: return f"{self._root}/{self.data_file_path_directory}" return self._root - def relative_bucket_path(self, partition: Tuple, bucket: int) -> str: + def relative_bucket_path(self, partition: Tuple, bucket: int, canonical_partition: bool = False) -> str: bucket_name = str(bucket) if bucket == BucketMode.POSTPONE_BUCKET.value: bucket_name = "postpone" @@ -113,7 +121,10 @@ def relative_bucket_path(self, partition: Tuple, bucket: int) -> str: if _is_null_or_whitespace_only(val): val = self.default_part_value else: - val = str(val) + val = str(val).lower() if canonical_partition and isinstance(val, bool) else str(val) + if canonical_partition: + field_name = _escape_partition_component(field_name) + val = _escape_partition_component(val) partition_parts.append(f"{field_name}={val}") if partition_parts: relative_parts = partition_parts + relative_parts @@ -124,8 +135,8 @@ def relative_bucket_path(self, partition: Tuple, bucket: int) -> str: return "/".join(relative_parts) - def bucket_path(self, partition: Tuple, bucket: int) -> str: - relative_path = self.relative_bucket_path(partition, bucket) + def bucket_path(self, partition: Tuple, bucket: int, canonical_partition: bool = False) -> str: + relative_path = self.relative_bucket_path(partition, bucket, canonical_partition) return f"{self._root}/{relative_path}" def create_external_path_provider( @@ -149,6 +160,44 @@ def global_index_path_factory(self) -> 'IndexPathFactory': self.global_index_external_path is not None, ) + def new_bucket_index_path(self, partition: Tuple, bucket: int, file_name: str) -> Tuple[str, bool]: + """Return a new bucket index's path and whether to persist its external location.""" + if self.index_file_in_data_file_dir: + external = self.create_external_path_provider(partition, bucket) + if external is not None: + return external.get_next_external_data_path(file_name), True + # Python data directories historically use str(value) without + # escaping. Record the actual location when Java renders it + # differently, so its readers can find the DV beside those files. + return (f"{self.bucket_path(partition, bucket)}/{file_name}", + self._partition_path_requires_explicit_location(partition)) + factory = self.global_index_path_factory() + return factory.to_path(file_name), factory.is_external_path() + + def _partition_path_requires_explicit_location(self, partition: Tuple) -> bool: + # FLOAT/DOUBLE formatting can differ from Python's float repr, and + # this factory does not carry field types to distinguish the two. + return (any(isinstance(value, float) for value in partition) + or self.relative_bucket_path(partition, 0) != self.relative_bucket_path(partition, 0, True)) + + def bucket_index_path(self, partition: Tuple, bucket: int, index_file, file_io=None) -> str: + """Resolve an existing bucket index, including the legacy Python DV layout.""" + if index_file.external_path: + return index_file.external_path + legacy_path = f"{self.index_path()}/{index_file.file_name}" + if not self.index_file_in_data_file_dir: + return legacy_path + path = f"{self.bucket_path(partition, bucket, True)}/{index_file.file_name}" + # Older Python DV writers ignored the option. Prefer the Java location + # when present, and use the old directory only for an existing DV file. + if file_io is not None and index_file.index_type == 'DELETION_VECTORS' and not file_io.exists(path): + python_path = f"{self.bucket_path(partition, bucket)}/{index_file.file_name}" + if python_path != path and file_io.exists(python_path): + return python_path + if file_io.exists(legacy_path): + return legacy_path + return path + class IndexPathFactory: diff --git a/paimon-python/pypaimon/write/file_store_commit.py b/paimon-python/pypaimon/write/file_store_commit.py index 66e88f2f5a68..10ac3e2b53c3 100644 --- a/paimon-python/pypaimon/write/file_store_commit.py +++ b/paimon-python/pypaimon/write/file_store_commit.py @@ -75,12 +75,16 @@ def _abort_commit_messages(table, commit_messages: List[CommitMessage]): try: index_file = entry.index_file file_name = index_file.file_name - path = ( - index_file.external_path - or table.path_factory() - .global_index_path_factory() - .to_path(file_name) - ) + if index_file.index_type == 'DELETION_VECTORS': + path = table.path_factory().bucket_index_path( + tuple(entry.partition.values), entry.bucket, index_file, table.file_io) + else: + path = ( + index_file.external_path + or table.path_factory() + .global_index_path_factory() + .to_path(file_name) + ) table.file_io.delete_quietly(path) except Exception as error: logger.warning( diff --git a/paimon-python/pypaimon/write/table_delete.py b/paimon-python/pypaimon/write/table_delete.py index 5c475c31b1f6..18dff8b91020 100644 --- a/paimon-python/pypaimon/write/table_delete.py +++ b/paimon-python/pypaimon/write/table_delete.py @@ -261,7 +261,7 @@ def _read_existing_deletion_vectors( if not entry.index_file.dv_ranges: continue - dv_path = self._index_file_path(entry.index_file) + dv_path = self._index_file_path(entry.index_file, partition, bucket) for data_file_name, meta in entry.index_file.dv_ranges.items(): deletion_file = DeletionFile( dv_index_path=dv_path, @@ -283,7 +283,8 @@ def _write_deletion_vector_index( deletion_vectors: Dict[str, DeletionVector], ) -> IndexManifestEntry: file_name = f"{FileStorePathFactory.INDEX_PREFIX}{uuid.uuid4()}-1" - path = f"{self.table.path_factory().index_path()}/{file_name}" + path, external = self.table.path_factory().new_bucket_index_path( + tuple(partition.values), bucket, file_name) position = 1 dv_ranges = {} @@ -315,6 +316,7 @@ def _write_deletion_vector_index( file_size=len(data), row_count=len(dv_ranges), dv_ranges=dv_ranges, + external_path=path if external else None, ) return IndexManifestEntry( kind=_ADD, @@ -323,7 +325,6 @@ def _write_deletion_vector_index( index_file=index_file, ) - def _index_file_path(self, index_file: IndexFileMeta) -> str: - if index_file.external_path: - return index_file.external_path - return f"{self.table.path_factory().index_path()}/{index_file.file_name}" + def _index_file_path(self, index_file: IndexFileMeta, partition: GenericRow, bucket: int) -> str: + return self.table.path_factory().bucket_index_path( + tuple(partition.values), bucket, index_file, self.file_io) From 7325d8493407be6c357a2c579d0b8fb7215755be Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Mon, 14 Sep 2026 23:50:02 +0800 Subject: [PATCH 2/6] [python] Remove NaN-specific split grouping --- .../pypaimon/read/interval_partition.py | 14 +-------- .../pypaimon/tests/interval_partition_test.py | 31 +++++-------------- 2 files changed, 8 insertions(+), 37 deletions(-) diff --git a/paimon-python/pypaimon/read/interval_partition.py b/paimon-python/pypaimon/read/interval_partition.py index 5c40a53e3f10..2112b4d895ba 100644 --- a/paimon-python/pypaimon/read/interval_partition.py +++ b/paimon-python/pypaimon/read/interval_partition.py @@ -42,20 +42,9 @@ class IntervalPartition: def __init__(self, input_files: List[DataFileMeta]): self.files = input_files.copy() self.key_comparator = default_key_comparator - # Manifest FLOAT/DOUBLE values decode to Python float. Match the native - # planner's conservative fallback for NaN boundaries: all files share - # one section, but each file remains a separate merge input. - self.has_nan_key = any( - isinstance(value, float) and math.isnan(value) - for file in self.files - for key in (file.min_key, file.max_key) if key is not None - for value in key.values) - if not self.has_nan_key: - self.files.sort(key=cmp_to_key(self._compare_files)) + self.files.sort(key=cmp_to_key(self._compare_files)) def partition(self) -> List[List[SortedRun]]: - if self.has_nan_key: - return [[SortedRun(files=[file]) for file in self.files]] result = [] section: List[DataFileMeta] = [] bound = None @@ -129,7 +118,6 @@ def default_key_comparator(key1: GenericRow, key2: GenericRow) -> int: if val2 is None: return 1 # Preserve Java's ordering of signed zeros in composite key bounds. - # NaN bounds take the conservative path before this comparator is used. if (isinstance(val1, float) and isinstance(val2, float) and val1 == 0.0 and val2 == 0.0): sign1, sign2 = math.copysign(1.0, val1), math.copysign(1.0, val2) diff --git a/paimon-python/pypaimon/tests/interval_partition_test.py b/paimon-python/pypaimon/tests/interval_partition_test.py index 2043ff12ef22..6fbae3624dbd 100644 --- a/paimon-python/pypaimon/tests/interval_partition_test.py +++ b/paimon-python/pypaimon/tests/interval_partition_test.py @@ -17,7 +17,7 @@ """Floating-point key bounds preserve overlapping split groups.""" -from decimal import Decimal, InvalidOperation +from decimal import Decimal from types import SimpleNamespace import pytest @@ -50,13 +50,12 @@ def key(values): @pytest.mark.parametrize('type_name', ['FLOAT', 'DOUBLE']) -@pytest.mark.parametrize('minimum,maximum,point', [ - ((1.0, 0), (float('nan'), 10), (2.0, 100)), - ((-0.0, 0), (0.0, 10), (-0.0, 100)), -]) -def test_floating_key_ranges_keep_versions_in_one_split(type_name, minimum, maximum, point): +def test_signed_zero_key_ranges_keep_versions_in_one_split(type_name): fields = _key_fields(type_name) - files = [_file('broad', minimum, maximum, fields), _file('point', point, point, fields)] + files = [ + _file('broad', (-0.0, 0), (0.0, 10), fields), + _file('point', (-0.0, 100), (-0.0, 100), fields), + ] sections = IntervalPartition(files).partition() assert len(sections) == 1 assert sorted([f.file_name for f in run.files] for run in sections[0]) == [['broad'], ['point']] @@ -69,27 +68,11 @@ def test_floating_key_ranges_keep_versions_in_one_split(type_name, minimum, maxi assert not splits[0].raw_convertible -@pytest.mark.parametrize('type_name', ['FLOAT', 'DOUBLE']) -def test_nan_boundary_keeps_disjoint_files_in_separate_runs(type_name): - fields = _key_fields(type_name) - files = [ - _file('finite', (1.0, 0), (2.0, 0), fields), - _file('nan', (float('nan'), 100), (float('nan'), 100), fields), - ] - sections = IntervalPartition(files).partition() - # NaN metadata uses the same conservative grouping as the native planner. - assert len(sections) == 1 - assert sorted([f.file_name for f in run.files] for run in sections[0]) == [['finite'], ['nan']] - - -def test_decimal_keys_keep_numeric_equality_and_invalid_nan_errors(): +def test_decimal_keys_keep_numeric_equality(): fields = _key_fields('DECIMAL(10, 2)') left = GenericRow([Decimal('-0'), 1], fields) right = GenericRow([Decimal('0'), 1], fields) assert default_key_comparator(left, right) == 0 - invalid = GenericRow([Decimal('NaN'), 1], fields) - with pytest.raises(InvalidOperation): - default_key_comparator(invalid, right) @pytest.mark.parametrize('type_name', ['FLOAT', 'DOUBLE']) From eef62150a29a41dedff082cb68d0279b85ace7a6 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Tue, 15 Sep 2026 09:09:18 +0800 Subject: [PATCH 3/6] [python] Fix planner CI coverage and validation regressions --- .github/workflows/ci-python.yml | 4 ++++ paimon-python/README.md | 3 ++- paimon-python/conftest.py | 2 ++ paimon-python/pypaimon/read/scan_distribution.py | 6 ++++-- paimon-python/pypaimon/read/table_scan.py | 6 ++++++ .../pypaimon/tests/deletion_vector_path_test.py | 4 ++-- .../pypaimon/tests/global_index_test.py | 2 ++ .../tests/native_plan_capabilities_test.py | 2 ++ .../tests/native_plan_distribution_test.py | 2 ++ .../tests/native_plan_incremental_test.py | 4 ++-- .../tests/native_plan_integration_test.py | 2 ++ paimon-python/pypaimon/tests/native_plan_test.py | 16 ++++++++++++++++ 12 files changed, 46 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-python.yml b/.github/workflows/ci-python.yml index a33b09820ea6..ca5938da1e0d 100644 --- a/.github/workflows/ci-python.yml +++ b/.github/workflows/ci-python.yml @@ -34,6 +34,9 @@ jobs: # Lint + test on selected versions only (3.6 / 3.7 low end, 3.10 / 3.11 / 3.12 / 3.13 current), not every Python version. test: name: Tests / Python ${{ matrix.python-version }} + env: + # Real native planning is covered by the Rust main job below. + PYTEST_ADDOPTS: "-m 'not native_plan'" timeout-minutes: 90 runs-on: ubuntu-latest container: "python:${{ matrix.python-version }}-slim" @@ -148,6 +151,7 @@ jobs: python -c "import datasets, lerobot; print('datasets', datasets.__version__, 'lerobot', lerobot.__version__)" fi + # Required by SQL tests; native planner tests run against main below. python -m pip install pypaimon-rust if [[ "${{ matrix.python-version }}" == "3.11" ]]; then # Run the RoboMIND pipeline tests with synthetic local HDF5 data. diff --git a/paimon-python/README.md b/paimon-python/README.md index 1730deedba99..db4d215fa0bc 100644 --- a/paimon-python/README.md +++ b/paimon-python/README.md @@ -63,7 +63,8 @@ and prereleases before 0.4.0 use the Python planner for deletion vectors. When using an unreleased 0.4.0 development wheel, rebuild it with these fixes; package version checks cannot distinguish local builds with identical versions. -Append scans support `with_shard()` and `with_slice()`; primary-key scans support +Append scans support `with_shard()` and `with_slice()` with Rust 0.4 or newer, +which preserves the file order needed for positional selection; primary-key scans support bucket-based `with_shard()`. Data-evolution position selection requires the binding's `TableScan.with_row_position_slice()` and `with_row_position_shard()`. Selection occurs before reader filtering and deletion vectors, so surviving row diff --git a/paimon-python/conftest.py b/paimon-python/conftest.py index b64f9d47c6d8..7928854abc94 100644 --- a/paimon-python/conftest.py +++ b/paimon-python/conftest.py @@ -38,6 +38,8 @@ def _native_plan_enabled(): def pytest_configure(config): config.addinivalue_line( "markers", "python_plan: keep Python planner assertions on the Python lane") + config.addinivalue_line( + "markers", "native_plan: exercise the real Rust planner in the Rust main CI job") if not _native_plan_enabled(): return diff --git a/paimon-python/pypaimon/read/scan_distribution.py b/paimon-python/pypaimon/read/scan_distribution.py index 416ae3ef14df..dcb2d1b61592 100644 --- a/paimon-python/pypaimon/read/scan_distribution.py +++ b/paimon-python/pypaimon/read/scan_distribution.py @@ -26,8 +26,10 @@ def validate_shard(index: int, count: int) -> None: if not isinstance(count, int) or count <= 0: raise ValueError("number_of_para_subtasks must be a positive integer") - if not isinstance(index, int) or index < 0 or index >= count: - raise ValueError("idx_of_this_subtask must be non-negative and less than number_of_para_subtasks") + if not isinstance(index, int) or index < 0: + raise ValueError("idx_of_this_subtask must be a non-negative integer") + if index >= count: + raise ValueError("idx_of_this_subtask must be less than number_of_para_subtasks") def validate_slice(start: int, end: int) -> None: diff --git a/paimon-python/pypaimon/read/table_scan.py b/paimon-python/pypaimon/read/table_scan.py index ce56bc29eedf..04edea9a70bf 100755 --- a/paimon-python/pypaimon/read/table_scan.py +++ b/paimon-python/pypaimon/read/table_scan.py @@ -133,6 +133,12 @@ def _native_plan_supported_impl(self) -> bool: or not self._native_global_index_result_supported() or getattr(fs, 'only_read_real_buckets', False)): return False + # Positional append distribution needs the stable partition/file order + # introduced in 0.4. Older bindings can assign different rows per call. + if (not self.table.is_primary_key_table and not fs.data_evolution + and (fs.idx_of_this_subtask is not None or fs.start_pos_of_this_subtask is not None) + and not native_version_at_least(0, 4)): + return False if getattr(fs, 'deletion_vectors_enabled', False): # 0.4.0 includes Python-written DV decoding and legacy bucket paths. if not native_version_at_least(0, 4, 0): diff --git a/paimon-python/pypaimon/tests/deletion_vector_path_test.py b/paimon-python/pypaimon/tests/deletion_vector_path_test.py index 006b21c36cbf..93ddd6c2a473 100644 --- a/paimon-python/pypaimon/tests/deletion_vector_path_test.py +++ b/paimon-python/pypaimon/tests/deletion_vector_path_test.py @@ -32,9 +32,9 @@ from pypaimon.write.table_delete import TableDeleteByRowId -_PLANNERS = ['python', pytest.param('native', marks=pytest.mark.skipif( +_PLANNERS = ['python', pytest.param('native', marks=[pytest.mark.native_plan, pytest.mark.skipif( not native_version_at_least(0, 4, 0), - reason='pypaimon-rust>=0.4.0 required for native DV paths'))] + reason='pypaimon-rust>=0.4.0 required for native DV paths')])] def _table(tmp_path, layout, first_partition='a', partition_type=None): diff --git a/paimon-python/pypaimon/tests/global_index_test.py b/paimon-python/pypaimon/tests/global_index_test.py index 3e6ab7485a7d..a5c2094862ab 100644 --- a/paimon-python/pypaimon/tests/global_index_test.py +++ b/paimon-python/pypaimon/tests/global_index_test.py @@ -356,6 +356,8 @@ def test_split_planning_merges_indexed_and_unindexed_ranges(self): ) scanner = FileScanner.__new__(FileScanner) + scanner.idx_of_this_subtask = None + scanner.start_pos_of_this_subtask = None scanner.manifest_scanner = unittest.mock.MagicMock( return_value=([], unittest.mock.Mock(id=3))) scanner._global_index_result = None diff --git a/paimon-python/pypaimon/tests/native_plan_capabilities_test.py b/paimon-python/pypaimon/tests/native_plan_capabilities_test.py index 36e2eb641cc2..2a35b75b7c1f 100644 --- a/paimon-python/pypaimon/tests/native_plan_capabilities_test.py +++ b/paimon-python/pypaimon/tests/native_plan_capabilities_test.py @@ -24,6 +24,7 @@ from unittest.mock import patch import pyarrow as pa +import pytest from pypaimon import CatalogFactory, Schema from pypaimon.common.identifier import Identifier @@ -42,6 +43,7 @@ from pypaimon.write.table_delete import TableDeleteByRowId +@pytest.mark.native_plan @unittest.skipUnless(native_runtime_available(), "pypaimon_rust split-planning API not installed") class NativePlanCapabilitiesTest(unittest.TestCase): diff --git a/paimon-python/pypaimon/tests/native_plan_distribution_test.py b/paimon-python/pypaimon/tests/native_plan_distribution_test.py index 824faab67db9..efb0f6cebeff 100644 --- a/paimon-python/pypaimon/tests/native_plan_distribution_test.py +++ b/paimon-python/pypaimon/tests/native_plan_distribution_test.py @@ -23,6 +23,7 @@ from unittest.mock import patch import pyarrow as pa +import pytest from pypaimon import CatalogFactory, Schema from pypaimon.deletionvectors.bitmap_deletion_vector import BitmapDeletionVector @@ -146,6 +147,7 @@ def _append_dv_table(self, name, deleted_positions): return table, rows +@pytest.mark.native_plan @unittest.skipUnless(native_runtime_available(), 'pypaimon_rust split-planning API required') class NativePlanDistributionTest(_DistributionFixture, unittest.TestCase): diff --git a/paimon-python/pypaimon/tests/native_plan_incremental_test.py b/paimon-python/pypaimon/tests/native_plan_incremental_test.py index 74bd7a0f1ce9..7eaac1ad8c2f 100644 --- a/paimon-python/pypaimon/tests/native_plan_incremental_test.py +++ b/paimon-python/pypaimon/tests/native_plan_incremental_test.py @@ -31,9 +31,9 @@ @pytest.fixture(params=[False, pytest.param( - True, marks=pytest.mark.skipif( + True, marks=[pytest.mark.native_plan, pytest.mark.skipif( not native_method_available('ReadBuilder', 'new_incremental_scan'), - reason='pypaimon_rust combined incremental planning API required'))], + reason='pypaimon_rust combined incremental planning API required')])], ids=['python', 'native']) def native(request): return request.param diff --git a/paimon-python/pypaimon/tests/native_plan_integration_test.py b/paimon-python/pypaimon/tests/native_plan_integration_test.py index 5ed13200a99a..19138a8052a7 100644 --- a/paimon-python/pypaimon/tests/native_plan_integration_test.py +++ b/paimon-python/pypaimon/tests/native_plan_integration_test.py @@ -20,6 +20,7 @@ from unittest.mock import patch import pyarrow as pa +import pytest from pypaimon import CatalogFactory, Schema from pypaimon.globalindex.global_index_result import GlobalIndexResult @@ -46,6 +47,7 @@ def _has_native_row_ranges(): return hasattr(ReadBuilder, 'with_row_ranges') +@pytest.mark.native_plan @unittest.skipUnless(_has_native_planner(), "pypaimon_rust with split-planning API not installed") class NativePlanIntegrationTest(unittest.TestCase): diff --git a/paimon-python/pypaimon/tests/native_plan_test.py b/paimon-python/pypaimon/tests/native_plan_test.py index d363e8a1edfd..6d6875163a65 100644 --- a/paimon-python/pypaimon/tests/native_plan_test.py +++ b/paimon-python/pypaimon/tests/native_plan_test.py @@ -788,6 +788,22 @@ def test_explicit_row_ranges_require_runtime_api(self): self.assertIs(result, fallback) native.assert_not_called() + def test_append_distribution_requires_stable_native_order(self): + for available in (False, True): + for selection in ('idx_of_this_subtask', 'start_pos_of_this_subtask'): + with self.subTest(available=available, selection=selection): + fs = Mock(partition_key_predicate=None) + scan = _scan(True, fs) + setattr(fs, selection, 0) + fs.scan.return_value = fallback = object() + with patch('pypaimon.read.native_plan.native_version_at_least', + return_value=available): + self.assertEqual(scan._native_plan_supported(), available) + if not available: + with patch('pypaimon.read.native_plan.native_plan') as native: + self.assertIs(scan.plan(), fallback) + native.assert_not_called() + def test_watermark_forwarding_requires_current_runtime(self): for available in (False, True): with self.subTest(available=available): From ead0e87f640868cbb286f2ee8da3601697c4db23 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Tue, 15 Sep 2026 11:06:36 +0800 Subject: [PATCH 4/6] [python] Resolve typed Java deletion vector paths --- paimon-python/README.md | 2 + .../pypaimon/common/options/core_options.py | 9 ++ .../pypaimon/table/file_store_table.py | 3 +- .../tests/deletion_vector_path_test.py | 98 +++++++++++++-- .../pypaimon/utils/file_store_path_factory.py | 116 +++++++++++++++++- 5 files changed, 218 insertions(+), 10 deletions(-) diff --git a/paimon-python/README.md b/paimon-python/README.md index db4d215fa0bc..2667bb5c45a7 100644 --- a/paimon-python/README.md +++ b/paimon-python/README.md @@ -57,6 +57,8 @@ checked before planning. Deletion-vector scans require `pypaimon-rust>=0.4.0`, which includes schema-aware decoding of Python-written index manifests and legacy bucket-index path compatibility. The reader honors explicit paths, then bucket paths, and can read older Python files placed in `table/index`. +Bucket paths use the partition field types and `partition.legacy-name` to match +Java formatting, including timestamp precision and different JVM float spellings. New Python writes honor `index-file-in-data-file-dir` and retain explicit paths when Python and Java partition-directory formatting differs. Older releases and prereleases before 0.4.0 use the Python planner for deletion vectors. diff --git a/paimon-python/pypaimon/common/options/core_options.py b/paimon-python/pypaimon/common/options/core_options.py index cd9913ff0ed7..ce77d21f5a84 100644 --- a/paimon-python/pypaimon/common/options/core_options.py +++ b/paimon-python/pypaimon/common/options/core_options.py @@ -1133,6 +1133,15 @@ class CoreOptions: ) ) + PARTITION_GENERATE_LEGACY_NAME: ConfigOption[bool] = ( + ConfigOptions.key("partition.legacy-name") + .boolean_type() + .default_value(True) + .with_description( + "Use legacy Java toString partition names; otherwise use casts to string." + ) + ) + DYNAMIC_PARTITION_OVERWRITE: ConfigOption[bool] = ( ConfigOptions.key("dynamic-partition-overwrite") .boolean_type() diff --git a/paimon-python/pypaimon/table/file_store_table.py b/paimon-python/pypaimon/table/file_store_table.py index 32f07d8534fb..5519ee087dc4 100644 --- a/paimon-python/pypaimon/table/file_store_table.py +++ b/paimon-python/pypaimon/table/file_store_table.py @@ -382,12 +382,13 @@ def path_factory(self) -> 'FileStorePathFactory': return FileStorePathFactory( root=str(self.table_path), partition_keys=self.partition_keys, + partition_types=[field.type for field in self.partition_keys_fields], default_part_value=self.options.options.get( CoreOptions.PARTITION_DEFAULT_NAME, "__DEFAULT_PARTITION__"), format_identifier=format_identifier, data_file_prefix="data-", changelog_file_prefix="changelog-", - legacy_partition_name=True, + legacy_partition_name=self.options.options.get(CoreOptions.PARTITION_GENERATE_LEGACY_NAME), file_suffix_include_compression=False, file_compression=file_compression, data_file_path_directory=None, diff --git a/paimon-python/pypaimon/tests/deletion_vector_path_test.py b/paimon-python/pypaimon/tests/deletion_vector_path_test.py index 93ddd6c2a473..3ef766414bcd 100644 --- a/paimon-python/pypaimon/tests/deletion_vector_path_test.py +++ b/paimon-python/pypaimon/tests/deletion_vector_path_test.py @@ -19,6 +19,8 @@ from pathlib import Path from dataclasses import replace +from datetime import date, datetime, time, timedelta +from decimal import Decimal from unittest.mock import patch import pyarrow as pa @@ -37,13 +39,14 @@ reason='pypaimon-rust>=0.4.0 required for native DV paths')])] -def _table(tmp_path, layout, first_partition='a', partition_type=None): +def _table(tmp_path, layout, first_partition='a', partition_type=None, legacy_partition_name=True): catalog = CatalogFactory.create({'warehouse': str(tmp_path / 'warehouse')}) catalog.create_database('db', True) options = { 'bucket': '-1', 'data-evolution.enabled': 'true', 'row-tracking.enabled': 'true', 'deletion-vectors.enabled': 'true', 'scan.native-plan.enabled': 'false', 'index-file-in-data-file-dir': str(layout.startswith('bucket')).lower(), + 'partition.legacy-name': str(legacy_partition_name).lower(), } if layout == 'bucket-external': options.update({ @@ -54,6 +57,8 @@ def _table(tmp_path, layout, first_partition='a', partition_type=None): elif layout == 'global-external': options['global-index.external-path'] = (tmp_path / 'external-index').as_uri() second_partition = 'b' if partition_type is None else False if pa.types.is_boolean(partition_type) else 2.0 + if partition_type is not None and pa.types.is_timestamp(partition_type): + second_partition = first_partition + timedelta(days=1) schema = pa.schema([('p', pa.string() if partition_type is None else partition_type), ('k', pa.int64())]) catalog.create_table('db.t', Schema.from_pyarrow_schema( schema, partition_keys=['p'], options=options), False) @@ -235,16 +240,56 @@ def test_bucket_dv_preserves_python_typed_partition_directory(tmp_path, planner, @pytest.mark.parametrize('planner', _PLANNERS) -@pytest.mark.parametrize('value,partition_type,canonical_name', [('a/b', None, 'a%2Fb'), (True, pa.bool_(), 'true')]) +@pytest.mark.parametrize('value,partition_type,canonical_name', [ + ('a/b', None, 'a%2Fb'), (True, pa.bool_(), 'true'), + (0.1, pa.float32(), '0.1'), + (0.0001, pa.float32(), '1.0E-4'), + (1e7, pa.float32(), '1.0E7'), + (-0.0, pa.float32(), '-0.0'), + (1.4e-45, pa.float32(), '1.4E-45'), + (3.4028234663852886e38, pa.float32(), '3.4028235E38'), + (1.17549435e-38, pa.float32(), '1.17549435E-38'), + (1.17549435e-38, pa.float32(), '1.1754944E-38'), + (2.68873286e11, pa.float32(), '2.68873286E11'), + (2.68873286e11, pa.float32(), '2.6887329E11'), + (9.64991956e24, pa.float32(), '9.6499195E24'), + (1e23, pa.float64(), '9.999999999999999E22'), + (1e23, pa.float64(), '1.0E23'), + (-2.3345394554987242e17, pa.float64(), '-2.33453945549872416E17'), + (5e-324, pa.float64(), '4.9E-324'), + (datetime(2026, 9, 15, 12), pa.timestamp('s'), '2026-09-15 12%3A00%3A00'), + (datetime(2026, 9, 15, 12), pa.timestamp('ms'), '2026-09-15 12%3A00%3A00.000'), + (datetime(2026, 9, 15, 12, 0, 0, 120000), pa.timestamp('ms'), '2026-09-15 12%3A00%3A00.120'), + (datetime(2026, 9, 15, 12), pa.timestamp('us'), '2026-09-15 12%3A00%3A00.000000'), + (datetime(2026, 9, 15, 12, 0, 0, 123456), pa.timestamp('us'), '2026-09-15 12%3A00%3A00.123456'), + (datetime(2026, 9, 15, 12), pa.timestamp('ns'), '2026-09-15 12%3A00%3A00.000000000'), +]) def test_java_canonical_bucket_dv_without_external_path(tmp_path, planner, value, partition_type, canonical_name): - table = _table(tmp_path, 'bucket', value, partition_type) + _check_java_bucket_dv(tmp_path, planner, value, partition_type, canonical_name, False) + + +@pytest.mark.parametrize('planner', _PLANNERS) +@pytest.mark.parametrize('value,partition_type,canonical_name', [ + (datetime(2026, 9, 15, 12), pa.timestamp('ms'), '2026-09-15T12%3A00'), + (datetime(2026, 9, 15, 12, 0, 1), pa.timestamp('ms'), '2026-09-15T12%3A00%3A01'), + (datetime(2026, 9, 15, 12, 0, 0, 120000), pa.timestamp('ms'), '2026-09-15T12%3A00%3A00.120'), + (datetime(2026, 9, 15, 12, 0, 0, 120100), pa.timestamp('us'), '2026-09-15T12%3A00%3A00.120100'), +]) +def test_java_legacy_bucket_dv_without_external_path(tmp_path, planner, value, partition_type, canonical_name): + _check_java_bucket_dv(tmp_path, planner, value, partition_type, canonical_name, True) + + +def _check_java_bucket_dv(tmp_path, planner, value, partition_type, canonical_name, legacy_partition_name): + table = _table(tmp_path, 'bucket', value, partition_type, legacy_partition_name) _delete(table, [0]) old = _entries(table, 2)[0] canonical_path = str(Path(table.table_path) / ('p=' + canonical_name) / ('bucket-' + str(old.bucket)) / old.index_file.file_name) # Model the persisted layout produced by Java: the bucket path is canonical # and no explicit index location is needed in its manifest metadata. - with table.file_io.new_input_stream(old.index_file.external_path) as stream: + old_path = table.path_factory().bucket_index_path( + tuple(old.partition.values), old.bucket, old.index_file, table.file_io) + with table.file_io.new_input_stream(old_path) as stream: data = stream.read() with table.file_io.new_output_stream(canonical_path) as stream: stream.write(data) @@ -257,9 +302,28 @@ def test_java_canonical_bucket_dv_without_external_path(tmp_path, planner, value finally: commit.close() # Case-insensitive local filesystems consider p=True and p=true identical. - if not Path(old.index_file.external_path).samefile(canonical_path): - table.file_io.delete_quietly(old.index_file.external_path) - plan = _read(table, planner, 3, [1, 2, 3]) + if not Path(old_path).samefile(canonical_path): + table.file_io.delete_quietly(old_path) + # Both previous Python locations must lose to every Java spelling. + with table.file_io.new_output_stream(old_path) as stream: + stream.write(b'not the canonical deletion vector') + with table.file_io.new_output_stream(table.path_factory().index_path() + '/' + old.index_file.file_name) as stream: + stream.write(b'not a bucket deletion vector') + if partition_type is not None and pa.types.is_floating(partition_type): + other_partition = str(Path(table.table_path) / 'p=2.0' / ('bucket-' + str(old.bucket)) / + old.index_file.file_name) + with table.file_io.new_output_stream(other_partition) as stream: + stream.write(b'not the requested floating partition') + if planner == 'native' and partition_type is not None and pa.types.is_floating(partition_type): + # Rust does not support floating partitions; exercise the real adapter + # fallback against the same Java-produced DV layout. + table = table.copy({'scan.native-plan.enabled': 'true'}) + planner = 'python' + with patch('pypaimon.read.native_plan.native_plan', wraps=native_plan) as native_call: + plan = _read(table, planner, 3, [1, 2, 3]) + assert native_call.call_count == 1 + else: + plan = _read(table, planner, 3, [1, 2, 3]) paths = [dv.dv_index_path for split in plan.splits() for dv in split.data_deletion_files or [] if dv is not None] assert paths == [canonical_path] @@ -268,6 +332,26 @@ def test_java_canonical_bucket_dv_without_external_path(tmp_path, planner, value _read(table, planner, 3, [1, 2, 3]) +@pytest.mark.parametrize('legacy_partition_name,expected', [ + (False, 'ts=2026-09-15 12%3A00%3A00.000/day=1970-01-02/tm=12%3A34%3A56'), + (True, 'ts=2026-09-15T12%3A00/day=1/tm=45296120'), +]) +def test_typed_partition_paths_follow_partition_key_order(tmp_path, legacy_partition_name, expected): + catalog = CatalogFactory.create({'warehouse': str(tmp_path)}) + catalog.create_database('db', True) + schema = pa.schema([ + ('id', pa.int64()), ('ts', pa.timestamp('ms')), ('p/q', pa.float32()), + ('day', pa.date32()), ('tm', pa.time32('ms')), ('d', pa.decimal128(10, 9)), + ]) + catalog.create_table('db.t', Schema.from_pyarrow_schema( + schema, partition_keys=['p/q', 'ts', 'day', 'tm', 'd'], + options={'partition.legacy-name': str(legacy_partition_name).lower()}), False) + factory = catalog.get_table('db.t').path_factory() + partition = (0.1, datetime(2026, 9, 15, 12), date(1970, 1, 2), time(12, 34, 56, 120000), Decimal('0E-9')) + assert factory.relative_bucket_path(partition, 2, True) == ( + 'p%2Fq=0.1/' + expected + '/d=0.000000000/bucket-2') + + @pytest.mark.parametrize('planner', _PLANNERS) def test_multiple_dv_index_files_in_one_bucket_keep_all_deletions(tmp_path, planner): table = _table(tmp_path, 'bucket') diff --git a/paimon-python/pypaimon/utils/file_store_path_factory.py b/paimon-python/pypaimon/utils/file_store_path_factory.py index 413bc463163b..6136af727b52 100644 --- a/paimon-python/pypaimon/utils/file_store_path_factory.py +++ b/paimon-python/pypaimon/utils/file_store_path_factory.py @@ -15,9 +15,14 @@ # specific language governing permissions and limitations # under the License. +import struct +from datetime import date +from decimal import Decimal from typing import List, Optional, Tuple +from pypaimon.casting.row_to_string import cast_value_to_string, _is_unsupported from pypaimon.common.external_path_provider import ExternalPathProvider +from pypaimon.schema.data_types import DataType from pypaimon.table.bucket_mode import BucketMode @@ -36,6 +41,39 @@ def _escape_partition_component(value: str) -> str: for char in value) +def _floating_partition_string(value, single_precision: bool) -> str: + # Use a shortest round-tripping form for the initial lookup. Older JVMs + # can use different digits; the read fallback matches their stored values. + encoding = '>f' if single_precision else '>d' + bits = struct.pack(encoding, value) + value = struct.unpack(encoding, bits)[0] + for precision in range(2, 10 if single_precision else 18): + text = format(value, '.{}g'.format(precision)) + try: + rounded = struct.pack(encoding, float(text)) + except OverflowError: + continue + if rounded != bits: + continue + decimal = Decimal(text) + if not decimal.is_finite(): + return str(value) + if decimal.is_zero() or Decimal('0.001') <= abs(decimal) < Decimal('1e7'): + text = format(decimal, 'f') + if '.' in text: + text = text.rstrip('0').rstrip('.') + if '.' not in text: + text += '.0' + else: + mantissa, exponent = format(decimal, 'e').split('e') + mantissa = mantissa.rstrip('0').rstrip('.') if '.' in mantissa else mantissa + if '.' not in mantissa: + mantissa += '.0' + text = '{}E{}'.format(mantissa, int(exponent)) + return text + return str(value) + + class FileStorePathFactory: MANIFEST_PATH = "manifest" MANIFEST_PREFIX = "manifest-" @@ -67,9 +105,11 @@ def __init__( external_path_weights: Optional[List[int]] = None, index_file_in_data_file_dir: bool = False, global_index_external_path: Optional[str] = None, + partition_types: Optional[List[DataType]] = None, ): self._root = root.rstrip('/') self.partition_keys = partition_keys + self.partition_types = partition_types self.default_part_value = default_part_value self.format_identifier = format_identifier self.data_file_prefix = data_file_prefix @@ -107,6 +147,38 @@ def data_file_path(self) -> str: return self._root def relative_bucket_path(self, partition: Tuple, bucket: int, canonical_partition: bool = False) -> str: + if canonical_partition and partition: + partition = self._canonical_partition(partition) + return self._relative_bucket_path(partition, bucket, canonical_partition) + + def _canonical_partition(self, partition: Tuple) -> Tuple[str, ...]: + values = [] + for i, value in enumerate(partition): + data_type = self.partition_types[i] if self.partition_types is not None else None + type_name = str(data_type).split('(', 1)[0].split()[0] + if _is_null_or_whitespace_only(value): + text = self.default_part_value + elif type_name in ('FLOAT', 'REAL', 'DOUBLE'): + text = _floating_partition_string(value, type_name != 'DOUBLE') + elif self.legacy_partition_name and type_name == 'DATE': + text = str((value - date(1970, 1, 1)).days) + elif self.legacy_partition_name and type_name.startswith('TIMESTAMP'): + text = value.isoformat(timespec='minutes') + if value.second or value.microsecond: + text = value.isoformat(timespec='microseconds' if value.microsecond else 'seconds') + if value.microsecond and value.microsecond % 1000 == 0: + text = text[:-3] + elif self.legacy_partition_name and type_name.startswith('TIME'): + text = str(((value.hour * 60 + value.minute) * 60 + value.second) * 1000 + + value.microsecond // 1000) + elif data_type is not None and not _is_unsupported(data_type): + text = cast_value_to_string(value, data_type) + else: + text = str(value).lower() if isinstance(value, bool) else str(value) + values.append(text) + return tuple(values) + + def _relative_bucket_path(self, partition: Tuple, bucket: int, canonical_partition: bool) -> str: bucket_name = str(bucket) if bucket == BucketMode.POSTPONE_BUCKET.value: bucket_name = "postpone" @@ -175,8 +247,8 @@ def new_bucket_index_path(self, partition: Tuple, bucket: int, file_name: str) - return factory.to_path(file_name), factory.is_external_path() def _partition_path_requires_explicit_location(self, partition: Tuple) -> bool: - # FLOAT/DOUBLE formatting can differ from Python's float repr, and - # this factory does not carry field types to distinguish the two. + # FLOAT/DOUBLE spellings also vary between JVM versions, so persist + # their actual Python location even if one Java spelling matches it. return (any(isinstance(value, float) for value in partition) or self.relative_bucket_path(partition, 0) != self.relative_bucket_path(partition, 0, True)) @@ -192,12 +264,52 @@ def bucket_index_path(self, partition: Tuple, bucket: int, index_file, file_io=N # when present, and use the old directory only for an existing DV file. if file_io is not None and index_file.index_type == 'DELETION_VECTORS' and not file_io.exists(path): python_path = f"{self.bucket_path(partition, bucket)}/{index_file.file_name}" + alternate = self._find_floating_bucket_index(partition, bucket, index_file.file_name, file_io, python_path) + if alternate is not None: + return alternate if python_path != path and file_io.exists(python_path): return python_path if file_io.exists(legacy_path): return legacy_path return path + def _find_floating_bucket_index(self, partition, bucket, file_name, file_io, python_path): + floating = [str(data_type).split()[0] in ('FLOAT', 'REAL', 'DOUBLE') + for data_type in self.partition_types or []] + if not any(is_float and value is not None for is_float, value in zip(floating, partition)): + return None + # Float/Double.toString changed across JDK releases. Only if the usual + # path is missing, inspect floating partition components and compare + # their exact encoded values (including the sign of zero). + paths = [self.data_file_path()] + for i, text in enumerate(self._canonical_partition(partition)): + prefix = _escape_partition_component(self.partition_keys[i]) + '=' + if not floating[i] or partition[i] is None: + paths = [path + '/' + prefix + _escape_partition_component(text) for path in paths] + continue + encoding = '>d' if str(self.partition_types[i]).split()[0] == 'DOUBLE' else '>f' + expected = struct.pack(encoding, partition[i]) + matched = [] + for path in paths: + if not file_io.exists(path): + continue + for status in file_io.list_status(path): + name = status.base_name + if not name.startswith(prefix): + continue + try: + if struct.pack(encoding, float(name[len(prefix):])) == expected: + matched.append(path + '/' + name) + except (ValueError, OverflowError): + continue + paths = sorted(matched) + bucket_name = 'postpone' if bucket == BucketMode.POSTPONE_BUCKET.value else str(bucket) + for path in paths: + candidate = '{}/{}{}/{}'.format(path, self.BUCKET_PATH_PREFIX, bucket_name, file_name) + if candidate != python_path and file_io.exists(candidate): + return candidate + return None + class IndexPathFactory: From 10b5984866bf3e7e23fc2f47fc2dfae1554be7f2 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Tue, 15 Sep 2026 12:12:43 +0800 Subject: [PATCH 5/6] [python] Align LTZ partition paths with Java timezone formatting --- .../tests/deletion_vector_path_test.py | 74 ++++++++++++++++++- .../pypaimon/utils/file_store_path_factory.py | 14 +++- 2 files changed, 85 insertions(+), 3 deletions(-) diff --git a/paimon-python/pypaimon/tests/deletion_vector_path_test.py b/paimon-python/pypaimon/tests/deletion_vector_path_test.py index 3ef766414bcd..da61d04dc4af 100644 --- a/paimon-python/pypaimon/tests/deletion_vector_path_test.py +++ b/paimon-python/pypaimon/tests/deletion_vector_path_test.py @@ -17,9 +17,13 @@ """Deletion-vector locations must survive writes, further deletes and time travel.""" +import os +import subprocess +import sys +import time as system_time from pathlib import Path from dataclasses import replace -from datetime import date, datetime, time, timedelta +from datetime import date, datetime, time, timedelta, timezone from decimal import Decimal from unittest.mock import patch @@ -29,6 +33,7 @@ from pypaimon import CatalogFactory, Schema from pypaimon.manifest.index_manifest_file import IndexManifestFile from pypaimon.read.native_plan import native_plan, native_version_at_least +from pypaimon.schema.data_types import AtomicType from pypaimon.write.file_store_commit import _abort_commit_messages from pypaimon.write.commit_message import CommitMessage from pypaimon.write.table_delete import TableDeleteByRowId @@ -279,6 +284,73 @@ def test_java_legacy_bucket_dv_without_external_path(tmp_path, planner, value, p _check_java_bucket_dv(tmp_path, planner, value, partition_type, canonical_name, True) +@pytest.mark.skipif(not hasattr(system_time, 'tzset'), reason='requires POSIX timezone support') +@pytest.mark.parametrize('local_zone', ['UTC', 'Asia/Shanghai']) +@pytest.mark.parametrize('type_name', ['TIMESTAMP_LTZ(3)', 'TIMESTAMP(3) WITH LOCAL TIME ZONE']) +@pytest.mark.parametrize('value', [ + datetime(2026, 9, 15, 20, 0, 0, 120000), + datetime(2026, 9, 15, 20, 0, 0, 120000, tzinfo=timezone.utc), + datetime(2026, 9, 16, 4, 0, 0, 120000, tzinfo=timezone(timedelta(hours=8))), +]) +@pytest.mark.parametrize('legacy', [False, True]) +def test_ltz_canonical_partition_normalizes_timezone(tmp_path, monkeypatch, local_zone, type_name, value, legacy): + catalog = CatalogFactory.create({'warehouse': str(tmp_path)}) + catalog.create_database('db', True) + catalog.create_table('db.t', Schema.from_pyarrow_schema( + pa.schema([('p', pa.timestamp('ms', tz='UTC'))]), partition_keys=['p'], + options={'partition.legacy-name': str(legacy).lower()}), False) + factory = catalog.get_table('db.t').path_factory() + factory.partition_types = [AtomicType(type_name)] + # Naive LTZ values represent UTC, just as GenericRow serialization does. + with monkeypatch.context() as context: + context.setenv('TZ', local_zone) + system_time.tzset() + try: + if legacy: + expected = '2026-09-15T20%3A00%3A00.120' + elif local_zone == 'UTC': + expected = '2026-09-15 20%3A00%3A00.120' + else: + expected = '2026-09-16 04%3A00%3A00.120' + assert factory.relative_bucket_path((value,), 2, True) == 'p=' + expected + '/bucket-2' + finally: + context.undo() + system_time.tzset() + + +@pytest.mark.skipif(not hasattr(system_time, 'tzset'), reason='requires POSIX timezone support') +@pytest.mark.parametrize('planner', _PLANNERS) +@pytest.mark.parametrize('local_zone', ['UTC', 'Asia/Shanghai']) +@pytest.mark.parametrize('legacy,unit,micros,fraction', [ + (False, 'ms', 0, '.000'), (False, 'us', 120100, '.120100'), + (True, 'ms', 120000, '.120'), (True, 'us', 120100, '.120100'), +]) +def test_java_ltz_bucket_dv_without_external_path(tmp_path, planner, local_zone, legacy, unit, micros, fraction): + if legacy: + canonical_name = '2026-09-15T20%3A00%3A00' + fraction + elif local_zone == 'UTC': + canonical_name = '2026-09-15 20%3A00%3A00' + fraction + else: + canonical_name = '2026-09-16 04%3A00%3A00' + fraction + # A fresh process gives Python and Rust the same default timezone without + # changing Rust's process-global timezone cache between test cases. + script = ''' +import sys +from datetime import datetime, timezone +from pathlib import Path +import pyarrow as pa +from pypaimon.tests.deletion_vector_path_test import _check_java_bucket_dv +_check_java_bucket_dv( + Path(sys.argv[1]), sys.argv[2], + datetime(2026, 9, 15, 20, 0, 0, int(sys.argv[4]), tzinfo=timezone.utc), + pa.timestamp(sys.argv[3], tz='UTC'), sys.argv[5], sys.argv[6] == 'True') +''' + result = subprocess.run( + [sys.executable, '-c', script, str(tmp_path), planner, unit, str(micros), canonical_name, str(legacy)], + env=dict(os.environ, TZ=local_zone), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) + assert result.returncode == 0, result.stdout + + def _check_java_bucket_dv(tmp_path, planner, value, partition_type, canonical_name, legacy_partition_name): table = _table(tmp_path, 'bucket', value, partition_type, legacy_partition_name) _delete(table, [0]) diff --git a/paimon-python/pypaimon/utils/file_store_path_factory.py b/paimon-python/pypaimon/utils/file_store_path_factory.py index 6136af727b52..d712f7e81528 100644 --- a/paimon-python/pypaimon/utils/file_store_path_factory.py +++ b/paimon-python/pypaimon/utils/file_store_path_factory.py @@ -16,14 +16,15 @@ # under the License. import struct -from datetime import date +from datetime import date, timezone from decimal import Decimal from typing import List, Optional, Tuple -from pypaimon.casting.row_to_string import cast_value_to_string, _is_unsupported +from pypaimon.casting.row_to_string import cast_value_to_string, _format_timestamp, _is_unsupported from pypaimon.common.external_path_provider import ExternalPathProvider from pypaimon.schema.data_types import DataType from pypaimon.table.bucket_mode import BucketMode +from pypaimon.table.row.generic_row import _is_ltz_type, _normalize_ltz, _parse_type_precision_scale def _is_null_or_whitespace_only(value) -> bool: @@ -156,6 +157,12 @@ def _canonical_partition(self, partition: Tuple) -> Tuple[str, ...]: for i, value in enumerate(partition): data_type = self.partition_types[i] if self.partition_types is not None else None type_name = str(data_type).split('(', 1)[0].split()[0] + if value is not None and _is_ltz_type(str(data_type).upper()): + # Legacy Timestamp.toString() uses UTC fields. Java's non-legacy + # cast uses TimeZone.getDefault(); neither includes an offset. + value = _normalize_ltz(value) + if not self.legacy_partition_name: + value = value.replace(tzinfo=timezone.utc).astimezone().replace(tzinfo=None) if _is_null_or_whitespace_only(value): text = self.default_part_value elif type_name in ('FLOAT', 'REAL', 'DOUBLE'): @@ -168,6 +175,9 @@ def _canonical_partition(self, partition: Tuple) -> Tuple[str, ...]: text = value.isoformat(timespec='microseconds' if value.microsecond else 'seconds') if value.microsecond and value.microsecond % 1000 == 0: text = text[:-3] + elif type_name.startswith('TIMESTAMP'): + precision, _ = _parse_type_precision_scale(data_type) + text = _format_timestamp(value, precision) elif self.legacy_partition_name and type_name.startswith('TIME'): text = str(((value.hour * 60 + value.minute) * 60 + value.second) * 1000 + value.microsecond // 1000) From 3fc97e24f06e774bb0a788fdd9b27448321c778f Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Tue, 15 Sep 2026 13:53:39 +0800 Subject: [PATCH 6/6] [python] Fix native plan cache assertions and test interleaved shards --- .../tests/multimodal_temporal_test.py | 10 ++--- .../tests/native_plan_integration_test.py | 39 +++++++++++++++++++ 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/paimon-python/pypaimon/tests/multimodal_temporal_test.py b/paimon-python/pypaimon/tests/multimodal_temporal_test.py index 5ee34556d4d1..8523a377bbbe 100644 --- a/paimon-python/pypaimon/tests/multimodal_temporal_test.py +++ b/paimon-python/pypaimon/tests/multimodal_temporal_test.py @@ -29,7 +29,7 @@ from pypaimon.multimodal import temporal from pypaimon.catalog.table_query_auth import TableQueryAuthResult from pypaimon.read.reader.format_pyarrow_reader import FormatPyArrowReader -from pypaimon.read.scanner.file_scanner import FileScanner +from pypaimon.read.table_scan import TableScan class MultimodalTemporalTest(unittest.TestCase): @@ -2065,15 +2065,15 @@ def test_alignment_reuses_payload_scan_plans_across_batches(self): on="event_time", by="episode_id", direction="nearest", tolerance=0, ) - original_scan = FileScanner.scan + original_plan = TableScan.plan with mock.patch.object( - FileScanner, "scan", autospec=True, - side_effect=original_scan) as scan: + TableScan, "plan", autospec=True, + side_effect=original_plan) as plan: reader = aligned.to_arrow_batch_reader(batch_size=1) self.assertEqual(8, sum(batch.num_rows for batch in reader)) - self.assertEqual(4, scan.call_count) + self.assertEqual(4, plan.call_count) def test_empty_source_stays_pinned_after_first_append(self): anchors = self._table("pinned_empty_anchors", { diff --git a/paimon-python/pypaimon/tests/native_plan_integration_test.py b/paimon-python/pypaimon/tests/native_plan_integration_test.py index 19138a8052a7..f275325eb2c3 100644 --- a/paimon-python/pypaimon/tests/native_plan_integration_test.py +++ b/paimon-python/pypaimon/tests/native_plan_integration_test.py @@ -137,6 +137,45 @@ def test_append_matches_normal_plan(self): self._write('ap_t', [{'k': 3, 'v': 'c'}]) self._assert_matches('ap_t') + def test_append_distribution_matches_interleaved_partition_buckets(self): + self.schema = pa.schema([('k', pa.int64()), ('v', pa.string()), ('p', pa.string())]) + self.cat.create_table('default.interleaved_t', Schema.from_pyarrow_schema( + self.schema, partition_keys=['p'], options={'bucket': '5', 'bucket-key': 'k'}), False) + partitions = ['p1', 'p1', 'p2', 'p1', 'p2', 'p1', 'p2', 'p1', 'p2', 'p1', 'p2', 'p1', 'p2', 'p1'] + self._write('interleaved_t', [ + {'k': 1001 + i, 'v': 'first', 'p': partition} for i, partition in enumerate(partitions)]) + self._write('interleaved_t', [ + {'k': 1005 + i, 'v': 'second', 'p': partition} + for i, partition in enumerate(['p2', 'p1', 'p2', 'p2'])]) + + def read(native, shard=None, slice_=None, limit=None): + table = self.cat.get_table('default.interleaved_t').copy( + {'scan.native-plan.enabled': str(native).lower()}) + builder = table.new_read_builder() + if limit is not None: + builder.with_limit(limit) + scan = builder.new_scan() + if shard is not None: + scan.with_shard(*shard) + if slice_ is not None: + scan.with_slice(*slice_) + if native: + with patch.object(scan.file_scanner, 'scan', side_effect=AssertionError('native plan fell back')): + plan = scan.plan() + else: + plan = scan.plan() + # Parallel reads with a limit can return any subset; compare planned order serially. + rows = builder.new_read().to_arrow(plan.splits(), parallelism=1).to_pylist() + return plan.snapshot_id, sorted(rows, key=lambda row: (row['k'], row['v'], row['p'])) + + selections = [{'shard': (i, 3)} for i in range(3)] + [ + {'slice_': (0, 6)}, {'slice_': (4, 10)}, {'slice_': (10, 18)}, + {'slice_': (0, 99)}, {'shard': (1, 3), 'limit': 2}, {'slice_': (4, 10), 'limit': 2}, + ] + for selection in selections: + with self.subTest(selection=selection): + self.assertEqual(read(False, **selection), read(True, **selection)) + @unittest.skipUnless(native_family_search_modes_available(), "pypaimon-rust 0.4+ required") def test_dynamic_family_search_mode_uses_native_plan(self):