Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 64 additions & 24 deletions dataflow/utils/storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -1148,22 +1148,28 @@ def get_keys_from_dataframe(self) -> list[str]:

elif ext == "jsonl":
with open(file_path, 'r', encoding='utf-8') as f:
line = f.readline()
if line:
return list(json.loads(line).keys())
for line in f:
if line.strip():
return list(json.loads(line).keys())
return []

elif ext == "parquet":
# Parquet 格式在 Footer 存有 Schema,读取开销几乎为零
# Restore pandas columns, excluding stored indexes, without reading rows.
import pyarrow.parquet as pq
return pq.ParquetFile(file_path).schema.names
with pq.ParquetFile(file_path) as parquet_file:
return parquet_file.schema_arrow.empty_table().to_pandas().columns.tolist()
elif ext in ("json", "pickle"):
return self._load_local_file(file_path, ext).columns.tolist()

except pd.errors.EmptyDataError:
return []
except Exception as e:
self.logger.warning(f"Failed to read header from {file_path} directly: {e}. Falling back to read().")
self.logger.warning(f"Failed to read column metadata from {file_path}: {e}. Returning no columns.")
return []

def _load_local_file(self, file_path: str, file_type: str,batchsize: int=None) -> pd.DataFrame:
"""Load data from local file based on file type."""
file_type = file_type.lower()
if file_type == "jsonl" and self.operator_step == 0 and self.jsonl_read_policy is not None:
if batchsize is not None:
return self.jsonl_read_policy.iter_chunks(file_path, batchsize)
Expand All @@ -1178,32 +1184,47 @@ def _load_local_file(self, file_path: str, file_type: str,batchsize: int=None) -
elif file_type == "csv":
return pd.read_csv(file_path, chunksize=batchsize)
return super()._load_local_file(file_path, file_type)
except pd.errors.EmptyDataError:
return pd.DataFrame()
except Exception as e:
raise ValueError(f"Failed to load {file_type} file: {str(e)}")

def get_record_count(self) -> int:
"""显式获取记录总数,支持缓存以提高性能"""
"""Count logical records, caching only while the current input is unchanged."""
if self.operator_step == 0 and self.first_entry_file_name.lower().endswith(".jsonl") and self.jsonl_read_policy is not None:
return self.jsonl_read_policy.inspect(self.first_entry_file_name)["accepted"]
if hasattr(self, "_cached_record_count"):
return self._cached_record_count

file_path = self._get_cache_file_path(self.operator_step)
local_cache = self.cache_type if self.operator_step != 0 else file_path.split(".")[-1]
local_cache = self.cache_type if self.operator_step != 0 else file_path.split(".")[-1].lower()
stat = os.stat(file_path)
identity = (
self.operator_step, os.path.realpath(file_path), local_cache,
stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns,
)
if getattr(self, "_record_count_identity", None) == identity:
return self._cached_record_count

if local_cache == "parquet":
import pyarrow.parquet as pq
count = pq.ParquetFile(file_path).metadata.num_rows
elif local_cache in ["jsonl", "csv"]:
count = 0
with open(file_path, 'rb') as f:
for _ in f: count += 1
if local_cache == "csv": count -= 1 # 扣除表头
with pq.ParquetFile(file_path) as parquet_file:
count = parquet_file.metadata.num_rows
elif local_cache in ("jsonl", "csv"):
try:
reader = (
pd.read_csv(file_path, chunksize=10000)
if local_cache == "csv"
else pd.read_json(file_path, lines=True, chunksize=10000)
)
with reader:
count = sum(len(chunk) for chunk in reader)
except pd.errors.EmptyDataError:
count = 0
else:
df = self._load_local_file(file_path, local_cache)
count = len(df)

self._cached_record_count = count
self._record_count_identity = identity
return count

def read(self, output_type: Literal["dataframe", "dict"]="dataframe") -> Any:
Expand All @@ -1227,17 +1248,36 @@ def iter_batches(self) -> Generator[pd.DataFrame, None, None]:

def iter_chunks(self) -> Generator[pd.DataFrame, None, None]:
"""
专门供 Pipeline 调用:产生原始的 DataFrame 流。
Yield DataFrame batches, materializing fallback formats before slicing.

JSONL and CSV use chunked readers. JSON, Parquet and Pickle retain the
inherited full-file loader, but still expose batches of at most batch_size.
"""
if (
not isinstance(self.batch_size, int)
or isinstance(self.batch_size, bool)
or self.batch_size < 1
):
raise ValueError("batch_size must be a positive integer")
file_path = self._get_cache_file_path(self.operator_step)
local_cache = self.cache_type if self.operator_step != 0 else file_path.split(".")[-1]
local_cache = self.cache_type if self.operator_step != 0 else file_path.split(".")[-1].lower()

# 这里的 _load_local_file 必须确保在 batch_size 不为 None 时返回迭代器
reader = self._load_local_file(file_path, local_cache, self.batch_size)

# 确保 reader 是可迭代的
if hasattr(reader, "__iter__") or isinstance(reader, pd.io.json._json.JsonReader):
if isinstance(reader, pd.DataFrame):
locations = reader.attrs.get("dataflow_source_locations")
for start in range(0, len(reader), self.batch_size):
frame = reader.iloc[start:start + self.batch_size].reset_index(drop=True)
if locations is not None:
frame.attrs["dataflow_source_locations"] = locations[start:start + self.batch_size]
yield frame
return

try:
for chunk in reader:
yield chunk
else:
yield reader # 如果不支持流式,就产出整个 DF
if len(chunk):
yield chunk
finally:
close = getattr(reader, "close", None)
if close is not None:
close()
232 changes: 232 additions & 0 deletions test/cpu_only/test_stream_storage_record_contract.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,232 @@
"""Streaming storage must expose batches and metadata as logical records."""

import pandas as pd
import pytest

from dataflow.core import OperatorABC
from dataflow.pipeline import StreamBatchedPipelineABC
from dataflow.utils.storage import StreamBatchedFileStorage


pytestmark = pytest.mark.cpu


def write_source(path, frame):
extension = path.suffix.lower()
if extension == ".json":
frame.to_json(path, orient="records")
elif extension == ".pickle":
frame.to_pickle(path)
elif extension == ".parquet":
pytest.importorskip("pyarrow")
frame.to_parquet(path, index=False)
elif extension == ".jsonl":
frame.to_json(path, orient="records", lines=True)
else:
frame.to_csv(path, index=False)


def storage_for(path, tmp_path):
return StreamBatchedFileStorage(
str(path), cache_path=str(tmp_path / "cache")
).step()


@pytest.mark.parametrize("extension", ["json", "pickle", "parquet", "csv", "jsonl"])
@pytest.mark.parametrize("uppercase", [False, True], ids=["lowercase", "uppercase"])
def test_batches_headers_and_counts_describe_the_same_records(tmp_path, extension, uppercase):
frame = pd.DataFrame({"value": [1, 2, 3], "text": ["one", "two", "three"]})
source = tmp_path / f"source.{extension.upper() if uppercase else extension}"
write_source(source, frame)
storage = storage_for(source, tmp_path)
storage.batch_size = 2

batches = list(storage.iter_batches())
assert all(isinstance(batch, pd.DataFrame) for batch in batches)
assert [len(batch) for batch in batches] == [2, 1]
assert storage.get_keys_from_dataframe() == ["value", "text"]
assert storage.get_record_count() == 3
assert pd.concat(batches).to_dict(orient="records") == frame.to_dict(orient="records")
assert storage.read("dataframe").to_dict(orient="records") == frame.iloc[:2].to_dict(orient="records")


@pytest.mark.parametrize("size", [None, 0, -1, 1.5, True])
def test_invalid_batch_size_is_rejected(tmp_path, size):
source = tmp_path / "source.json"
write_source(source, pd.DataFrame({"value": [1]}))
storage = storage_for(source, tmp_path)
storage.batch_size = size
with pytest.raises(ValueError, match="batch_size"):
list(storage.iter_batches())


def test_parquet_header_contains_top_level_fields(tmp_path):
source = tmp_path / "source.parquet"
write_source(source, pd.DataFrame({"nested": [{"leaf": 1}], "value": [2]}))
assert storage_for(source, tmp_path).get_keys_from_dataframe() == ["nested", "value"]


@pytest.mark.parametrize(
"index",
[
pd.Index([10, 20]),
pd.Index([10, 20], name="row_id"),
pd.MultiIndex.from_tuples(
[("a", 10), ("b", 20)], names=["group", "row_id"]
),
],
ids=["unnamed", "named", "multiindex"],
)
def test_parquet_header_excludes_persisted_index(tmp_path, index):
pytest.importorskip("pyarrow")
source = tmp_path / "source.parquet"
frame = pd.DataFrame({"value": [1, 2]}, index=index)
frame.to_parquet(source)
storage = storage_for(source, tmp_path)
storage.batch_size = 1

assert storage.get_keys_from_dataframe() == frame.columns.tolist()
assert all(batch.columns.tolist() == ["value"] for batch in storage.iter_batches())
index_key = index.names[-1] or "__index_level_0__"
pipeline = StreamRecordsPipeline(source, tmp_path / "pipeline", input_key=index_key)
with pytest.raises(KeyError, match=index_key):
pipeline.compile()


def test_csv_count_uses_logical_records_and_skips_blank_lines(tmp_path):
source = tmp_path / "source.csv"
source.write_text('value,text\n1,"first\ncontinued"\n\n2,second\n', encoding="utf-8")
storage = storage_for(source, tmp_path)
storage.batch_size = 1
assert storage.get_record_count() == 2
assert sum(len(frame) for frame in storage.iter_batches()) == 2


@pytest.mark.parametrize("contents", ["", " \n\t\n", "value,text\n"])
def test_empty_csv_has_zero_records(tmp_path, contents):
source = tmp_path / "source.csv"
source.write_text(contents, encoding="utf-8")
storage = storage_for(source, tmp_path)
storage.batch_size = 1
assert storage.get_record_count() == 0
expected_keys = ["value", "text"] if contents.startswith("value") else []
assert storage.get_keys_from_dataframe() == expected_keys
assert list(storage.iter_batches()) == []
assert storage.read("dataframe").empty


def test_malformed_csv_is_not_treated_as_empty(tmp_path):
source = tmp_path / "source.csv"
source.write_text('value,text\n1,"unterminated\n', encoding="utf-8")
storage = storage_for(source, tmp_path)
storage.batch_size = 1
with pytest.raises(ValueError):
list(storage.iter_batches())
with pytest.raises(pd.errors.ParserError):
storage.get_record_count()


@pytest.mark.parametrize("prefix", ["", "\n", "\n \t\n"])
def test_jsonl_metadata_skips_blank_lines_like_reader(tmp_path, prefix):
source = tmp_path / "source.jsonl"
source.write_text(prefix + '{"value": 1}\n\n{"value": 2}\n', encoding="utf-8")
storage = storage_for(source, tmp_path)
storage.batch_size = 1
assert storage.get_keys_from_dataframe() == ["value"]
assert storage.get_record_count() == 2
batches = list(storage.iter_batches())
assert all(len(frame) for frame in batches)
assert sum(len(frame) for frame in batches) == 2


def test_closing_batch_iterator_closes_csv_reader(tmp_path, monkeypatch):
source = tmp_path / "source.csv"
write_source(source, pd.DataFrame({"value": [1, 2, 3]}))
storage = storage_for(source, tmp_path)
storage.batch_size = 1
readers = []
read_csv = pd.read_csv

def capture_reader(*args, **kwargs):
reader = read_csv(*args, **kwargs)
readers.append(reader)
return reader

monkeypatch.setattr(pd, "read_csv", capture_reader)
batches = storage.iter_batches()
assert next(batches)["value"].tolist() == [1]
assert not readers[0].handles.handle.closed
batches.close()
assert readers[0].handles.handle.closed


def test_source_symlink_suffix_controls_format(tmp_path):
target = tmp_path / "records"
pd.DataFrame({"value": [1, 2]}).to_csv(target, index=False)
source = tmp_path / "source.csv"
try:
source.symlink_to(target)
except OSError:
pytest.skip("Creating symbolic links is unavailable")
storage = storage_for(source, tmp_path)
storage.batch_size = 1
assert storage.get_record_count() == 2
assert sum(len(frame) for frame in storage.iter_batches()) == 2


def test_count_changes_when_current_storage_step_changes(tmp_path):
source = tmp_path / "source.csv"
write_source(source, pd.DataFrame({"value": [1, 2, 3]}))
storage = storage_for(source, tmp_path)
assert storage.get_record_count() == 3
storage.write(pd.DataFrame({"value": [1]}))
storage.step()
assert storage.get_record_count() == 1


def test_count_cache_is_invalidated_after_same_file_changes(tmp_path):
source = tmp_path / "source.csv"
write_source(source, pd.DataFrame({"value": [1, 2]}))
storage = storage_for(source, tmp_path)
assert storage.get_record_count() == 2
pd.DataFrame({"value": [3]}).to_csv(source, index=False, header=False, mode="a")
assert storage.get_record_count() == 3


class ScaleRecords(OperatorABC):
def __init__(self):
super().__init__()
self.batch_lengths = []

def run(self, storage, input_key="value", output_key="result"):
frame = storage.read("dataframe").copy()
self.batch_lengths.append(len(frame))
frame[output_key] = frame[input_key] * 2
storage.write(frame)


class StreamRecordsPipeline(StreamBatchedPipelineABC):
def __init__(self, source, cache, input_key="value"):
super().__init__()
self.storage = StreamBatchedFileStorage(str(source), cache_path=str(cache))
self.scale = ScaleRecords()
self.input_key = input_key

def forward(self):
self.scale.run(self.storage.step(), input_key=self.input_key, output_key="result")


@pytest.mark.parametrize("extension", ["json", "pickle", "parquet", "jsonl"])
def test_compiled_stream_pipeline_accepts_source(tmp_path, extension):
source = tmp_path / f"source.{extension}"
write_source(source, pd.DataFrame({"value": [1, 2, 3]}))
if extension == "jsonl":
source.write_text("\n \t\n" + source.read_text(encoding="utf-8"), encoding="utf-8")
pipeline = StreamRecordsPipeline(source, tmp_path / "cache")
scale = pipeline.scale
pipeline.compile()
pipeline.forward(batch_size=2, resume_from_last=False)
output = pd.read_json(tmp_path / "cache" / "dataflow_cache_step_step1.jsonl", lines=True)
assert output["value"].tolist() == [1, 2, 3]
assert output["result"].tolist() == [2, 4, 6]
assert scale.batch_lengths == [2, 1]
Loading