Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 29 additions & 18 deletions dataflow/operators/core_text/generate/chunked_prompted_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ class ChunkedPromptedGenerator(OperatorABC):
- 若输入超过max_chunk_len,采用递归二分法切分;
- 从指定输入文件路径读取内容,生成结果保存至指定输出文件路径;
- 生成结果是以separator拼接的字符串。
- 生成失败或响应批次无效时抛出异常,不覆盖既有输出文件或发布成功缓存。
"""

def __init__(
Expand Down Expand Up @@ -43,6 +44,7 @@ def get_desc(lang: str = "zh"):
"采用递归二分方式进行chunk切分,确保每段不超过max_chunk_len tokens。"
"从给定的输入文件路径读取内容,生成结果保存至指定输出文件路径。"
"多个生成结果以separator拼接成最终输出字符串。"
"生成失败或响应批次无效时抛出异常,不覆盖既有输出文件。"
"输入参数:\n"
"- llm_serving:LLM服务对象,需实现LLMServingABC接口\n"
"- system_prompt:系统提示词,定义模型行为,默认为'You are a helpful agent.'\n"
Expand All @@ -59,6 +61,7 @@ def get_desc(lang: str = "zh"):
"Splits long text inputs into chunks using recursive bisection to ensure each chunk does not exceed max_chunk_len tokens."
"Reads content from specified input file paths and saves generated results to designated output file paths."
"Multiple generated results are joined as a string using the specified separator."
" Serving failures and invalid response batches raise before output files are modified."
"Input Parameters:\n"
"- llm_serving: LLM serving object implementing LLMServingABC interface\n"
"- system_prompt: System prompt to define model behavior, default is 'You are a helpful agent.'\n"
Expand Down Expand Up @@ -95,8 +98,6 @@ def run(
dataframe = storage.read("dataframe")
self.logger.info(f"Loaded DataFrame with {len(dataframe)} rows.")

all_generated_results = []

all_llm_inputs = []
row_chunk_map = [] # 记录每个row对应的chunk数量

Expand All @@ -116,33 +117,43 @@ def run(
self.logger.info(f"Total {len(all_llm_inputs)} chunks to generate")

try:
if self.json_schema:
if not all_llm_inputs:
all_responses = []
elif self.json_schema:
all_responses = self.llm_serving.generate_from_input(
all_llm_inputs, json_schema=self.json_schema
)
else:
all_responses = self.llm_serving.generate_from_input(all_llm_inputs)
except Exception as e:
self.logger.error(f"Global generation failed: {e}")
all_generated_results = [[] for _ in range(len(dataframe))]
else:
# === 按row重新划分responses ===
all_generated_results = []
idx = 0
for num_chunks in row_chunk_map:
if num_chunks == 0:
all_generated_results.append([])
else:
all_generated_results.append(all_responses[idx:idx + num_chunks])
idx += num_chunks

for (i, row), gen_results in zip(dataframe.iterrows(), all_generated_results):
raise

if not isinstance(all_responses, (list, tuple)):
raise TypeError("generate_from_input must return a list or tuple of strings")
if len(all_responses) != len(all_llm_inputs):
raise RuntimeError(
f"Response count mismatch: expected {len(all_llm_inputs)}, got {len(all_responses)}"
)
if not all(isinstance(response, str) for response in all_responses):
raise TypeError("Every generated response must be a string")

# Validate the entire batch before opening any destination file.
all_generated_results = []
idx = 0
for num_chunks in row_chunk_map:
all_generated_results.append(all_responses[idx:idx + num_chunks])
idx += num_chunks

output_paths = []
for (_, row), gen_results in zip(dataframe.iterrows(), all_generated_results):
source_path = Path(row[input_path_key])
output_path = str(source_path.with_name(f'{source_path.stem}_llm_output.txt'))
with open(output_path, 'w', encoding='utf-8') as f:
f.write(self.separator.join(gen_results))
dataframe.at[i, output_path_key] = output_path
output_paths.append(output_path)
dataframe[output_path_key] = output_paths

output_file = storage.write(dataframe)
self.logger.info(f"Generation complete. Output saved to {output_file}")
return output_path_key
return output_path_key
199 changes: 199 additions & 0 deletions test/cpu_only/test_chunked_generator_failures.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,199 @@
"""Generation failures must not publish empty or partially validated output."""

from pathlib import Path

import pandas as pd
import pytest
import tiktoken

from dataflow.utils.storage import DataFlowStorage, FileStorage

pytestmark = pytest.mark.cpu


class CharacterEncoder:
def encode(self, text):
return list(text)


class FakeServing:
def __init__(self, responses=None, error=None):
self.responses = responses
self.error = error
self.calls = []

def generate_from_input(self, user_inputs, **kwargs):
self.calls.append((list(user_inputs), kwargs))
if self.error is not None:
raise self.error
return self.responses


class MemoryStorage(DataFlowStorage):
def __init__(self, dataframe):
self.dataframe = dataframe.copy()
self.result = None

def get_keys_from_dataframe(self):
return self.dataframe.columns.tolist()

def read(self, output_type="dataframe"):
return self.dataframe.copy()

def write(self, dataframe):
self.result = dataframe.copy()
return "memory://chunked-results"


@pytest.fixture
def generator_type(monkeypatch):
# The public operator evaluates its default tokenizer during import. Avoid
# a tokenizer download; an injected encoder is part of its existing API.
with monkeypatch.context() as patch:
patch.setattr(tiktoken, "get_encoding", lambda name: CharacterEncoder())
from dataflow.operators.core_text import ChunkedPromptedGenerator
return ChunkedPromptedGenerator


def failure_inputs(tmp_path):
inputs = [tmp_path / "first.txt", tmp_path / "second.txt"]
for path, text in zip(inputs, ["abcd", "xy"]):
path.write_text(text, encoding="utf-8")
outputs = [path.with_name(f"{path.stem}_llm_output.txt") for path in inputs]
for path in outputs:
path.write_text("valuable previous output", encoding="utf-8")
metadata = tmp_path / "source.jsonl"
pd.DataFrame({"input_path": [str(path) for path in inputs]}).to_json(
metadata, orient="records", lines=True
)
storage = FileStorage(str(metadata), cache_path=str(tmp_path / "cache")).step()
return storage, inputs, outputs, metadata


def assert_unpublished(storage, inputs, outputs, metadata, original_metadata):
assert [path.read_text(encoding="utf-8") for path in inputs] == ["abcd", "xy"]
assert all(
path.read_text(encoding="utf-8") == "valuable previous output"
for path in outputs
)
assert metadata.read_bytes() == original_metadata
assert not Path(storage._get_cache_file_path(1)).exists()


def test_serving_exception_propagates_without_overwriting_outputs(
tmp_path, generator_type
):
storage, inputs, outputs, metadata = failure_inputs(tmp_path)
original_metadata = metadata.read_bytes()
failure = RuntimeError("offline serving failure")
serving = FakeServing(error=failure)
generator = generator_type(serving, enc=CharacterEncoder(), max_chunk_len=2)
with pytest.raises(RuntimeError) as raised:
generator.run(storage, "input_path", "output_path")
assert raised.value is failure
assert_unpublished(storage, inputs, outputs, metadata, original_metadata)


@pytest.mark.parametrize(
"responses,error_type",
[
(None, TypeError),
("abc", TypeError),
({0: "a", 1: "b", 2: "c"}, TypeError),
(lambda: (value for value in ["a", "b", "c"]), TypeError),
(["a", "b", None], TypeError),
(["a", "b", 123], TypeError),
(["a", "b"], RuntimeError),
(["a", "b", "c", "extra"], RuntimeError),
],
ids=[
"none",
"string",
"mapping",
"generator",
"late-none",
"late-number",
"short",
"long",
],
)
def test_invalid_response_batch_does_not_write_any_output(
tmp_path, generator_type, responses, error_type
):
storage, inputs, outputs, metadata = failure_inputs(tmp_path)
original_metadata = metadata.read_bytes()
if callable(responses):
responses = responses()
generator = generator_type(
FakeServing(responses), enc=CharacterEncoder(), max_chunk_len=2
)
with pytest.raises(error_type):
generator.run(storage, "input_path", "output_path")
assert_unpublished(storage, inputs, outputs, metadata, original_metadata)


@pytest.mark.parametrize("response_type", [list, tuple])
def test_valid_batches_preserve_chunk_alignment_prompts_and_schema(
tmp_path, generator_type, response_type
):
inputs = [tmp_path / "first.txt", tmp_path / "empty.txt", tmp_path / "third.txt"]
for path, text in zip(inputs, ["abcd", "", "xy"]):
path.write_text(text, encoding="utf-8")
metadata = tmp_path / "source.jsonl"
pd.DataFrame({"input_path": [str(path) for path in inputs]}).to_json(
metadata, orient="records", lines=True
)
storage = FileStorage(str(metadata), cache_path=str(tmp_path / "cache")).step()
schema = {"type": "string"}
serving = FakeServing(response_type(["first-a", "first-b", "", "third"]))
generator = generator_type(
serving,
enc=CharacterEncoder(),
max_chunk_len=2,
system_prompt="instruction",
json_schema=schema,
separator="|",
)
assert generator.run(storage, "input_path", "output_path") == "output_path"
assert serving.calls == [
(
["instruction\nab", "instruction\ncd", "instruction\n", "instruction\nxy"],
{"json_schema": schema},
)
]
result = pd.read_json(storage._get_cache_file_path(1), lines=True)
assert [
Path(path).read_text(encoding="utf-8") for path in result["output_path"]
] == ["first-a|first-b", "", "third"]
assert [path.read_text(encoding="utf-8") for path in inputs] == ["abcd", "", "xy"]


@pytest.mark.parametrize("index", [[10, 30], [7, 7]])
def test_output_metadata_aligns_by_row_position_even_with_custom_indices(
tmp_path, generator_type, index
):
inputs = [tmp_path / "first.txt", tmp_path / "second.txt"]
for path in inputs:
path.write_text("x", encoding="utf-8")
frame = pd.DataFrame({"input_path": [str(path) for path in inputs]}, index=index)
storage = MemoryStorage(frame)
generator = generator_type(FakeServing(["first", "second"]), enc=CharacterEncoder())
generator.run(storage, "input_path", "output_path")
expected = [str(path.with_name(f"{path.stem}_llm_output.txt")) for path in inputs]
assert storage.result.index.tolist() == index
assert storage.result["output_path"].tolist() == expected
assert [Path(path).read_text(encoding="utf-8") for path in expected] == [
"first",
"second",
]
pd.testing.assert_frame_equal(storage.dataframe, frame)


def test_zero_rows_skip_serving_and_publish_empty_output_metadata(generator_type):
storage = MemoryStorage(pd.DataFrame({"input_path": pd.Series(dtype=str)}))
serving = FakeServing(error=AssertionError("empty batch must not call serving"))
generator = generator_type(serving, enc=CharacterEncoder())
assert generator.run(storage, "input_path", "output_path") == "output_path"
assert serving.calls == []
assert storage.result.empty
assert storage.result.columns.tolist() == ["input_path", "output_path"]
Loading