Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions dataflow/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -405,11 +405,17 @@ def text2model_init(cache: Path = typer.Option(Path("."), help="Cache dir")):
@text_app.command("train")
def text2model_train(input_dir: Path = typer.Argument(Path("."), help="Input directory"),
input_keys: Optional[str] = typer.Option(None, help="Input keys"),
lf_yaml: Optional[Path] = typer.Option(None, help="LlamaFactory yaml")):
lf_yaml: Optional[Path] = typer.Option(None, help="LlamaFactory yaml"),
cache: Path = typer.Option(Path("."), help="Cache dir")):
try:
from dataflow.cli_funcs.cli_text import cli_text2model_train # type: ignore
lf = str(lf_yaml) if lf_yaml else "./.cache/train_config.yaml"
ok = cli_text2model_train(input_keys=input_keys, lf_yaml=lf, input_dir=str(input_dir))
lf = str(lf_yaml) if lf_yaml else f"{cache}/.cache/train_config.yaml"
ok = cli_text2model_train(
input_keys=input_keys,
lf_yaml=lf,
input_dir=str(input_dir),
cache_path=str(cache),
)
except Exception as e:
_echo(f"text2model train error: {e}", "red")
raise typer.Exit(code=1)
Expand Down Expand Up @@ -439,4 +445,4 @@ def webui(
raise typer.Exit(code=1)

if __name__ == "__main__":
app()
app()
17 changes: 13 additions & 4 deletions dataflow/cli_funcs/cli_text.py
Original file line number Diff line number Diff line change
Expand Up @@ -350,7 +350,12 @@ def cli_text2model_init(cache_path: str = "./") -> bool:
return False


def cli_text2model_train(input_keys: str = None, lf_yaml: str = "./.cache/train_config.yaml", input_dir: str = "./") -> bool:
def cli_text2model_train(
input_keys: str = None,
lf_yaml: str = None,
input_dir: str = "./",
cache_path: str = "./",
) -> bool:
"""
Start Text2Model training using complete pipeline
"""
Expand All @@ -359,7 +364,12 @@ def cli_text2model_train(input_keys: str = None, lf_yaml: str = "./.cache/train_
print(f"Processing fields: {input_keys}")

current_dir = Path(os.getcwd())
config_path_obj = Path(lf_yaml)

cache_path_obj = Path(cache_path)
if not cache_path_obj.is_absolute():
cache_path_obj = current_dir / cache_path_obj

config_path_obj = Path(lf_yaml) if lf_yaml else cache_path_obj / ".cache" / "train_config.yaml"
if not config_path_obj.is_absolute():
config_path_obj = current_dir / config_path_obj

Expand All @@ -371,7 +381,6 @@ def cli_text2model_train(input_keys: str = None, lf_yaml: str = "./.cache/train_
print("Run 'dataflow text2model init' first")
return False

cache_path_obj = current_dir
input_path = Path(input_dir)
if not input_path.is_absolute():
input_path = current_dir / input_path
Expand Down Expand Up @@ -406,7 +415,7 @@ def cli_text2model_train(input_keys: str = None, lf_yaml: str = "./.cache/train_
# Step 2: Text2QA Pipeline
print(f"{Fore.CYAN}Step 2: Text2QA generation...{Style.RESET_ALL}")

script2_path = cache_path_obj / "text_to_qa_pipeline.py"
script2_path = current_dir / "text_to_qa_pipeline.py"
args2 = ["--cache", str(cache_path_obj)]
if not run_script_with_args(script2_path, "Text2QA generation", args2, cwd=str(current_dir)):
print(f"{Fore.RED}❌ Step 2: Text2QA generation failed{Style.RESET_ALL}")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -222,9 +222,7 @@ def check_environment(self):
"""Check training environment"""
print("Checking training environment...")

# 使用绝对路径 - 直接获取调用者工作目录
caller_cwd = Path(os.environ.get('PWD', os.getcwd()))
data_dir = caller_cwd / ".cache" / "data"
data_dir = self.cache_path / ".cache" / "data"

# Check data files with detailed info
data_files = [
Expand Down
300 changes: 300 additions & 0 deletions dataflow/cli_funcs/text2model_pipeline/merge_json_jsonl.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,300 @@
#!/usr/bin/env python3
import json
import os
import argparse
from pathlib import Path


def extract_text_from_json(json_file_path):
"""从JSON/JSONL文件中提取文本内容"""
try:
with open(json_file_path, 'r', encoding='utf-8') as f:
if json_file_path.suffix == '.jsonl':
# 处理JSONL文件(每行一个JSON对象)
texts = []
for line_num, line in enumerate(f, 1):
line = line.strip()
if not line:
continue
try:
data = json.loads(line)
text_content = extract_text_from_data(data)
if text_content:
texts.append(text_content)
except json.JSONDecodeError as e:
print(f" [WARN] Invalid JSON at line {line_num} in {json_file_path}: {e}")
continue
return texts
else:
# 处理标准JSON文件
data = json.load(f)
text_content = extract_text_from_data(data)
return [text_content] if text_content else []

except Exception as e:
print(f" [ERROR] Error reading {json_file_path}: {e}")
return []


def extract_text_from_data(data):
"""从JSON数据中提取文本内容,保持原始状态"""
text_fields = ['text', 'content', 'body', 'raw_text', 'message', 'description', 'summary']

def clean_text(text):
"""最小化处理,保持原始文本"""
if not isinstance(text, str):
return None

text = text.strip()
if not text:
return None

return text # 直接返回,不做任何清理或截断

if isinstance(data, dict):
# 查找文本字段
for field in text_fields:
if field in data and data[field]:
content = data[field]
if isinstance(content, str):
cleaned = clean_text(content)
if cleaned:
return cleaned
elif isinstance(content, (list, dict)):
# 递归处理嵌套结构
nested_text = extract_text_from_data(content)
if nested_text:
return clean_text(nested_text)

# 如果没找到标准字段,尝试查找所有字符串值
for key, value in data.items():
if isinstance(value, str) and len(value.strip()) > 50:
cleaned = clean_text(value)
if cleaned:
return cleaned

elif isinstance(data, list):
# 如果是列表,处理每个元素
texts = []
for item in data:
if isinstance(item, dict):
text_content = extract_text_from_data(item)
if text_content:
texts.append(text_content)
elif isinstance(item, str):
cleaned = clean_text(item)
if cleaned:
texts.append(cleaned)

if texts:
combined = "\n\n".join(texts)
return clean_text(combined)

elif isinstance(data, str):
return clean_text(data)

return None


def create_text_files(json_files, cache_path_obj):
"""将JSON文件中的文本内容转换为独立的文本文件,保持原始格式"""
temp_dir = cache_path_obj / ".cache" / "gpu" / "temp_texts"
temp_dir.mkdir(parents=True, exist_ok=True)

text_paths = []
processed_count = 0

print("\nExtracting text content from JSON files...")
print("-" * 50)

for i, json_file in enumerate(json_files):
print(f"Processing: {json_file.name}")

# 提取文本内容
text_contents = extract_text_from_json(json_file)

if not text_contents:
print(f" [WARN] No valid text content found in {json_file}")
continue

# 为每个文本内容创建文件
for j, text_content in enumerate(text_contents):
if not text_content or not isinstance(text_content, str) or not text_content.strip():
continue

# 最终验证文本(最小验证)
final_text = validate_and_clean_text(text_content)
if not final_text:
print(" [WARN] Text content failed validation")
continue

# 创建文本文件名
base_name = json_file.stem
if len(text_contents) > 1:
temp_file = temp_dir / f"{base_name}_{j}.txt"
else:
temp_file = temp_dir / f"{base_name}.txt"

try:
with open(temp_file, 'w', encoding='utf-8') as tf:
tf.write(final_text)

# 验证写入的文件
if verify_text_file(temp_file):
text_paths.append(str(temp_file))
processed_count += 1
print(f" [OK] Created: {temp_file.name} ({len(final_text)} chars)")
else:
print(f" [ERROR] Failed validation: {temp_file.name}")
temp_file.unlink() # 删除无效文件

except Exception as e:
print(f" [ERROR] Error creating {temp_file}: {e}")

print(f"\n[OK] Successfully processed {processed_count} text contents from {len(json_files)} JSON files")
return text_paths


def validate_and_clean_text(text):
"""完全保持原始文本,不做任何验证或清理"""
if not isinstance(text, str):
return None

if not text:
return None

return text # 完全保持原始文本


def verify_text_file(file_path):
"""验证文本文件是否可以被正确读取"""
try:
with open(file_path, 'r', encoding='utf-8') as f:
content = f.read()

# 基本检查
if not content or not content.strip():
return False

return True

except Exception as e:
print(f" [WARN] File verification failed: {e}")
return False


def scan_json_files(base_dir, cache_path="./", mode="paths"):
"""
扫描目录下的JSON/JSONL文件,排除.cache目录
mode: 'paths' - 创建文件路径索引, 'texts' - 提取文本内容
"""
base_path = Path(base_dir)
cache_path_obj = Path(cache_path)
if not cache_path_obj.is_absolute():
caller_cwd = Path(os.environ.get('PWD', os.getcwd()))
cache_path_obj = caller_cwd / cache_path_obj

# 确保缓存目录存在
gpu_cache_dir = cache_path_obj / ".cache" / "gpu"
gpu_cache_dir.mkdir(parents=True, exist_ok=True)

text_input_file = gpu_cache_dir / "text_input.jsonl"

print(f"Scanning directory: {base_path}")
print(f"Output file: {text_input_file}")
print(f"Mode: {'Extract text content' if mode == 'texts' else 'File paths only'}")
print("-" * 50)

json_files = []

# 遍历所有文件和目录
for root, dirs, files in os.walk(base_path):
# 排除.cache目录
dirs[:] = [d for d in dirs if not d.startswith('.cache')]

# 检查每个文件
for file in files:
if file.endswith(('.json', '.jsonl')):
file_path = Path(root) / file
json_files.append(file_path.absolute())
print(f"Found: {file_path}")

if not json_files:
print("No JSON/JSONL files found!")
return False

print(f"\nTotal found: {len(json_files)} files")

# 根据模式处理
if mode == "texts":
print("Mode: Extract text content and create text files...")
text_paths = create_text_files(json_files, cache_path_obj)

if not text_paths:
print("[ERROR] No valid text content found in any JSON files!")
return False

# 创建text_input.jsonl文件,包含文本文件路径
try:
with open(text_input_file, 'w', encoding='utf-8') as f:
for text_path in text_paths:
record = {"text_path": text_path}
f.write(json.dumps(record, ensure_ascii=False) + '\n')

print(f"[OK] Created: {text_input_file}")
print(f"[OK] Contains {len(text_paths)} text file paths")
return True

except Exception as e:
print(f"[ERROR] Error creating file: {e}")
return False

else:
print("Mode: Create file path index...")
# 创建text_input.jsonl文件,包含JSON文件路径
try:
with open(text_input_file, 'w', encoding='utf-8') as f:
for json_file in json_files:
record = {"text_path": str(json_file)}
f.write(json.dumps(record, ensure_ascii=False) + '\n')

print(f"[OK] Created: {text_input_file}")
print(f"[OK] Contains {len(json_files)} file paths")
return True

except Exception as e:
print(f"[ERROR] Error creating file: {e}")
return False


def main():
parser = argparse.ArgumentParser(description='Scan JSON/JSONL files and create input for text processing pipeline')
parser.add_argument('input_dir', nargs='?', default='./', help='Input directory to scan (default: ./)')
parser.add_argument('--cache', default='./', help='Cache directory path (default: ./)')
parser.add_argument('--mode', choices=['paths', 'texts'], default='texts',
help='Processing mode: "paths" for file paths only, "texts" for extract text content (default: texts)')

args = parser.parse_args()

print("Enhanced JSON/JSONL File Scanner & Text Processor")
print("=" * 60)

success = scan_json_files(args.input_dir, args.cache, args.mode)

if success:
print("[OK] Processing completed successfully!")
if args.mode == 'texts':
print("Ready for CorpusTextSplitterBatch processing with extracted text files.")
print("\nNext steps:")
print("1. Run your text2qa pipeline")
print("2. The pipeline will process the text files automatically")
else:
print("Ready for CorpusTextSplitterBatch processing with JSON file paths.")
print("\nNote: Make sure your JSON files contain text fields like 'text', 'content', or 'body'")
else:
print("[ERROR] Processing failed!")
raise SystemExit(1)


if __name__ == "__main__":
main()
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ def __init__(self, cache_base="./"):
)

self.extract_format_qa = QAExtractor(
output_json_file="./.cache/data/qa.json",
output_json_file=str(cache_path / ".cache" / "data" / "qa.json"),
)

def forward(self):
Expand Down
Loading
Loading