#!/usr/bin/env python3 """No-API P0 fixture validation for extension-stage publish/template split.""" from __future__ import annotations import argparse import csv import json import shutil import sys from dataclasses import dataclass, field from datetime import datetime from pathlib import Path from typing import Any PROJECT_ROOT = Path(__file__).resolve().parents[1] if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) from gfa_core.extension_stage.manager import ExtensionStageManager # noqa: E402 DEFAULT_OUTPUT_ROOT = PROJECT_ROOT / "项目记录" / "拓展分析阶段_P0_fixture_validation_20260622" @dataclass(frozen=True) class FixtureCase: case_id: str description: str script_name: str script_content: str data_files: dict[str, bytes] original_command: str command_template: str output_contract: list[dict[str, Any]] input_roles: dict[str, Any] = field(default_factory=dict) input_bindings: dict[str, Any] = field(default_factory=dict) comparator_plan: dict[str, Any] = field(default_factory=dict) requirements: dict[str, Any] = field(default_factory=dict) expected_status: str = "published" def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--output-root", type=Path, default=DEFAULT_OUTPUT_ROOT) parser.add_argument("--run", action="store_true") parser.add_argument("--report", action="store_true") parser.add_argument("--clean", action="store_true") args = parser.parse_args() out = args.output_root.resolve() if args.clean and out.exists(): shutil.rmtree(out) out.mkdir(parents=True, exist_ok=True) if args.run or not args.report: run_all(out) if args.report: write_summary(out) return 0 def run_all(out: Path) -> None: manager = ExtensionStageManager( project_root=PROJECT_ROOT, store_root=out / "extension_stages", workspace_root=out / "workspace", input_root=out / "input", ) rows = [] for case in cases(): row = run_case(manager, out, case) rows.append(row) write_json(out / "summary.json", summarize(rows)) write_csv(out / "summary.csv", rows) def run_case(manager: ExtensionStageManager, out: Path, case: FixtureCase) -> dict[str, Any]: case_root = out / "cases" / case.case_id if case_root.exists(): shutil.rmtree(case_root) case_root.mkdir(parents=True, exist_ok=True) extension = prepare_extension(manager, case) extension_id = extension["extension_id"] row: dict[str, Any] = { "schema_version": "extension_p0_fixture_result_v1", "created_at": now(), "case_id": case.case_id, "description": case.description, "extension_id": extension_id, "expected_status": case.expected_status, "trial_status": "", "confirm_success": False, "validate_success": False, "golden_status": "", "template_gate_status": "", "publish_success": False, "published_run_success": False, "published_runnable": False, "template_eligible": False, "environment_failed": False, "stdout_captured": False, "first_failure_phase": "", "last_error": "", "success": False, } try: trial = manager.trial_run(extension_id) write_json(case_root / "trial_run.json", trial) trial_run = trial.get("run") or {} row["trial_status"] = str(trial_run.get("status") or "") row["environment_failed"] = row["trial_status"] == "environment_failed" row["last_error"] = str(trial_run.get("error") or "") row["stdout_captured"] = stdout_nonempty(trial_run) if case.expected_status == "environment_failed": row["success"] = row["environment_failed"] if not row["success"]: row["first_failure_phase"] = "trial" write_case_outputs(manager, extension_id, case_root, row) return row if case.expected_status == "stdout_only_failed": row["success"] = row["trial_status"] == "failed" and row["stdout_captured"] if not row["success"]: row["first_failure_phase"] = "trial" write_case_outputs(manager, extension_id, case_root, row) return row if not trial.get("success"): row["first_failure_phase"] = "trial" write_case_outputs(manager, extension_id, case_root, row) return row outputs = (trial.get("extension") or {}).get("outputs_detected") or trial_run.get("outputs") or [] confirm = manager.confirm(extension_id, outputs=outputs, input_bindings=case.input_bindings, nl_triggers=[case.case_id]) write_json(case_root / "confirm_response.json", confirm) row["confirm_success"] = bool(confirm.get("success")) validation = manager.validate(extension_id) write_json(case_root / "validate_response.json", validation) validation_run = validation.get("run") or {} golden = validation_run.get("golden_comparison") or {} gate = validation_run.get("template_gate") or {} row["validate_success"] = bool(validation.get("success")) row["golden_status"] = str(golden.get("status") or "") row["template_gate_status"] = str(gate.get("status") or "") if not row["validate_success"]: row["first_failure_phase"] = "validation" row["last_error"] = str(validation_run.get("error") or "") write_case_outputs(manager, extension_id, case_root, row) return row publish = manager.publish(extension_id) write_json(case_root / "publish_response.json", publish) published_extension = publish.get("extension") or {} checks = published_extension.get("publish_checks") or {} row["publish_success"] = bool(publish.get("success")) row["published_runnable"] = bool(checks.get("published_runnable")) row["template_eligible"] = bool(checks.get("template_eligible")) row["template_gate_status"] = str(checks.get("template_gate_status") or row["template_gate_status"]) if not row["publish_success"]: row["first_failure_phase"] = "publish" row["last_error"] = "; ".join(checks.get("errors") or []) write_case_outputs(manager, extension_id, case_root, row) return row job_id = f"{case.case_id}_job" materialize_job_input(manager, job_id, case.data_files) published_run = manager.run(extension_id=extension_id, job_id=job_id) write_json(case_root / "published_run_response.json", published_run) row["published_run_success"] = bool(published_run.get("success")) if not row["published_run_success"]: row["first_failure_phase"] = "published_run" row["last_error"] = str((published_run.get("run") or {}).get("error") or "") row["success"] = row["published_run_success"] and row["published_runnable"] if case.case_id == "generic_no_archetype": row["success"] = row["success"] and not row["template_eligible"] and row["template_gate_status"] == "not_applicable" write_case_outputs(manager, extension_id, case_root, row) return row except Exception as exc: row["last_error"] = str(exc) row["first_failure_phase"] = row.get("first_failure_phase") or "exception" write_json(case_root / "error.json", {"error": str(exc), "row": row}) write_case_outputs(manager, extension_id, case_root, row) return row def prepare_extension(manager: ExtensionStageManager, case: FixtureCase) -> dict[str, Any]: extension_id = "p0_" + safe_token(case.case_id) root = manager.store_root / extension_id if root.exists(): shutil.rmtree(root) raw_code = root / "raw_code" raw_data = root / "raw_test_data" converted = root / "converted_code" for path in (raw_code, raw_data, converted): path.mkdir(parents=True, exist_ok=True) (raw_code / case.script_name).write_text(case.script_content, encoding="utf-8") (converted / case.script_name).write_text(case.script_content, encoding="utf-8") (raw_code / case.script_name).chmod(0o755) (converted / case.script_name).chmod(0o755) for name, payload in case.data_files.items(): target = raw_data / name target.parent.mkdir(parents=True, exist_ok=True) target.write_bytes(payload) extension = { "schema_version": "extension_stage_v2", "extension_id": extension_id, "name": safe_token(case.case_id), "display_name": case.case_id, "description": case.description, "created_at": now(), "status": "draft", "runtime": {"type": "conda", "env": "test4py311", "language": "shell"}, "paths": {"root": str(root), "raw_code": "raw_code", "raw_test_data": "raw_test_data", "converted_code": "converted_code"}, "entrypoint": {"original_command": case.original_command, "command_template": case.command_template}, "command_template": case.command_template, "analysis_type": "generic_cli", "tool_family": "generic", "converter": {"type": "deterministic_fixture", "api_independent": False}, "input_roles": case.input_roles, "input_bindings": case.input_bindings, "parameters": {}, "confirmed_parameters": {}, "outputs": {"directory": "outputs", "patterns": [item.get("name_pattern", "*") for item in case.output_contract]}, "output_contract": case.output_contract, "comparator_plan": case.comparator_plan, "generated_code_artifacts": [{"path": case.script_name, "generated_by": "p0_fixture"}], "build_record": "deterministic_p0_fixture", "requirements": case.requirements, "source_files": {"code": [f"raw_code/{case.script_name}"], "test_data": [f"raw_test_data/{name}" for name in case.data_files]}, "build_status": "analyzed", "outputs_confirmed": [], "outputs_detected": [], "validation_runs": [], "publish_checks": {}, "template_gate": {}, "nl_triggers": [case.case_id], } root.mkdir(parents=True, exist_ok=True) write_json(root / "extension.json", extension) return extension def cases() -> list[FixtureCase]: copy_script = """#!/usr/bin/env bash set -euo pipefail cp "$1" "$2" """ vcf_trait_script = """#!/usr/bin/env bash set -euo pipefail vcf="$1" trait="$2" metrics="$3" report="$4" python3 - "$vcf" "$trait" "$metrics" "$report" <<'PY' import json, sys vcf, trait, metrics, report = sys.argv[1:5] samples = [] for line in open(vcf, encoding='utf-8', errors='ignore'): if line.startswith('#CHROM'): samples = line.rstrip('\\n').split('\\t')[9:] trait_rows = [x for x in open(trait, encoding='utf-8', errors='ignore').read().splitlines()[1:] if x.strip()] open(metrics, 'w', encoding='utf-8').write(json.dumps({'sample_count': len(samples), 'trait_rows': len(trait_rows)}, sort_keys=True) + '\\n') open(report, 'w', encoding='utf-8').write('sample_count\\ttrait_rows\\n%d\\t%d\\n' % (len(samples), len(trait_rows))) PY """ stdout_script = """#!/usr/bin/env bash set -euo pipefail echo "rows $(wc -l < "$1")" """ binary_script = """#!/usr/bin/env bash set -euo pipefail python3 - "$1" "$2" <<'PY' import sys open(sys.argv[2], 'wb').write(open(sys.argv[1], 'rb').read()) PY """ generic_script = """#!/usr/bin/env bash set -euo pipefail awk 'BEGIN{print "metric\\tvalue"} NR>1{n++} END{print "rows\\t" n+0}' "$1" > "$2" """ return [ FixtureCase( "positional_args_shell", "Validate positional $1/$2 shell interface through original_command and command_template.", "run_positional.sh", copy_script, {"input.tsv": b"gene\tvalue\nA\t1\n"}, "bash run_positional.sh input.tsv x.txt", "bash {code_dir}/run_positional.sh {input_file} {output_dir}/x.txt", [{"name_pattern": "x.txt", "semantic_role": "copied_table", "required": True}], input_roles={"input_file": {"role": "generic_input_file", "required": True}}, ), FixtureCase( "vcf_trait_bindings", "Preserve open GWAS-style VCF plus trait table input_bindings.", "run_vcf_trait.sh", vcf_trait_script, { "subset.clean.vcf.gz": b"##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS1\tS2\n1\t10\trs1\tA\tG\t.\tPASS\t.\tGT\t0/1\t0/0\n", "trait.tsv": b"sample\ttrait\nS1\t1.2\nS2\t0.7\n", }, "bash run_vcf_trait.sh subset.clean.vcf.gz trait.tsv gwas_metrics.json sample_match_report.tsv", "bash {code_dir}/run_vcf_trait.sh {input_dir}/subset.clean.vcf.gz {input_dir}/trait.tsv {output_dir}/gwas_metrics.json {output_dir}/sample_match_report.tsv", [ {"name_pattern": "gwas_metrics.json", "semantic_role": "gwas_metrics", "required": True}, {"name_pattern": "sample_match_report.tsv", "semantic_role": "gwas_sample_match_report", "required": True}, ], input_bindings={ "input_vcf": {"placeholder": "input_file", "format_hint": "vcf.gz", "semantic_hint": "variant_calls", "required": True}, "trait_table": {"placeholder": "input_file", "format_hint": "tsv", "semantic_hint": "phenotype_table", "required": True}, }, ), FixtureCase( "missing_system_command", "Classify absent external command as environment_failed instead of API/spec failure.", "run_missing.sh", "#!/usr/bin/env bash\nset -euo pipefail\ndefinitely_missing_p0_tool \"$1\" > \"$2\"\n", {"input.txt": b"hello\n"}, "definitely_missing_p0_tool input.txt > x.txt", "definitely_missing_p0_tool {input_file} > {output_dir}/x.txt", [{"name_pattern": "x.txt", "required": True}], requirements={"system": ["definitely_missing_p0_tool"]}, expected_status="environment_failed", ), FixtureCase( "json_metrics", "Compare JSON metric files as publishable output.", "run_json.sh", "#!/usr/bin/env bash\nset -euo pipefail\ncp \"$1\" \"$2\"\n", {"metrics.json": b"{\"a\": 1, \"b\": 2}\n"}, "bash run_json.sh metrics.json metrics.out.json", "bash {code_dir}/run_json.sh {input_file} {output_dir}/metrics.out.json", [{"name_pattern": "metrics.out.json", "semantic_role": "json_metrics", "required": True, "compare_mode": "auto"}], input_roles={"input_file": {"role": "generic_input_file", "required": True}}, ), FixtureCase( "stdout_only", "Capture stdout-only behavior and classify no-file output as a fixture-level expected failure.", "run_stdout.sh", stdout_script, {"input.txt": b"a\nb\n"}, "bash run_stdout.sh input.txt", "bash {code_dir}/run_stdout.sh {input_file}", [{"name_pattern": "stdout.txt", "required": True}], input_roles={"input_file": {"role": "generic_input_file", "required": True}}, expected_status="stdout_only_failed", ), FixtureCase( "binary_compressed_presence", "Validate binary/compressed-like file presence and byte-identical golden comparison.", "run_binary.sh", binary_script, {"input.bin": b"\x1f\x8bP0-fixture-binary\n"}, "bash run_binary.sh input.bin binary.dat", "bash {code_dir}/run_binary.sh {input_file} {output_dir}/binary.dat", [{"name_pattern": "binary.dat", "semantic_role": "binary_artifact", "required": True, "compare_mode": "binary_sha256"}], input_roles={"input_file": {"role": "generic_input_file", "required": True}}, ), FixtureCase( "generic_no_archetype", "Publish runnable generic extension while keeping template_eligible=false.", "run_generic.sh", generic_script, {"table.tsv": b"id\tvalue\nA\t1\nB\t2\n"}, "bash run_generic.sh table.tsv summary.tsv", "bash {code_dir}/run_generic.sh {input_file} {output_dir}/summary.tsv", [{"name_pattern": "summary.tsv", "semantic_role": "summary_table", "required": True}], input_bindings={"input_table": {"placeholder": "input_file", "format_hint": "tsv", "semantic_hint": "generic_table", "required": True}}, ), ] def materialize_job_input(manager: ExtensionStageManager, job_id: str, data_files: dict[str, bytes]) -> None: input_dir = manager.input_root / safe_token(job_id) if input_dir.exists(): shutil.rmtree(input_dir) input_dir.mkdir(parents=True, exist_ok=True) for name, payload in data_files.items(): target = input_dir / name target.parent.mkdir(parents=True, exist_ok=True) target.write_bytes(payload) def write_case_outputs(manager: ExtensionStageManager, extension_id: str, case_root: Path, row: dict[str, Any]) -> None: try: extension = manager.load(extension_id) write_json(case_root / "extension.json", extension) except Exception: pass write_json(case_root / "result_row.json", row) def stdout_nonempty(run: dict[str, Any]) -> bool: workspace = Path(str(run.get("workspace") or "")) rel = str((run.get("logs") or {}).get("stdout") or "logs/stdout.log") path = workspace / rel if path.exists(): return path.stat().st_size > 0 try: extension_root = workspace.parents[1] store_root = extension_root.parent relative_workspace = workspace.relative_to(extension_root) for status_dir in ("failed", "success"): relocated = store_root / status_dir / extension_root.name / relative_workspace / rel if relocated.exists(): return relocated.stat().st_size > 0 except Exception: pass return False def summarize(rows: list[dict[str, Any]]) -> dict[str, Any]: return { "schema_version": "extension_p0_fixture_summary_v1", "created_at": now(), "case_count": len(rows), "passed": sum(1 for row in rows if row.get("success")), "failed": sum(1 for row in rows if not row.get("success")), "published_runnable": sum(1 for row in rows if row.get("published_runnable")), "template_eligible": sum(1 for row in rows if row.get("template_eligible")), "environment_failed": sum(1 for row in rows if row.get("environment_failed")), "rows": rows, } def write_summary(out: Path) -> None: rows = [read_json(path) for path in sorted((out / "cases").glob("*/result_row.json"))] write_json(out / "summary.json", summarize(rows)) write_csv(out / "summary.csv", rows) def safe_token(value: str) -> str: return "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in str(value)).strip("._") or "unknown" def now() -> str: return datetime.now().replace(microsecond=0).isoformat() def write_json(path: Path, data: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") def read_json(path: Path) -> Any: return json.loads(path.read_text(encoding="utf-8")) def write_csv(path: Path, rows: list[dict[str, Any]]) -> None: path.parent.mkdir(parents=True, exist_ok=True) keys = sorted({key for row in rows for key in row}) with path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=keys) writer.writeheader() for row in rows: writer.writerow({key: json.dumps(value, ensure_ascii=False) if isinstance(value, (dict, list)) else value for key, value in row.items()}) if __name__ == "__main__": raise SystemExit(main())