初始提交
This commit is contained in:
potaser
Executable
+70
@@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from src.rag.extractors.v16_1_2.paper_metadata_enricher_v16_1_2 import (
|
||||
PaperMetadataEnricherV16,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Add canonical DOI/PMID/PMCID and citation metadata to a V16 result.json."
|
||||
)
|
||||
parser.add_argument("--result-json", required=True, type=Path)
|
||||
parser.add_argument("--pdf", type=Path)
|
||||
parser.add_argument("--doi", default="")
|
||||
parser.add_argument("--pmid", default="")
|
||||
parser.add_argument("--pmcid", default="")
|
||||
parser.add_argument("--output", type=Path)
|
||||
parser.add_argument("--in-place", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
source = args.result_json.resolve()
|
||||
if not source.exists():
|
||||
raise SystemExit(f"Missing result JSON: {source}")
|
||||
if not args.in_place and not args.output:
|
||||
raise SystemExit("Use --in-place or provide --output.")
|
||||
destination = source if args.in_place else args.output.resolve()
|
||||
|
||||
data = json.loads(source.read_text(encoding="utf-8"))
|
||||
enricher = PaperMetadataEnricherV16()
|
||||
paper = enricher.resolve(
|
||||
pdf_path=args.pdf.resolve() if args.pdf else None,
|
||||
doi=args.doi,
|
||||
pmid=args.pmid,
|
||||
pmcid=args.pmcid,
|
||||
)
|
||||
enricher.enrich_output(data, paper)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(
|
||||
json.dumps(data, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
audit_path = destination.parent / "paper_metadata_enrichment_audit.json"
|
||||
enricher.save_audit(audit_path, paper)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result_json": str(destination),
|
||||
"audit": str(audit_path),
|
||||
"paper": paper,
|
||||
"relations": len(data.get("relations", [])),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+405
@@ -0,0 +1,405 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
import unicodedata
|
||||
from collections import Counter
|
||||
from difflib import SequenceMatcher
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ir_measures import P, RR, Recall, nDCG, Qrel, ScoredDoc, calc_aggregate
|
||||
|
||||
|
||||
CONFIGS = [
|
||||
"A0_full",
|
||||
"A1_no_planner",
|
||||
"A2_no_neo4j",
|
||||
"A3_no_vector",
|
||||
"A4_one_hop",
|
||||
"A5_no_bge",
|
||||
]
|
||||
KS = (5, 10, 30)
|
||||
|
||||
|
||||
def normalize(text: Any) -> str:
|
||||
value = html.unescape(str(text or ""))
|
||||
value = unicodedata.normalize("NFKC", value).lower()
|
||||
value = re.sub(r"<[^>]+>", " ", value)
|
||||
value = re.sub(r"https?://\S+", " ", value)
|
||||
value = re.sub(r"[^a-z0-9\u4e00-\u9fff.%+-]+", " ", value)
|
||||
return " ".join(value.split())
|
||||
|
||||
|
||||
def tokens(text: Any) -> list[str]:
|
||||
return re.findall(r"[a-z0-9.%+-]+|[\u4e00-\u9fff]", normalize(text))
|
||||
|
||||
|
||||
def evidence_similarity(gold: str, candidate: str) -> dict[str, float | bool]:
|
||||
ng = normalize(gold)
|
||||
nc = normalize(candidate)
|
||||
if not ng or not nc:
|
||||
return {
|
||||
"matched": False,
|
||||
"containment": False,
|
||||
"token_recall": 0.0,
|
||||
"token_precision": 0.0,
|
||||
"sequence_ratio": 0.0,
|
||||
"match_score": 0.0,
|
||||
}
|
||||
containment = ng in nc or nc in ng
|
||||
gt = set(tokens(gold))
|
||||
ct = set(tokens(candidate))
|
||||
overlap = len(gt & ct)
|
||||
token_recall = overlap / len(gt) if gt else 0.0
|
||||
token_precision = overlap / len(ct) if ct else 0.0
|
||||
sequence_ratio = SequenceMatcher(None, ng, nc).ratio()
|
||||
match_score = max(
|
||||
sequence_ratio,
|
||||
0.72 * token_recall + 0.28 * token_precision,
|
||||
)
|
||||
matched = bool(
|
||||
containment
|
||||
or sequence_ratio >= 0.82
|
||||
or (token_recall >= 0.72 and token_precision >= 0.28)
|
||||
)
|
||||
return {
|
||||
"matched": matched,
|
||||
"containment": containment,
|
||||
"token_recall": token_recall,
|
||||
"token_precision": token_precision,
|
||||
"sequence_ratio": sequence_ratio,
|
||||
"match_score": match_score,
|
||||
}
|
||||
|
||||
|
||||
def evidence_doc_id(evidence: str) -> str:
|
||||
digest = hashlib.sha1(normalize(evidence).encode("utf-8")).hexdigest()[:20]
|
||||
return f"EVID::{digest}"
|
||||
|
||||
|
||||
def paper_id(source: dict[str, Any]) -> str:
|
||||
return str(
|
||||
source.get("paper_id")
|
||||
or source.get("pmcid")
|
||||
or source.get("pmid")
|
||||
or source.get("doi")
|
||||
or source.get("paper_title")
|
||||
or source.get("title")
|
||||
or ""
|
||||
).strip()
|
||||
|
||||
|
||||
def dedupe_ranked_sources(
|
||||
qid: str,
|
||||
gold_source: str,
|
||||
sources: list[dict[str, Any]],
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
ranked: list[dict[str, Any]] = []
|
||||
audit: list[dict[str, Any]] = []
|
||||
seen_evidence: set[str] = set()
|
||||
gold_doc_id = f"GOLD::{qid}"
|
||||
for raw_rank, source in enumerate(sources, 1):
|
||||
evidence = str(source.get("evidence") or "")
|
||||
normalized = normalize(evidence)
|
||||
if not normalized or normalized in seen_evidence:
|
||||
continue
|
||||
seen_evidence.add(normalized)
|
||||
similarity = evidence_similarity(gold_source, evidence)
|
||||
doc_id = gold_doc_id if similarity["matched"] else evidence_doc_id(evidence)
|
||||
row = {
|
||||
"rank": len(ranked) + 1,
|
||||
"raw_rank": raw_rank,
|
||||
"doc_id": doc_id,
|
||||
"paper_id": paper_id(source),
|
||||
"evidence": evidence,
|
||||
"source": source.get("source", ""),
|
||||
"relation": source.get("relation", ""),
|
||||
"target": source.get("target", ""),
|
||||
**similarity,
|
||||
}
|
||||
ranked.append(row)
|
||||
audit.append(row)
|
||||
return ranked, audit
|
||||
|
||||
|
||||
def metric_key(measure: Any) -> str:
|
||||
return str(measure)
|
||||
|
||||
|
||||
def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
|
||||
if not rows:
|
||||
return
|
||||
with path.open("w", encoding="utf-8-sig", newline="") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--results-dir", required=True)
|
||||
parser.add_argument("--questions", required=True)
|
||||
parser.add_argument("--judgments", required=True)
|
||||
parser.add_argument("--output-dir", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
results_dir = Path(args.results_dir)
|
||||
questions = json.loads(Path(args.questions).read_text(encoding="utf-8-sig"))
|
||||
question_by_id = {str(item["id"]): item for item in questions}
|
||||
judgment_rows = list(
|
||||
csv.DictReader(open(args.judgments, encoding="utf-8-sig"))
|
||||
)
|
||||
judgment_by_key = {
|
||||
(row["config_id"], row["question_id"]): row for row in judgment_rows
|
||||
}
|
||||
output_dir = Path(args.output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
(output_dir / "trec_runs").mkdir(exist_ok=True)
|
||||
|
||||
qrels = [
|
||||
Qrel(str(item["id"]), f"GOLD::{item['id']}", 1) for item in questions
|
||||
]
|
||||
qrels_path = output_dir / "known_item_qrels.tsv"
|
||||
with qrels_path.open("w", encoding="utf-8", newline="") as handle:
|
||||
for item in questions:
|
||||
qid = str(item["id"])
|
||||
handle.write(f"{qid}\t0\tGOLD::{qid}\t1\n")
|
||||
|
||||
measures = [
|
||||
P @ 5,
|
||||
P @ 10,
|
||||
P @ 30,
|
||||
Recall @ 5,
|
||||
Recall @ 10,
|
||||
Recall @ 30,
|
||||
RR @ 30,
|
||||
nDCG @ 5,
|
||||
nDCG @ 10,
|
||||
nDCG @ 30,
|
||||
]
|
||||
summary_rows: list[dict[str, Any]] = []
|
||||
query_rows: list[dict[str, Any]] = []
|
||||
match_audit: list[dict[str, Any]] = []
|
||||
|
||||
for config in CONFIGS:
|
||||
run: list[ScoredDoc] = []
|
||||
diversity: dict[int, list[float]] = {k: [] for k in KS}
|
||||
diversity_returned: dict[int, list[float]] = {k: [] for k in KS}
|
||||
duplicate_rates: list[float] = []
|
||||
retrieved_counts: list[int] = []
|
||||
per_question_ranked: dict[str, list[dict[str, Any]]] = {}
|
||||
|
||||
run_path = output_dir / "trec_runs" / f"{config}.run"
|
||||
with run_path.open("w", encoding="utf-8", newline="") as run_handle:
|
||||
for item in questions:
|
||||
qid = str(item["id"])
|
||||
record = json.loads(
|
||||
(results_dir / config / f"{qid}.json").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
)
|
||||
response = record.get("response") or {}
|
||||
raw_sources = response.get("sources") or []
|
||||
ranked, audit = dedupe_ranked_sources(
|
||||
qid, str(item.get("source_text") or ""), raw_sources
|
||||
)
|
||||
per_question_ranked[qid] = ranked
|
||||
retrieved_counts.append(len(ranked))
|
||||
duplicate_rates.append(
|
||||
1 - (len(ranked) / len(raw_sources)) if raw_sources else 0.0
|
||||
)
|
||||
for row in audit:
|
||||
match_audit.append(
|
||||
{
|
||||
"config_id": config,
|
||||
"question_id": qid,
|
||||
"gold_source_text": item.get("source_text", ""),
|
||||
**row,
|
||||
}
|
||||
)
|
||||
for row in ranked:
|
||||
score = 10000.0 - float(row["rank"])
|
||||
run.append(ScoredDoc(qid, str(row["doc_id"]), score))
|
||||
run_handle.write(
|
||||
f"{qid} Q0 {row['doc_id']} {row['rank']} "
|
||||
f"{score:.6f} {config}\n"
|
||||
)
|
||||
for k in KS:
|
||||
top = ranked[:k]
|
||||
unique_papers = len(
|
||||
{str(row["paper_id"]) for row in top if row["paper_id"]}
|
||||
)
|
||||
diversity[k].append(unique_papers / k)
|
||||
diversity_returned[k].append(
|
||||
unique_papers / len(top) if top else 0.0
|
||||
)
|
||||
|
||||
aggregate = calc_aggregate(measures, qrels, run)
|
||||
summary: dict[str, Any] = {
|
||||
"config_id": config,
|
||||
"questions": len(questions),
|
||||
}
|
||||
for measure in measures:
|
||||
summary[metric_key(measure)] = float(aggregate[measure])
|
||||
for k in KS:
|
||||
summary[f"source_diversity@{k}"] = sum(diversity[k]) / len(
|
||||
diversity[k]
|
||||
)
|
||||
summary[f"source_diversity_returned@{k}"] = sum(
|
||||
diversity_returned[k]
|
||||
) / len(diversity_returned[k])
|
||||
summary["mean_exact_duplicate_rate"] = sum(duplicate_rates) / len(
|
||||
duplicate_rates
|
||||
)
|
||||
summary["mean_deduplicated_source_count"] = sum(retrieved_counts) / len(
|
||||
retrieved_counts
|
||||
)
|
||||
|
||||
answer_counts = Counter()
|
||||
answer_scores: list[float] = []
|
||||
for item in questions:
|
||||
qid = str(item["id"])
|
||||
judgment = judgment_by_key[(config, qid)]
|
||||
verdict = judgment["answer_verdict"]
|
||||
answer_counts[verdict] += 1
|
||||
answer_scores.append(
|
||||
{"correct": 1.0, "partial": 0.5}.get(verdict, 0.0)
|
||||
)
|
||||
ranked = per_question_ranked[qid]
|
||||
first_rank = next(
|
||||
(
|
||||
int(row["rank"])
|
||||
for row in ranked
|
||||
if row["doc_id"] == f"GOLD::{qid}"
|
||||
),
|
||||
0,
|
||||
)
|
||||
query_rows.append(
|
||||
{
|
||||
"config_id": config,
|
||||
"question_id": qid,
|
||||
"question": item["question"],
|
||||
"answer_verdict": verdict,
|
||||
"answer_score": {
|
||||
"correct": 1.0,
|
||||
"partial": 0.5,
|
||||
}.get(verdict, 0.0),
|
||||
"evidence_verdict_llm": judgment["evidence_verdict"],
|
||||
"known_gold_first_rank": first_rank,
|
||||
"known_gold_hit@5": int(bool(first_rank and first_rank <= 5)),
|
||||
"known_gold_hit@10": int(
|
||||
bool(first_rank and first_rank <= 10)
|
||||
),
|
||||
"known_gold_hit@30": int(
|
||||
bool(first_rank and first_rank <= 30)
|
||||
),
|
||||
"deduplicated_source_count": len(ranked),
|
||||
"unique_papers@10": len(
|
||||
{
|
||||
str(row["paper_id"])
|
||||
for row in ranked[:10]
|
||||
if row["paper_id"]
|
||||
}
|
||||
),
|
||||
"judge_reason": judgment["reason"],
|
||||
}
|
||||
)
|
||||
summary["answer_correct"] = answer_counts["correct"]
|
||||
summary["answer_partial"] = answer_counts["partial"]
|
||||
summary["answer_wrong"] = answer_counts["wrong"]
|
||||
summary["answer_insufficient"] = answer_counts["insufficient"]
|
||||
summary["answer_correctness_1_0.5_0"] = sum(answer_scores) / len(
|
||||
answer_scores
|
||||
)
|
||||
summary_rows.append(summary)
|
||||
|
||||
scored_rows: list[dict[str, Any]] = []
|
||||
for item in questions:
|
||||
qid = str(item["id"])
|
||||
judgment = judgment_by_key[(config, qid)]
|
||||
verdict = judgment["answer_verdict"]
|
||||
record = json.loads(
|
||||
(results_dir / config / f"{qid}.json").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
)
|
||||
scored_rows.append(
|
||||
{
|
||||
"id": qid,
|
||||
"question": item["question"],
|
||||
"gold_answer": item["gold_answer"],
|
||||
"deepseek_answer": (record.get("response") or {}).get(
|
||||
"answer", ""
|
||||
),
|
||||
"score": {"correct": 1.0, "partial": 0.5}.get(
|
||||
verdict, 0.0
|
||||
),
|
||||
"label": verdict,
|
||||
"reason": judgment["reason"],
|
||||
"model": "DeepSeek V4Flash + RAG",
|
||||
"status": "answered",
|
||||
}
|
||||
)
|
||||
config_base = output_dir / f"{config}_same_scale_correctness"
|
||||
(config_base.with_suffix(".json")).write_text(
|
||||
json.dumps(scored_rows, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
write_csv(config_base.with_suffix(".csv"), scored_rows)
|
||||
config_summary = {
|
||||
"model": f"DeepSeek V4Flash + RAG ({config})",
|
||||
"n": len(scored_rows),
|
||||
"total_score": sum(float(row["score"]) for row in scored_rows),
|
||||
"answer_correctness": summary["answer_correctness_1_0.5_0"],
|
||||
"label_counts": dict(answer_counts),
|
||||
"score_counts": dict(
|
||||
Counter(str(row["score"]) for row in scored_rows)
|
||||
),
|
||||
"scoring_note": (
|
||||
"Unified 1/0.5/0 scale. Labels come from the calibrated "
|
||||
"DeepSeek evidence-aware judge, not hard-coded question IDs."
|
||||
),
|
||||
}
|
||||
(output_dir / f"{config}_same_scale_correctness_summary.json").write_text(
|
||||
json.dumps(config_summary, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
write_csv(output_dir / "ir_metrics_summary.csv", summary_rows)
|
||||
write_csv(output_dir / "ir_metrics_per_question.csv", query_rows)
|
||||
write_csv(output_dir / "gold_evidence_match_audit.csv", match_audit)
|
||||
report = {
|
||||
"protocol": "known-item retrieval evaluation",
|
||||
"implementation": "ir_measures 0.4.1 / pytrec-eval-terrier",
|
||||
"qrels": str(qrels_path),
|
||||
"ks": KS,
|
||||
"matching_rule": {
|
||||
"containment": True,
|
||||
"sequence_ratio_threshold": 0.82,
|
||||
"token_recall_threshold": 0.72,
|
||||
"token_precision_threshold": 0.28,
|
||||
},
|
||||
"important_limitation": (
|
||||
"Each question has one known gold source text. Therefore Recall@k "
|
||||
"is equivalent to known-item hit rate. Precision@k and nDCG@k "
|
||||
"treat unjudged evidence as non-relevant and may underestimate "
|
||||
"true retrieval quality. Formal publication-grade P/nDCG requires "
|
||||
"pooled relevance judgments over retrieved evidence."
|
||||
),
|
||||
"summary": summary_rows,
|
||||
}
|
||||
(output_dir / "ir_evaluation_report.json").write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+371
@@ -0,0 +1,371 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from src.rag.project_config import env
|
||||
|
||||
|
||||
CONFIGS = [
|
||||
"A0_full",
|
||||
"A1_no_planner",
|
||||
"A2_no_neo4j",
|
||||
"A3_no_vector",
|
||||
"A4_one_hop",
|
||||
"A5_no_bge",
|
||||
]
|
||||
|
||||
|
||||
def atomic_json(path: Path, payload: Any) -> None:
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def extract_json(text: str) -> dict[str, Any]:
|
||||
text = (text or "").strip()
|
||||
try:
|
||||
return json.loads(text)
|
||||
except Exception:
|
||||
pass
|
||||
fenced = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", text, re.S)
|
||||
if fenced:
|
||||
try:
|
||||
return json.loads(fenced.group(1))
|
||||
except Exception:
|
||||
pass
|
||||
match = re.search(r"\{.*\}", text, re.S)
|
||||
if match:
|
||||
try:
|
||||
return json.loads(match.group(0))
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
||||
def compact_sources(sources: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
compact: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for source in sources:
|
||||
evidence = " ".join(str(source.get("evidence") or "").split())
|
||||
key = evidence.lower()
|
||||
if not evidence or key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
compact.append(
|
||||
{
|
||||
"srt": (
|
||||
f"{source.get('source','')} -- "
|
||||
f"{source.get('relation','')} --> "
|
||||
f"{source.get('target','')}"
|
||||
),
|
||||
"evidence": evidence[:260],
|
||||
}
|
||||
)
|
||||
if len(compact) >= 5:
|
||||
break
|
||||
return compact
|
||||
|
||||
|
||||
def judge_one(
|
||||
*,
|
||||
root: Path,
|
||||
output_dir: Path,
|
||||
gold: dict[str, Any],
|
||||
api_url: str,
|
||||
api_key: str,
|
||||
model: str,
|
||||
retries: int,
|
||||
) -> dict[str, Any]:
|
||||
qid = str(gold["id"])
|
||||
output_path = output_dir / f"{qid}.json"
|
||||
if output_path.exists():
|
||||
try:
|
||||
existing = json.loads(output_path.read_text(encoding="utf-8"))
|
||||
judgments = existing.get("judgments") or []
|
||||
if len(judgments) == len(CONFIGS):
|
||||
return {"question_id": qid, "status": "ok", "resumed": True}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
candidates: list[dict[str, Any]] = []
|
||||
for config in CONFIGS:
|
||||
record = json.loads(
|
||||
(root / config / f"{qid}.json").read_text(encoding="utf-8")
|
||||
)
|
||||
response = record.get("response") or {}
|
||||
candidates.append(
|
||||
{
|
||||
"config_id": config,
|
||||
"answer": response.get("answer", ""),
|
||||
"retrieved_evidence": compact_sources(response.get("sources") or []),
|
||||
}
|
||||
)
|
||||
|
||||
prompt = f"""
|
||||
You are evaluating six RAG answers to one plant-biology question.
|
||||
|
||||
The GOLD SOURCE TEXT is authoritative. The short GOLD ANSWER may be incomplete;
|
||||
if they differ, judge against the GOLD SOURCE TEXT.
|
||||
Do not reward verbosity. Do not penalize a concise correct answer.
|
||||
Judge each system independently.
|
||||
|
||||
Question:
|
||||
{gold['question']}
|
||||
|
||||
Gold answer:
|
||||
{gold['gold_answer']}
|
||||
|
||||
Gold source text:
|
||||
{gold['source_text']}
|
||||
|
||||
Candidate systems:
|
||||
{json.dumps(candidates, ensure_ascii=False)}
|
||||
|
||||
For each configuration, judge:
|
||||
1. answer_verdict:
|
||||
- correct: all facts needed by the question are correct; extra claims do not conflict.
|
||||
- partial: some required facts are correct but important requested facts are missing,
|
||||
or the answer mixes the target result with conflicting alternatives.
|
||||
- wrong: gives a conflicting or unrelated substantive answer.
|
||||
- insufficient: explicitly cannot answer or gives no substantive answer.
|
||||
2. evidence_verdict:
|
||||
- full: retrieved evidence contains all facts needed to answer.
|
||||
- partial: retrieved evidence contains some but not all required facts.
|
||||
- miss: retrieved evidence does not contain the target fact.
|
||||
3. usable: true only when answer_verdict is correct or partial.
|
||||
|
||||
Return JSON only:
|
||||
{{
|
||||
"question_id": "{qid}",
|
||||
"judgments": [
|
||||
{{
|
||||
"config_id": "A0_full",
|
||||
"answer_verdict": "correct|partial|wrong|insufficient",
|
||||
"evidence_verdict": "full|partial|miss",
|
||||
"usable": true,
|
||||
"reason": "maximum 15 words"
|
||||
}}
|
||||
]
|
||||
}}
|
||||
""".strip()
|
||||
|
||||
last_error = ""
|
||||
raw = ""
|
||||
for attempt in range(1, retries + 2):
|
||||
try:
|
||||
response = requests.post(
|
||||
api_url,
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json={
|
||||
"model": model,
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "Return valid JSON only. Apply the evaluation rubric consistently.",
|
||||
},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
"temperature": 0,
|
||||
"max_tokens": 4000,
|
||||
},
|
||||
timeout=240,
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
raw = (data.get("choices") or [{}])[0].get("message", {}).get(
|
||||
"content", ""
|
||||
)
|
||||
parsed = extract_json(raw)
|
||||
judgments = parsed.get("judgments") or []
|
||||
config_ids = {str(item.get("config_id")) for item in judgments}
|
||||
if len(judgments) == len(CONFIGS) and config_ids == set(CONFIGS):
|
||||
payload = {
|
||||
"question_id": qid,
|
||||
"question": gold["question"],
|
||||
"gold_answer": gold["gold_answer"],
|
||||
"gold_source_text": gold["source_text"],
|
||||
"judgments": judgments,
|
||||
"attempt": attempt,
|
||||
"model": model,
|
||||
}
|
||||
atomic_json(output_path, payload)
|
||||
return {"question_id": qid, "status": "ok", "attempt": attempt}
|
||||
last_error = "Incomplete or invalid judgment JSON"
|
||||
except Exception as exc:
|
||||
last_error = f"{type(exc).__name__}: {exc}"
|
||||
time.sleep(min(4 * attempt, 12))
|
||||
|
||||
atomic_json(
|
||||
output_path,
|
||||
{
|
||||
"question_id": qid,
|
||||
"question": gold["question"],
|
||||
"error": last_error,
|
||||
"raw": raw,
|
||||
"model": model,
|
||||
},
|
||||
)
|
||||
return {"question_id": qid, "status": "error", "error": last_error}
|
||||
|
||||
|
||||
def summarize(
|
||||
root: Path,
|
||||
judge_dir: Path,
|
||||
questions: list[dict[str, Any]],
|
||||
output_dir: Path,
|
||||
) -> None:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for gold in questions:
|
||||
qid = str(gold["id"])
|
||||
judged = json.loads((judge_dir / f"{qid}.json").read_text(encoding="utf-8"))
|
||||
for item in judged.get("judgments") or []:
|
||||
config = str(item["config_id"])
|
||||
record = json.loads(
|
||||
(root / config / f"{qid}.json").read_text(encoding="utf-8")
|
||||
)
|
||||
response = record.get("response") or {}
|
||||
rows.append(
|
||||
{
|
||||
"question_id": qid,
|
||||
"config_id": config,
|
||||
"question": gold["question"],
|
||||
"gold_answer": gold["gold_answer"],
|
||||
"answer_verdict": item.get("answer_verdict", ""),
|
||||
"evidence_verdict": item.get("evidence_verdict", ""),
|
||||
"usable": bool(item.get("usable")),
|
||||
"reason": item.get("reason", ""),
|
||||
"answer": response.get("answer", ""),
|
||||
"source_count": len(response.get("sources") or []),
|
||||
"result_file": str(root / config / f"{qid}.json"),
|
||||
}
|
||||
)
|
||||
csv_path = output_dir / "llm_judgments_100q.csv"
|
||||
with csv_path.open("w", encoding="utf-8-sig", newline="") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
summaries: list[dict[str, Any]] = []
|
||||
for config in CONFIGS:
|
||||
subset = [row for row in rows if row["config_id"] == config]
|
||||
answers = Counter(str(row["answer_verdict"]) for row in subset)
|
||||
evidence = Counter(str(row["evidence_verdict"]) for row in subset)
|
||||
n = len(subset)
|
||||
weighted_answer = (
|
||||
answers["correct"] + 0.5 * answers["partial"]
|
||||
) / n
|
||||
weighted_recall = (
|
||||
evidence["full"] + 0.5 * evidence["partial"]
|
||||
) / n
|
||||
summaries.append(
|
||||
{
|
||||
"config_id": config,
|
||||
"questions": n,
|
||||
"correct": answers["correct"],
|
||||
"partial": answers["partial"],
|
||||
"wrong": answers["wrong"],
|
||||
"insufficient": answers["insufficient"],
|
||||
"strict_answer_accuracy": answers["correct"] / n,
|
||||
"weighted_answer_accuracy": weighted_answer,
|
||||
"usable_answer_rate": sum(
|
||||
1 for row in subset if row["usable"]
|
||||
)
|
||||
/ n,
|
||||
"full_evidence_hits": evidence["full"],
|
||||
"partial_evidence_hits": evidence["partial"],
|
||||
"evidence_misses": evidence["miss"],
|
||||
"question_level_full_recall": evidence["full"] / n,
|
||||
"question_level_weighted_recall": weighted_recall,
|
||||
}
|
||||
)
|
||||
summary = {
|
||||
"evaluation": "DeepSeek LLM-assisted judge",
|
||||
"warning": (
|
||||
"These are automated judge estimates. They are not a substitute for "
|
||||
"human review and should be calibrated against the manually reviewed 20 questions."
|
||||
),
|
||||
"configs": summaries,
|
||||
}
|
||||
atomic_json(output_dir / "llm_judgment_summary_100q.json", summary)
|
||||
with (output_dir / "llm_judgment_summary_100q.csv").open(
|
||||
"w", encoding="utf-8-sig", newline=""
|
||||
) as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=list(summaries[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(summaries)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input-dir", required=True)
|
||||
parser.add_argument("--questions", required=True)
|
||||
parser.add_argument("--output-dir", required=True)
|
||||
parser.add_argument("--workers", type=int, default=4)
|
||||
parser.add_argument("--retries", type=int, default=2)
|
||||
args = parser.parse_args()
|
||||
|
||||
api_key = env("DEEPSEEK_API_KEY")
|
||||
if not api_key:
|
||||
raise RuntimeError("No DEEPSEEK_API_KEY configured")
|
||||
base_url = env("DEEPSEEK_BASE_URL", "https://api.deepseek.com").rstrip("/")
|
||||
model = env("V15_RELATION_LLM_MODEL", "deepseek-v4-flash")
|
||||
questions = json.loads(Path(args.questions).read_text(encoding="utf-8-sig"))
|
||||
root = Path(args.input_dir)
|
||||
output_dir = Path(args.output_dir)
|
||||
judge_dir = output_dir / "per_question"
|
||||
judge_dir.mkdir(parents=True, exist_ok=True)
|
||||
progress_path = output_dir / "judge_progress.json"
|
||||
|
||||
completed: list[dict[str, Any]] = []
|
||||
with ThreadPoolExecutor(max_workers=max(1, args.workers)) as pool:
|
||||
futures = [
|
||||
pool.submit(
|
||||
judge_one,
|
||||
root=root,
|
||||
output_dir=judge_dir,
|
||||
gold=gold,
|
||||
api_url=f"{base_url}/chat/completions",
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
retries=args.retries,
|
||||
)
|
||||
for gold in questions
|
||||
]
|
||||
for index, future in enumerate(as_completed(futures), 1):
|
||||
result = future.result()
|
||||
completed.append(result)
|
||||
atomic_json(
|
||||
progress_path,
|
||||
{
|
||||
"completed": index,
|
||||
"total": len(questions),
|
||||
"errors": [x for x in completed if x["status"] == "error"],
|
||||
},
|
||||
)
|
||||
print(
|
||||
f"[{index}/{len(questions)}] {result['question_id']} "
|
||||
f"{result['status']}",
|
||||
flush=True,
|
||||
)
|
||||
errors = [item for item in completed if item["status"] == "error"]
|
||||
if errors:
|
||||
raise RuntimeError(f"Judge failures: {errors}")
|
||||
summarize(root, judge_dir, questions, output_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+230
@@ -0,0 +1,230 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from src.rag.project_config import env
|
||||
|
||||
|
||||
def atomic_json(path: Path, payload: Any) -> None:
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def extract_json(text: str) -> dict[str, Any]:
|
||||
text = (text or "").strip()
|
||||
try:
|
||||
return json.loads(text)
|
||||
except Exception:
|
||||
pass
|
||||
match = re.search(r"\{.*\}", text, re.S)
|
||||
if match:
|
||||
try:
|
||||
return json.loads(match.group(0))
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
||||
def judge_one(
|
||||
gold: dict[str, Any],
|
||||
answer: dict[str, Any],
|
||||
output_dir: Path,
|
||||
api_url: str,
|
||||
api_key: str,
|
||||
model: str,
|
||||
) -> dict[str, Any]:
|
||||
qid = str(gold["id"])
|
||||
output = output_dir / f"{qid}.json"
|
||||
if output.exists():
|
||||
try:
|
||||
existing = json.loads(output.read_text(encoding="utf-8"))
|
||||
if existing.get("answer_verdict"):
|
||||
return {"question_id": qid, "status": "ok", "resumed": True}
|
||||
except Exception:
|
||||
pass
|
||||
candidate = str(answer.get("deepseek_answer") or "")
|
||||
prompt = f"""
|
||||
Evaluate one direct-LLM answer to a plant-biology question.
|
||||
The GOLD SOURCE TEXT is authoritative. The short GOLD ANSWER may be incomplete.
|
||||
Do not reward verbosity. Judge only factual correctness and completeness.
|
||||
|
||||
Question:
|
||||
{gold['question']}
|
||||
|
||||
Gold answer:
|
||||
{gold['gold_answer']}
|
||||
|
||||
Gold source text:
|
||||
{gold['source_text']}
|
||||
|
||||
Candidate answer:
|
||||
{candidate}
|
||||
|
||||
Return one compact JSON object only:
|
||||
{{"answer_verdict":"correct|partial|wrong|insufficient","usable":true,"reason":"maximum 12 words"}}
|
||||
|
||||
Definitions:
|
||||
- correct: all facts needed by the question are correct.
|
||||
- partial: some required facts are correct, but important requested facts are missing.
|
||||
- wrong: substantive answer conflicts with the gold source text.
|
||||
- insufficient: no substantive answer or explicitly cannot answer.
|
||||
- usable is true only for correct or partial.
|
||||
""".strip()
|
||||
last_error = ""
|
||||
for attempt in range(1, 4):
|
||||
try:
|
||||
response = requests.post(
|
||||
api_url,
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json={
|
||||
"model": model,
|
||||
"messages": [
|
||||
{"role": "system", "content": "Return valid JSON only."},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
"temperature": 0,
|
||||
"max_tokens": 1200,
|
||||
},
|
||||
timeout=180,
|
||||
)
|
||||
response.raise_for_status()
|
||||
content = (response.json().get("choices") or [{}])[0].get(
|
||||
"message", {}
|
||||
).get("content", "")
|
||||
parsed = extract_json(content)
|
||||
verdict = parsed.get("answer_verdict")
|
||||
if verdict in {"correct", "partial", "wrong", "insufficient"}:
|
||||
payload = {
|
||||
"question_id": qid,
|
||||
"question": gold["question"],
|
||||
"gold_answer": gold["gold_answer"],
|
||||
"gold_source_text": gold["source_text"],
|
||||
"deepseek_answer": candidate,
|
||||
"answer_verdict": verdict,
|
||||
"usable": bool(parsed.get("usable")),
|
||||
"reason": parsed.get("reason", ""),
|
||||
"attempt": attempt,
|
||||
"model": model,
|
||||
}
|
||||
atomic_json(output, payload)
|
||||
return {"question_id": qid, "status": "ok"}
|
||||
last_error = "invalid verdict"
|
||||
except Exception as exc:
|
||||
last_error = f"{type(exc).__name__}: {exc}"
|
||||
time.sleep(3 * attempt)
|
||||
atomic_json(output, {"question_id": qid, "error": last_error})
|
||||
return {"question_id": qid, "status": "error", "error": last_error}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--answers", required=True)
|
||||
parser.add_argument("--questions", required=True)
|
||||
parser.add_argument("--output-dir", required=True)
|
||||
parser.add_argument("--workers", type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
|
||||
api_key = env("DEEPSEEK_API_KEY")
|
||||
if not api_key:
|
||||
raise RuntimeError("No DEEPSEEK_API_KEY configured")
|
||||
base_url = env("DEEPSEEK_BASE_URL", "https://api.deepseek.com").rstrip("/")
|
||||
model = env("V15_RELATION_LLM_MODEL", "deepseek-v4-flash")
|
||||
|
||||
answers = json.loads(Path(args.answers).read_text(encoding="utf-8-sig"))
|
||||
questions = json.loads(Path(args.questions).read_text(encoding="utf-8-sig"))
|
||||
answer_by_id = {str(item["id"]): item for item in answers}
|
||||
output_dir = Path(args.output_dir)
|
||||
per_question = output_dir / "per_question"
|
||||
per_question.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
with ThreadPoolExecutor(max_workers=max(1, args.workers)) as pool:
|
||||
futures = [
|
||||
pool.submit(
|
||||
judge_one,
|
||||
gold,
|
||||
answer_by_id[str(gold["id"])],
|
||||
per_question,
|
||||
f"{base_url}/chat/completions",
|
||||
api_key,
|
||||
model,
|
||||
)
|
||||
for gold in questions
|
||||
]
|
||||
for index, future in enumerate(as_completed(futures), 1):
|
||||
result = future.result()
|
||||
results.append(result)
|
||||
print(
|
||||
f"[{index}/{len(questions)}] {result['question_id']} "
|
||||
f"{result['status']}",
|
||||
flush=True,
|
||||
)
|
||||
errors = [item for item in results if item["status"] == "error"]
|
||||
if errors:
|
||||
raise RuntimeError(errors)
|
||||
|
||||
rows: list[dict[str, Any]] = []
|
||||
for gold in questions:
|
||||
qid = str(gold["id"])
|
||||
judged = json.loads(
|
||||
(per_question / f"{qid}.json").read_text(encoding="utf-8")
|
||||
)
|
||||
verdict = judged["answer_verdict"]
|
||||
rows.append(
|
||||
{
|
||||
"id": qid,
|
||||
"question": gold["question"],
|
||||
"gold_answer": gold["gold_answer"],
|
||||
"deepseek_answer": judged["deepseek_answer"],
|
||||
"score": {"correct": 1.0, "partial": 0.5}.get(verdict, 0.0),
|
||||
"label": verdict,
|
||||
"reason": judged["reason"],
|
||||
"model": "DeepSeek V4Flash direct, unified judge",
|
||||
"status": "answered",
|
||||
}
|
||||
)
|
||||
with (output_dir / "direct_same_scale_correctness.csv").open(
|
||||
"w", encoding="utf-8-sig", newline=""
|
||||
) as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
(output_dir / "direct_same_scale_correctness.json").write_text(
|
||||
json.dumps(rows, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
counts: dict[str, int] = {}
|
||||
for row in rows:
|
||||
counts[row["label"]] = counts.get(row["label"], 0) + 1
|
||||
summary = {
|
||||
"model": "DeepSeek V4Flash direct, unified evidence-aware judge",
|
||||
"n": len(rows),
|
||||
"total_score": sum(float(row["score"]) for row in rows),
|
||||
"answer_correctness": sum(float(row["score"]) for row in rows)
|
||||
/ len(rows),
|
||||
"label_counts": counts,
|
||||
"scoring_note": (
|
||||
"Same correct=1, partial=0.5, wrong/insufficient=0 rubric "
|
||||
"used for the RAG ablation configurations."
|
||||
),
|
||||
}
|
||||
atomic_json(output_dir / "direct_same_scale_correctness_summary.json", summary)
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+131
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
CONFIGS = [
|
||||
"A0_full",
|
||||
"A1_no_planner",
|
||||
"A2_no_neo4j",
|
||||
"A3_no_vector",
|
||||
"A4_one_hop",
|
||||
"A5_no_bge",
|
||||
]
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def resolve_path(value: str | Path) -> Path:
|
||||
path = Path(value)
|
||||
if not path.is_absolute():
|
||||
path = PROJECT_ROOT / path
|
||||
return path
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--base-dir",
|
||||
default="experiments/rag_ablation_20260624",
|
||||
help="Base ablation experiment directory, relative to the project root unless absolute.",
|
||||
)
|
||||
parser.add_argument("--first-20-dir", default="")
|
||||
parser.add_argument("--remaining-80-dir", default="")
|
||||
parser.add_argument("--output-dir", default="")
|
||||
parser.add_argument("--questions", default="")
|
||||
parser.add_argument(
|
||||
"--allow-existing-output",
|
||||
action="store_true",
|
||||
help="Allow writing into an existing output directory without deleting it.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
base = resolve_path(args.base_dir)
|
||||
first_20 = resolve_path(args.first_20_dir) if args.first_20_dir else base / "paper_module_strict_official_20q_7907_20260624"
|
||||
remaining_80 = resolve_path(args.remaining_80_dir) if args.remaining_80_dir else base / "paper_module_remaining_80q_7907_20260624"
|
||||
output = resolve_path(args.output_dir) if args.output_dir else base / "paper_module_strict_official_100q_7907_20260624"
|
||||
questions_path = (
|
||||
resolve_path(args.questions)
|
||||
if args.questions
|
||||
else base / "00_inputs" / "QA100_questions_gold_answers_source_text_no_srt_audited_v1_20260624.json"
|
||||
)
|
||||
questions = json.loads(questions_path.read_text(encoding="utf-8-sig"))
|
||||
ids = [str(item["id"]) for item in questions]
|
||||
first_ids = {
|
||||
str(item["id"])
|
||||
for item in json.loads(
|
||||
(first_20 / "results" / "selected_questions.json").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
)
|
||||
}
|
||||
remaining_ids = {
|
||||
str(item["id"])
|
||||
for item in json.loads(
|
||||
(remaining_80 / "results" / "selected_questions.json").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
)
|
||||
}
|
||||
if first_ids & remaining_ids:
|
||||
raise RuntimeError(f"Overlapping IDs: {sorted(first_ids & remaining_ids)}")
|
||||
if first_ids | remaining_ids != set(ids):
|
||||
missing = set(ids) - (first_ids | remaining_ids)
|
||||
extra = (first_ids | remaining_ids) - set(ids)
|
||||
raise RuntimeError(f"Question mismatch missing={sorted(missing)} extra={sorted(extra)}")
|
||||
|
||||
if output.exists() and not args.allow_existing_output:
|
||||
raise FileExistsError(
|
||||
f"Refusing to write into existing output directory: {output}. "
|
||||
"Choose a new --output-dir or pass --allow-existing-output."
|
||||
)
|
||||
(output / "results").mkdir(parents=True, exist_ok=args.allow_existing_output)
|
||||
(output / "analysis").mkdir(parents=True, exist_ok=args.allow_existing_output)
|
||||
(output / "judge").mkdir(parents=True, exist_ok=args.allow_existing_output)
|
||||
(output / "results" / "selected_questions.json").write_text(
|
||||
json.dumps(questions, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
counts: dict[str, int] = {}
|
||||
for config in CONFIGS:
|
||||
target_dir = output / "results" / config
|
||||
target_dir.mkdir(parents=True, exist_ok=args.allow_existing_output)
|
||||
copied = 0
|
||||
for qid in ids:
|
||||
source_root = first_20 if qid in first_ids else remaining_80
|
||||
source = source_root / "results" / config / f"{qid}.json"
|
||||
if not source.exists():
|
||||
raise FileNotFoundError(source)
|
||||
record = json.loads(source.read_text(encoding="utf-8"))
|
||||
if record.get("error") or not record.get("response"):
|
||||
raise RuntimeError(f"Invalid result: {source}")
|
||||
shutil.copy2(source, target_dir / source.name)
|
||||
copied += 1
|
||||
counts[config] = copied
|
||||
manifest = {
|
||||
"questions": len(ids),
|
||||
"configs": CONFIGS,
|
||||
"expected_results": len(ids) * len(CONFIGS),
|
||||
"copied_results": sum(counts.values()),
|
||||
"per_config": counts,
|
||||
"sources": {
|
||||
"first_20": str(first_20),
|
||||
"remaining_80": str(remaining_80),
|
||||
"gold_questions": str(questions_path),
|
||||
},
|
||||
}
|
||||
(output / "merge_manifest.json").write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(json.dumps(manifest, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
PROJECT_ROOT = Path(
|
||||
os.environ.get("GENE_FAMILY_ROOT", "/data/gene-family-mas-v2")
|
||||
)
|
||||
GFANALYSIS_ROOT = Path(
|
||||
os.environ.get("GFANALYSIS_ROOT", "/data/wutx/00test/gfAnalysis")
|
||||
)
|
||||
for root in (PROJECT_ROOT, GFANALYSIS_ROOT):
|
||||
value = str(root)
|
||||
if value not in sys.path:
|
||||
sys.path.insert(0, value)
|
||||
|
||||
from src.rag.qa_v1_0_0.api import router
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="Plant Biology QA V1.0.0",
|
||||
version="1.0.0",
|
||||
description=(
|
||||
"Provisional-answer-guided, evidence-verified dual-KB QA. "
|
||||
"This service is independent from the legacy combined QA servers."
|
||||
),
|
||||
)
|
||||
app.include_router(router)
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health():
|
||||
return {"status": "ok", "service": "qa_v1_0_0", "version": "1.0.0"}
|
||||
@@ -0,0 +1,191 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
VARIANTS = {
|
||||
"v1_no_draft_one_round": {
|
||||
"use_provisional_draft": False,
|
||||
"max_rounds": 1,
|
||||
},
|
||||
"v1_draft_one_round": {
|
||||
"use_provisional_draft": True,
|
||||
"max_rounds": 1,
|
||||
},
|
||||
"v1_draft_two_rounds": {
|
||||
"use_provisional_draft": True,
|
||||
"max_rounds": 2,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def load_questions(path: Path) -> List[Dict[str, Any]]:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
if isinstance(payload, dict):
|
||||
for key in ("questions", "items", "data"):
|
||||
if isinstance(payload.get(key), list):
|
||||
payload = payload[key]
|
||||
break
|
||||
if not isinstance(payload, list):
|
||||
raise ValueError("Question JSON must be a list or contain questions/items/data")
|
||||
output = []
|
||||
for index, item in enumerate(payload, 1):
|
||||
if isinstance(item, str):
|
||||
output.append({"question_id": f"Q{index:03d}", "question": item})
|
||||
continue
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
question = str(
|
||||
item.get("question")
|
||||
or item.get("query")
|
||||
or item.get("text")
|
||||
or ""
|
||||
).strip()
|
||||
if question:
|
||||
output.append(
|
||||
{
|
||||
**item,
|
||||
"question_id": str(
|
||||
item.get("question_id")
|
||||
or item.get("id")
|
||||
or f"Q{index:03d}"
|
||||
),
|
||||
"question": question,
|
||||
}
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def run_one(
|
||||
endpoint: str,
|
||||
question: Dict[str, Any],
|
||||
variant_id: str,
|
||||
timeout: float,
|
||||
) -> Dict[str, Any]:
|
||||
config = VARIANTS[variant_id]
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
response = requests.post(
|
||||
endpoint,
|
||||
json={
|
||||
"question": question["question"],
|
||||
"debug": False,
|
||||
**config,
|
||||
},
|
||||
timeout=timeout,
|
||||
)
|
||||
payload = response.json()
|
||||
return {
|
||||
"question_id": question["question_id"],
|
||||
"question": question["question"],
|
||||
"variant_id": variant_id,
|
||||
"http_status": response.status_code,
|
||||
"success": bool(payload.get("success")) and response.status_code == 200,
|
||||
"answer": payload.get("answer", ""),
|
||||
"source_count": payload.get("source_count", 0),
|
||||
"round_count": len(payload.get("round_history", []) or []),
|
||||
"audit_path": payload.get("audit_path", ""),
|
||||
"timing": payload.get("timing", {}),
|
||||
"retrieval_hypothesis": payload.get("retrieval_hypothesis", {}),
|
||||
"claim_verification": payload.get(
|
||||
"provisional_claim_verification", []
|
||||
),
|
||||
"evidence_assessment": payload.get("evidence_assessment", {}),
|
||||
"sources": payload.get("sources", []),
|
||||
"error": payload.get("detail", ""),
|
||||
"client_elapsed": time.perf_counter() - started,
|
||||
}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"question_id": question["question_id"],
|
||||
"question": question["question"],
|
||||
"variant_id": variant_id,
|
||||
"success": False,
|
||||
"error": f"{type(exc).__name__}: {exc}",
|
||||
"client_elapsed": time.perf_counter() - started,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--questions", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--endpoint",
|
||||
default="http://127.0.0.1:7909/api/v1/qa-v1/chat",
|
||||
)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--variants",
|
||||
nargs="+",
|
||||
choices=list(VARIANTS),
|
||||
default=list(VARIANTS),
|
||||
)
|
||||
parser.add_argument("--limit", type=int, default=0)
|
||||
parser.add_argument("--timeout", type=float, default=900)
|
||||
args = parser.parse_args()
|
||||
|
||||
questions = load_questions(args.questions)
|
||||
if args.limit > 0:
|
||||
questions = questions[: args.limit]
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
summary = {
|
||||
"endpoint": args.endpoint,
|
||||
"question_file": str(args.questions),
|
||||
"question_count": len(questions),
|
||||
"variants": args.variants,
|
||||
"results": {},
|
||||
}
|
||||
for variant_id in args.variants:
|
||||
variant_dir = args.output_dir / variant_id
|
||||
variant_dir.mkdir(parents=True, exist_ok=True)
|
||||
successes = 0
|
||||
failures = 0
|
||||
for index, question in enumerate(questions, 1):
|
||||
target = variant_dir / f"{question['question_id']}.json"
|
||||
if target.exists():
|
||||
try:
|
||||
existing = json.loads(target.read_text(encoding="utf-8"))
|
||||
if existing.get("success"):
|
||||
successes += 1
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
result = run_one(
|
||||
args.endpoint,
|
||||
question,
|
||||
variant_id,
|
||||
args.timeout,
|
||||
)
|
||||
target.write_text(
|
||||
json.dumps(result, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
if result.get("success"):
|
||||
successes += 1
|
||||
else:
|
||||
failures += 1
|
||||
print(
|
||||
f"[{variant_id}] {index}/{len(questions)} "
|
||||
f"{question['question_id']} success={result.get('success')}",
|
||||
flush=True,
|
||||
)
|
||||
summary["results"][variant_id] = {
|
||||
"successes": successes,
|
||||
"failures": failures,
|
||||
}
|
||||
(args.output_dir / "run_summary.json").write_text(
|
||||
json.dumps(summary, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+67
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from src.rag.extractors.v16_1_2.v16_extractor import V16Extractor
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Run the formal V16.1.2 plant-science SRT extraction pipeline."
|
||||
)
|
||||
parser.add_argument("--pdf", required=True, type=Path)
|
||||
parser.add_argument("--output-dir", required=True, type=Path)
|
||||
parser.add_argument("--pmid", default="")
|
||||
parser.add_argument("--doi", default="")
|
||||
parser.add_argument("--disable-llm", action="store_true")
|
||||
parser.add_argument("--use-pc-data", action="store_true")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
pdf = args.pdf.resolve()
|
||||
output_dir = args.output_dir.resolve()
|
||||
if not pdf.exists():
|
||||
raise SystemExit(f"PDF does not exist: {pdf}")
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
extractor = V16Extractor(
|
||||
output_dir=output_dir,
|
||||
use_mineru=True,
|
||||
use_llm_relations=not args.disable_llm,
|
||||
use_rule_relation_fallback=False,
|
||||
use_vlm=False,
|
||||
use_neo4j=False,
|
||||
use_unified_storage=False,
|
||||
use_llm_definitions=False,
|
||||
use_pc_data=args.use_pc_data,
|
||||
use_disambiguation=args.use_pc_data,
|
||||
)
|
||||
result = extractor.extract(
|
||||
pdf,
|
||||
pmid=args.pmid or None,
|
||||
doi=args.doi or None,
|
||||
)
|
||||
summary = {
|
||||
"version": result.get("metadata", {}).get("version"),
|
||||
"pdf": str(pdf),
|
||||
"output_dir": str(output_dir),
|
||||
"entities": len(result.get("entities", [])),
|
||||
"relations": len(result.get("relations", [])),
|
||||
"relation_units": len(result.get("chunks", [])),
|
||||
"result_json": str(output_dir / "result.json"),
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+185
@@ -0,0 +1,185 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def build_run_command(
|
||||
row: dict,
|
||||
root: Path,
|
||||
output_dir: Path,
|
||||
python: Path,
|
||||
*,
|
||||
use_pc_data: bool = False,
|
||||
disable_llm: bool = False,
|
||||
) -> list[str]:
|
||||
command = [
|
||||
str(python),
|
||||
str(root / "scripts/run_v16_1_2_one_paper.py"),
|
||||
"--pdf",
|
||||
str(Path(row["destination_path"])),
|
||||
"--output-dir",
|
||||
str(output_dir),
|
||||
]
|
||||
pmid = str(row.get("pmid") or "").strip()
|
||||
doi = str(row.get("doi") or "").strip()
|
||||
if pmid:
|
||||
command.extend(["--pmid", pmid])
|
||||
if doi:
|
||||
command.extend(["--doi", doi])
|
||||
if use_pc_data:
|
||||
command.append("--use-pc-data")
|
||||
if disable_llm:
|
||||
command.append("--disable-llm")
|
||||
return command
|
||||
|
||||
|
||||
def run_one(
|
||||
row: dict,
|
||||
root: Path,
|
||||
output_root: Path,
|
||||
python: Path,
|
||||
*,
|
||||
use_pc_data: bool = False,
|
||||
disable_llm: bool = False,
|
||||
) -> dict:
|
||||
pdf = Path(row["destination_path"])
|
||||
paper_id = str(row.get("paper_id") or pdf.stem)
|
||||
out = output_root / paper_id
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
command = [
|
||||
*build_run_command(
|
||||
row,
|
||||
root,
|
||||
out,
|
||||
python,
|
||||
use_pc_data=use_pc_data,
|
||||
disable_llm=disable_llm,
|
||||
)
|
||||
]
|
||||
log_path = out / "run.log"
|
||||
started = time.time()
|
||||
with log_path.open("w", encoding="utf-8") as log:
|
||||
process = subprocess.run(
|
||||
command,
|
||||
cwd=root,
|
||||
env=os.environ.copy(),
|
||||
stdout=log,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
)
|
||||
ended = time.time()
|
||||
result_path = out / "result.json"
|
||||
result = {
|
||||
**row,
|
||||
"paper_id": paper_id,
|
||||
"output_dir": str(out),
|
||||
"result_json": str(result_path),
|
||||
"return_code": process.returncode,
|
||||
"wall_seconds": round(ended - started, 3),
|
||||
"success": process.returncode == 0 and result_path.exists(),
|
||||
}
|
||||
if result["success"]:
|
||||
command_text = " ".join(
|
||||
f"'{arg}'" if " " in arg else arg for arg in command
|
||||
)
|
||||
(out / "SUCCESSFUL_COMMAND.md").write_text(
|
||||
"# Successful V16.1.2 command\n\n"
|
||||
"Environment: current process environment/private `.env`.\n\n"
|
||||
f"```bash\n{command_text}\n```\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--manifest", required=True, type=Path)
|
||||
parser.add_argument("--output-root", required=True, type=Path)
|
||||
parser.add_argument("--workers", type=int, default=3)
|
||||
parser.add_argument("--use-pc-data", action="store_true")
|
||||
parser.add_argument("--disable-llm", action="store_true")
|
||||
parser.add_argument(
|
||||
"--project-root",
|
||||
type=Path,
|
||||
default=Path(os.environ.get("PLANTAI_PROJECT_ROOT", PROJECT_ROOT)),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--python",
|
||||
type=Path,
|
||||
default=Path(os.environ.get("PLANTAI_PYTHON", sys.executable)),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.workers < 1:
|
||||
raise SystemExit("--workers must be >= 1")
|
||||
rows = json.loads(args.manifest.read_text(encoding="utf-8"))
|
||||
args.output_root.mkdir(parents=True, exist_ok=True)
|
||||
started = time.time()
|
||||
results: list[dict] = []
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as pool:
|
||||
future_map = {
|
||||
pool.submit(
|
||||
run_one,
|
||||
row,
|
||||
args.project_root,
|
||||
args.output_root,
|
||||
args.python,
|
||||
use_pc_data=args.use_pc_data,
|
||||
disable_llm=args.disable_llm,
|
||||
): row
|
||||
for row in rows
|
||||
}
|
||||
for future in concurrent.futures.as_completed(future_map):
|
||||
row = future_map[future]
|
||||
try:
|
||||
result = future.result()
|
||||
except Exception as exc:
|
||||
result = {
|
||||
**row,
|
||||
"paper_id": Path(row["file_name"]).stem,
|
||||
"return_code": -1,
|
||||
"wall_seconds": 0,
|
||||
"success": False,
|
||||
"error": repr(exc),
|
||||
}
|
||||
results.append(result)
|
||||
results.sort(key=lambda item: int(item["index"]))
|
||||
(args.output_root / "batch_progress.json").write_text(
|
||||
json.dumps(results, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(
|
||||
f"[{len(results)}/{len(rows)}] "
|
||||
f"{result['paper_id']} success={result['success']} "
|
||||
f"wall={result['wall_seconds']}s",
|
||||
flush=True,
|
||||
)
|
||||
summary = {
|
||||
"started_epoch": started,
|
||||
"finished_epoch": time.time(),
|
||||
"wall_seconds": round(time.time() - started, 3),
|
||||
"workers": args.workers,
|
||||
"total": len(results),
|
||||
"success": sum(bool(item["success"]) for item in results),
|
||||
"failed": sum(not bool(item["success"]) for item in results),
|
||||
"results": results,
|
||||
}
|
||||
(args.output_root / "BATCH_RUN_SUMMARY.json").write_text(
|
||||
json.dumps(summary, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return 0 if summary["failed"] == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+43
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT=/data/gene-family-mas-v2
|
||||
GFANALYSIS_ROOT=/data/wutx/00test/gfAnalysis
|
||||
|
||||
if [[ -f "$ROOT/.env" ]]; then
|
||||
set -a
|
||||
source "$ROOT/.env"
|
||||
set +a
|
||||
fi
|
||||
|
||||
if [[ -f "$ROOT/.env.v15_deepseek" ]]; then
|
||||
set -a
|
||||
source "$ROOT/.env.v15_deepseek"
|
||||
set +a
|
||||
fi
|
||||
|
||||
if [[ -f "$ROOT/.runtime/taxonomy_bge_7907/service.env" ]]; then
|
||||
set -a
|
||||
source "$ROOT/.runtime/taxonomy_bge_7907/service.env"
|
||||
set +a
|
||||
fi
|
||||
|
||||
export GENE_FAMILY_ROOT="$ROOT"
|
||||
export GFANALYSIS_ROOT
|
||||
export PYTHONPATH="$ROOT:$GFANALYSIS_ROOT"
|
||||
|
||||
export V15_QDRANT_COLLECTION="${V15_QDRANT_COLLECTION:-v15_q1q2_1770_srt_bge_20260610}"
|
||||
export PC_QDRANT_COLLECTION="${PC_QDRANT_COLLECTION:-pc_srt_bge_full_20260620}"
|
||||
export BGE_RERANKER_MODEL="${BGE_RERANKER_MODEL:-$ROOT/bge-reranker-base}"
|
||||
|
||||
export QA_V1_ENABLE_PROVISIONAL_DRAFT="${QA_V1_ENABLE_PROVISIONAL_DRAFT:-1}"
|
||||
export QA_V1_ENABLE_SECOND_ROUND="${QA_V1_ENABLE_SECOND_ROUND:-1}"
|
||||
export QA_V1_ENABLE_BGE_RERANK="${QA_V1_ENABLE_BGE_RERANK:-1}"
|
||||
export QA_V1_BGE_DEVICE="${QA_V1_BGE_DEVICE:-cuda}"
|
||||
export QA_V1_MAX_ROUNDS="${QA_V1_MAX_ROUNDS:-2}"
|
||||
|
||||
cd "$ROOT"
|
||||
exec /data/miniconda3/envs/gfpy312/bin/python -m uvicorn \
|
||||
scripts.qa_v1_0_0_server:app \
|
||||
--host 0.0.0.0 \
|
||||
--port "${QA_V1_PORT:-7908}"
|
||||
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(os.environ.get("GENE_FAMILY_ROOT", "/data/gene-family-mas-v2"))
|
||||
GF = Path(os.environ.get("GFANALYSIS_ROOT", "/data/wutx/00test/gfAnalysis"))
|
||||
for path in (ROOT, GF):
|
||||
if str(path) not in sys.path:
|
||||
sys.path.insert(0, str(path))
|
||||
|
||||
from src.rag.qa_v1_0_0.pipeline import QAV1Pipeline
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("question")
|
||||
parser.add_argument("--no-draft", action="store_true")
|
||||
parser.add_argument("--max-rounds", type=int, default=2)
|
||||
parser.add_argument("--debug", action="store_true")
|
||||
args = parser.parse_args()
|
||||
response = QAV1Pipeline().answer(
|
||||
args.question,
|
||||
debug=args.debug,
|
||||
use_provisional_draft=not args.no_draft,
|
||||
max_rounds=args.max_rounds,
|
||||
)
|
||||
print(json.dumps(response, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user