From afa7cce9383e52d6c58e8fc1f9a9e1a42d5f0832 Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 12 Sep 2026 19:10:08 +0800 Subject: [PATCH] feat(aqua): unlock normalized KB identity via env with plant fallback - normalized_release EXPECTED_RELEASE defaults to NORMALIZED_RELEASE_ID (plant value) - from_env falls back to constant on empty env; backend_guard uses env-or-constant - start.sh sources env/active_library.env after prod.env (qa + legacy-rag) - build_all.sh --library renders library identity (no name whitelist) - add kb-libraries skeleton (plant/aqua identity cards) and tooling/aqua_kb Co-Authored-By: Claude Fable 5 --- .gitignore | 7 + agent/legacy-rag/scripts/start.sh | 4 + .../legacy-rag/src/rag/normalized_release.py | 11 +- agent/qa/scripts/start.sh | 4 + agent/qa/src/rag/normalized_release.py | 11 +- agent/qa/src/rag/qa_v1_1_73/backend_guard.py | 14 +- kb-libraries/aqua/configs/entity_alias.yaml | 159 +++++++++ .../aqua/configs/relation_taxonomy.yaml | 200 +++++++++++ kb-libraries/aqua/manifest.yaml | 7 + kb-libraries/aqua/registry/README.md | 1 + kb-libraries/plant/configs/README.md | 1 + kb-libraries/plant/manifest.yaml | 7 + kb-libraries/plant/registry/README.md | 1 + tooling/aqua_kb/pair_aqua_sources.py | 311 ++++++++++++++++++ tooling/aqua_kb/qc_sample.py | 156 +++++++++ tooling/aqua_kb/render_active_library_env.sh | 22 ++ tooling/build_all.sh | 27 ++ 17 files changed, 917 insertions(+), 26 deletions(-) create mode 100644 kb-libraries/aqua/configs/entity_alias.yaml create mode 100644 kb-libraries/aqua/configs/relation_taxonomy.yaml create mode 100644 kb-libraries/aqua/manifest.yaml create mode 100644 kb-libraries/aqua/registry/README.md create mode 100644 kb-libraries/plant/configs/README.md create mode 100644 kb-libraries/plant/manifest.yaml create mode 100644 kb-libraries/plant/registry/README.md create mode 100755 tooling/aqua_kb/pair_aqua_sources.py create mode 100755 tooling/aqua_kb/qc_sample.py create mode 100755 tooling/aqua_kb/render_active_library_env.sh diff --git a/.gitignore b/.gitignore index 04a6915..36e584a 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,10 @@ bioflow_service/bioflow.log # QA 数据库裸进程离线二进制载体（构建机从镜像/官网提取，不入 git） deploy/qa-db/bin/ + +# 水产库大文件目录(source 26G zip 解压产物、build 清单、registry 建库生成、data 运行数据) +kb-libraries/aqua/source/ +kb-libraries/aqua/build/ +kb-libraries/aqua/registry/* +!kb-libraries/aqua/registry/README.md +kb-libraries/aqua/data/ diff --git a/agent/legacy-rag/scripts/start.sh b/agent/legacy-rag/scripts/start.sh index 9f42ecc..66b3ace 100755 --- a/agent/legacy-rag/scripts/start.sh +++ b/agent/legacy-rag/scripts/start.sh @@ -33,6 +33,10 @@ set -a . "$ENV_FILE" set +a +if [[ -f "$AGENT_ROOT/env/active_library.env" ]]; then + set -a; . "$AGENT_ROOT/env/active_library.env"; set +a +fi + cd "$AGENT_ROOT" setsid "$PYTHON" -m uvicorn "$APP_MOD" \ --host "$BIND" --port "$PORT" \ diff --git a/agent/legacy-rag/src/rag/normalized_release.py b/agent/legacy-rag/src/rag/normalized_release.py index 4d2ed42..d500c2e 100644 --- a/agent/legacy-rag/src/rag/normalized_release.py +++ b/agent/legacy-rag/src/rag/normalized_release.py @@ -5,7 +5,8 @@ from dataclasses import dataclass from typing import Any, Iterable -EXPECTED_RELEASE = "plant_srt_pc2025_v3_20260817_normalized" +# 库名锁已 env 化：镜像文件 agent/legacy-rag/src/rag/normalized_release.py 必须逐字同步(md5 校验)。 +EXPECTED_RELEASE = os.environ.get("NORMALIZED_RELEASE_ID") or "plant_srt_pc2025_v3_20260817_normalized" MAX_SEEDS = 32 MAX_RESULTS = 200 @@ -37,12 +38,8 @@ class NormalizedReleaseSettings: @classmethod def from_env(cls) -> "NormalizedReleaseSettings": - release_id = os.environ.get("NORMALIZED_RELEASE_ID", "").strip() - collection = os.environ.get("V15_QDRANT_COLLECTION", "").strip() - if release_id != EXPECTED_RELEASE: - raise RuntimeError(f"release mismatch: {release_id!r}") - if collection != EXPECTED_RELEASE: - raise RuntimeError(f"collection mismatch: {collection!r}") + release_id = os.environ.get("NORMALIZED_RELEASE_ID", "").strip() or EXPECTED_RELEASE + collection = os.environ.get("V15_QDRANT_COLLECTION", "").strip() or EXPECTED_RELEASE return cls(release_id=release_id, qdrant_collection=collection) diff --git a/agent/qa/scripts/start.sh b/agent/qa/scripts/start.sh index 8908c1e..c666342 100755 --- a/agent/qa/scripts/start.sh +++ b/agent/qa/scripts/start.sh @@ -24,6 +24,10 @@ set -a . "$ENV_FILE" set +a +if [[ -f "$AGENT_ROOT/env/active_library.env" ]]; then + set -a; . "$AGENT_ROOT/env/active_library.env"; set +a +fi + # reranker 模型路径回退：env 未指定时用安装包内模型（install/models/bge-reranker-base） if [[ -z "${QA_V1_BGE_RERANKER_MODEL:-}" && -d "$AGENT_ROOT/../models/bge-reranker-base" ]]; then export QA_V1_BGE_RERANKER_MODEL="$AGENT_ROOT/../models/bge-reranker-base" diff --git a/agent/qa/src/rag/normalized_release.py b/agent/qa/src/rag/normalized_release.py index 4d2ed42..d500c2e 100644 --- a/agent/qa/src/rag/normalized_release.py +++ b/agent/qa/src/rag/normalized_release.py @@ -5,7 +5,8 @@ from dataclasses import dataclass from typing import Any, Iterable -EXPECTED_RELEASE = "plant_srt_pc2025_v3_20260817_normalized" +# 库名锁已 env 化：镜像文件 agent/legacy-rag/src/rag/normalized_release.py 必须逐字同步(md5 校验)。 +EXPECTED_RELEASE = os.environ.get("NORMALIZED_RELEASE_ID") or "plant_srt_pc2025_v3_20260817_normalized" MAX_SEEDS = 32 MAX_RESULTS = 200 @@ -37,12 +38,8 @@ class NormalizedReleaseSettings: @classmethod def from_env(cls) -> "NormalizedReleaseSettings": - release_id = os.environ.get("NORMALIZED_RELEASE_ID", "").strip() - collection = os.environ.get("V15_QDRANT_COLLECTION", "").strip() - if release_id != EXPECTED_RELEASE: - raise RuntimeError(f"release mismatch: {release_id!r}") - if collection != EXPECTED_RELEASE: - raise RuntimeError(f"collection mismatch: {collection!r}") + release_id = os.environ.get("NORMALIZED_RELEASE_ID", "").strip() or EXPECTED_RELEASE + collection = os.environ.get("V15_QDRANT_COLLECTION", "").strip() or EXPECTED_RELEASE return cls(release_id=release_id, qdrant_collection=collection) diff --git a/agent/qa/src/rag/qa_v1_1_73/backend_guard.py b/agent/qa/src/rag/qa_v1_1_73/backend_guard.py index 6fef734..4cc7579 100644 --- a/agent/qa/src/rag/qa_v1_1_73/backend_guard.py +++ b/agent/qa/src/rag/qa_v1_1_73/backend_guard.py @@ -71,18 +71,8 @@ def _qdrant_sample(url: str, collection: str) -> Dict[str, Any]: def _validate_normalized_backends() -> Dict[str, Any]: - release_id = os.environ.get("NORMALIZED_RELEASE_ID", "").strip() - if release_id != NORMALIZED_RELEASE: - raise BackendIdentityError( - f"normalized release mismatch: expected {NORMALIZED_RELEASE!r}, " - f"got {release_id!r}" - ) - collection = os.environ.get("V15_QDRANT_COLLECTION", "").strip() - if collection != NORMALIZED_RELEASE: - raise BackendIdentityError( - f"normalized collection mismatch: expected {NORMALIZED_RELEASE!r}, " - f"got {collection!r}" - ) + release_id = os.environ.get("NORMALIZED_RELEASE_ID", "").strip() or NORMALIZED_RELEASE + collection = os.environ.get("V15_QDRANT_COLLECTION", "").strip() or NORMALIZED_RELEASE qdrant_url = os.environ.get( "V15_QDRANT_URL", os.environ.get("QDRANT_URL", "http://127.0.0.1:6333") ).rstrip("/") diff --git a/kb-libraries/aqua/configs/entity_alias.yaml b/kb-libraries/aqua/configs/entity_alias.yaml new file mode 100644 index 0000000..aa36b88 --- /dev/null +++ b/kb-libraries/aqua/configs/entity_alias.yaml @@ -0,0 +1,159 @@ +Arabidopsis thaliana: + type: species + aliases: + - Arabidopsis thaliana + - A. thaliana + - Arabidopsis + - 拟南芥 + forbidden_expansions: + - Brassicaceae + - cruciferous plants + - 十字花科 + +Oryza sativa: + type: species + aliases: + - Oryza sativa + - O. sativa + - rice + - 水稻 + +Glycine max: + type: species + aliases: + - Glycine max + - G. max + - soybean + - soybean plant + - 大豆 + +Zea mays: + type: species + aliases: + - Zea mays + - Z. mays + - maize + - corn + - 玉米 + +Litchi chinensis: + type: species + aliases: + - Litchi chinensis + - L. chinensis + - litchi + - lychee + - 荔枝 + +Triticum aestivum: + type: species + retrieval_label: wheat + aliases: + - Triticum aestivum + - T. aestivum + - wheat + - bread wheat + - common wheat + - 小麦 + +Solanum lycopersicum: + type: species + aliases: + - Solanum lycopersicum + - S. lycopersicum + - tomato + - 番茄 + +Solanum melongena: + type: species + aliases: + - Solanum melongena + - S. melongena + - eggplant + - brinjal + - 茄子 + +Solanum tuberosum: + type: species + aliases: + - Solanum tuberosum + - S. tuberosum + - potato + - 马铃薯 + - 土豆 + +Ipomoea batatas: + type: species + aliases: + - Ipomoea batatas + - I. batatas + - sweet potato + - sweetpotato + - 甘薯 + - 红薯 + forbidden_expansions: + - Solanum tuberosum + - S. tuberosum + - potato + - 马铃薯 + - 土豆 + +GmP450 genes: + type: gene_family + aliases: + - GmP450 genes + - GmP450 gene family + - GmP450 + - soybean P450 genes + - soybean cytochrome P450 genes + +SmCYP450 genes: + type: gene_family + aliases: + - SmCYP450 genes + - SmCYP450 gene family + - SmCYP450 + - eggplant CYP450 genes + - eggplant cytochrome P450 genes + +SPL gene family: + type: gene_family + retrieval_label: SPL genes + aliases: + - SPL gene family + - SPL 基因家族 + - SPL genes + - SPLs + - SPL + - SBP genes + - SBP-box genes + - Squamosa promoter-binding protein-like genes + - SQUAMOSA promoter binding protein-like genes + +flowering: + type: biological_process + retrieval_terms: + - ovary + - anthesis + - expression + aliases: + - flowering + - flowering time + - floral transition + - 开花 + +GST genes: + type: gene_family + aliases: + - GST genes + - GST gene family + - GSTs + - GST + - glutathione S-transferase genes + - glutathione S-transferase gene family + - glutathione S-transferases + - glutathione S-transferase + - 谷胱甘肽S-转移酶 + - 谷胱甘肽 S-转移酶 + - 谷胱甘肽S-转移酶基因 + - 谷胱甘肽S-转移酶基因家族 diff --git a/kb-libraries/aqua/configs/relation_taxonomy.yaml b/kb-libraries/aqua/configs/relation_taxonomy.yaml new file mode 100644 index 0000000..6436d13 --- /dev/null +++ b/kb-libraries/aqua/configs/relation_taxonomy.yaml @@ -0,0 +1,200 @@ +expression: + location: + - expressed_in + - highly_expressed_in + - predominantly_expressed_in + - accumulated_at_higher_level_in + - detected_in + - enriched_in + up: + - upregulated_in + - induced_under + - induced_in + - increased_in + - increased_under + - enhanced_expression_in + - accumulated_at_higher_level_in + down: + - downregulated_in + - decreased_in + - decreased_under + - reduced_in + - repressed_in + condition: + - differentially_expressed_under + - expression_changed_under + - expression_changed_in + - responds_to + - responsive_to + - not_induced_under + +catalog_count: + - has_member_count + - identified_in + - systematically_identified_in + - genome_wide_identified_in + - were_identified_in + - was_identified_in + - identified + - were identified in + - was identified in + - identified in + - contains + - contain + - encodes + - include + - includes + - had + - has_number_of_members + - comprehensively_analyzed_in + - analyzed_in + - surveyed_in + +classification: + - belongs_to + - member_of + - classified_into + - divided_into + - separated_into + - grouped_into + - clustered_with + +localization: + - localizes_to + - located_in + - predicted_to_localize_to + - subcellular_localized_to + +structure: + - contains_domain + - contains_motif + - contains_cis_element + - has_intron_count + - has_exon_count + - has_conserved_domain + - lacks_binding_site_for + +function_effect: + - promotes + - inhibits + - enhances + - reduces + - affects + - regulates + - involved_in + - participates_in + - contributes_to + - associated_with + +perturbation_effect: + - overexpression_increases + - overexpression_enhances + - overexpression_promotes + - overexpression_causes + - knockdown_reduces + - knockdown_enhances + - silencing_reduces + - mutation_affects + +method: + - used_tool + - analyzed_by + - predicted_by + - aligned_with + - constructed_with + - identified_by + +dataset: + - has_accession + - available_at + - has_data_type + - from_database + - from_article + - used_for + +prior_work: + - reported_in + - previously_reported_in + - cited_as + - known_to + +stress_response: + - responds_to + - responsive_to + - induced_by + - induced_under + - induced_in + - upregulated_under + - upregulated_in + - involved_in_stress_response + - associated_with_stress_tolerance + - confers_tolerance_to + - enhances_tolerance_to + - improves_tolerance_to + - stress tolerance + - abiotic stress + - biotic stress + +detoxification: + - involved_in_detoxification + - detoxifies + - detoxification + - conjugates_with_glutathione + - catalyzes_glutathione_conjugation + - scavenges + - removes + - clears + - reduces_ros + - maintains_redox_homeostasis + - glutathione conjugation + - xenobiotic detoxification + +oxidative_stress: + - scavenges_ros + - scavenges_h2o2 + - reduces_h2o2 + - regulates_redox_homeostasis + - enhances_oxidative_stress_tolerance + - involved_in_antioxidant_response + - reactive oxygen species + - oxidative stress + - redox homeostasis + +functional_validation: + - overexpression_enhances + - overexpression_increases + - overexpression_confers + - overexpression_improves + - overexpression_promotes + - over-expression + - overexpression + - co_expression_enhances + - co-expression + - transgenic + - mutation_affects + - knockdown_reduces + - silencing_reduces + +family_expansion: + - expanded_by + - tandemly_duplicated + - segmentally_duplicated + - retained_after_wgd + - underwent_positive_selection + - functionally_diverged + - supergene family + - gene family expansion + - tandem duplication + - segmental duplication + - positive selection + +signaling_regulation: + - upregulated_by + - activated_by + - regulated_by + - downstream_of + - target_of + - ABA signaling + - signal transduction + - kinase + diff --git a/kb-libraries/aqua/manifest.yaml b/kb-libraries/aqua/manifest.yaml new file mode 100644 index 0000000..0d42840 --- /dev/null +++ b/kb-libraries/aqua/manifest.yaml @@ -0,0 +1,7 @@ +name: aqua +release_id: aqua_srt_v1_20260930_normalized +neo4j_database: aqua-normalized +qdrant_collection: aqua_srt_v1_20260930_normalized +alias_config: configs/rag/entity_alias.yaml +taxonomy_config: configs/rag/relation_taxonomy.yaml +registry_path: "" diff --git a/kb-libraries/aqua/registry/README.md b/kb-libraries/aqua/registry/README.md new file mode 100644 index 0000000..792dc52 --- /dev/null +++ b/kb-libraries/aqua/registry/README.md @@ -0,0 +1 @@ +留空——aqua 库 registry 由建库阶段生成后放入此目录;此目录仅为 kb-libraries 结构占位。 diff --git a/kb-libraries/plant/configs/README.md b/kb-libraries/plant/configs/README.md new file mode 100644 index 0000000..7d8e1c9 --- /dev/null +++ b/kb-libraries/plant/configs/README.md @@ -0,0 +1 @@ +留空——plant 库沿用编译源 agent/qa/configs/rag/ 与部署包内置配置;此目录仅为 kb-libraries 结构占位。 diff --git a/kb-libraries/plant/manifest.yaml b/kb-libraries/plant/manifest.yaml new file mode 100644 index 0000000..487af49 --- /dev/null +++ b/kb-libraries/plant/manifest.yaml @@ -0,0 +1,7 @@ +name: plant +release_id: plant_srt_pc2025_v3_20260817_normalized +neo4j_database: neo4j +qdrant_collection: plant_srt_pc2025_v3_20260817_normalized +alias_config: "" +taxonomy_config: "" +registry_path: "" diff --git a/kb-libraries/plant/registry/README.md b/kb-libraries/plant/registry/README.md new file mode 100644 index 0000000..7d8e1c9 --- /dev/null +++ b/kb-libraries/plant/registry/README.md @@ -0,0 +1 @@ +留空——plant 库沿用编译源 agent/qa/configs/rag/ 与部署包内置配置;此目录仅为 kb-libraries 结构占位。 diff --git a/tooling/aqua_kb/pair_aqua_sources.py b/tooling/aqua_kb/pair_aqua_sources.py new file mode 100755 index 0000000..9f2a31f --- /dev/null +++ b/tooling/aqua_kb/pair_aqua_sources.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""pair_aqua_sources.py — 水产库 source 目录 MD 卡与 PDF 文件名 DOI 配对联表工具。 + +输入(--source ): + 题目摘要_MD/*.md MD 卡(DOI 两种形态: https://doi.org/... 与 DOI：...) + PDF正文/*.pdf 文件名格式: 年份_标题_doi(下划线化)_8位hex.pdf + DOI汇总.txt (qc_sample 检材,本工具不读) + +产出(--out ): + manifest.csv doi,doi_norm,md_path,pdf_path,status,title,year,md_size,pdf_size,pdf_sha256 + 按 doi_norm 排序;status ∈ paired|md_only|pdf_only; + pdf_sha256 仅 pdf 行有值(ThreadPoolExecutor(8) 并行计算)。 + census.json {md_count,pdf_count,paired,md_only,pdf_only,undecodable, + doi_conflicts,generated_at,source_dir}(与 manifest 同目录) + +只依赖标准库。写盘先写 *.tmp 再 os.replace,重跑幂等。 +""" +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import re +import sys +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from pathlib import Path + +MD_URL_RE = re.compile(r"https?://doi\.org/([^\s\)\]）]+)") +MD_DOI_RE = re.compile(r"DOI：([^\s\]（(]+)") +MD_TITLE_RE = re.compile(r"^#\s+(.+)$") +MD_YEAR_RE = re.compile(r"年份[：:]\s*(\d{4})") +DIGITS_RE = re.compile(r"^\d{4,9}$") +HEX8_RE = re.compile(r"^[0-9a-fA-F]{8}$") + +CSV_HEADER = [ + "doi", "doi_norm", "md_path", "pdf_path", "status", + "title", "year", "md_size", "pdf_size", "pdf_sha256", +] + + +def norm_doi(d: str) -> str: + """归一化:去首尾空白、小写、去尾部标点;内部 _ 不换 /(MD 卡里已是斜杠形态)。""" + return d.strip().lower().rstrip(".,;)]") + + +def extract_md_doi(text: str) -> str: + """两种 DOI 形态均尝试,取首个非空。""" + m = MD_URL_RE.search(text) + if not m: + m = MD_DOI_RE.search(text) + return m.group(1).strip() if m else "" + + +def md_metadata(text: str) -> tuple[str, str]: + title = "" + for line in text.splitlines(): + m = MD_TITLE_RE.match(line) + if m: + title = m.group(1).strip() + break + m = MD_YEAR_RE.search(text) + return title, m.group(1) if m else "" + + +def decode_pdf_doi(filename: str, md_norms) -> tuple[str, str]: + """从 PDF 文件名解码 DOI,返回 (doi_norm, doi_norm) 或 ("", "")。 + + 算法:找最后一个 10. 出现位置;此后按下划线切段;首段必须为 4-9 位数字 + (即「10 + 4-9 位数字」);从第 2 段起,段开头为字母数字的并入 DOI; + 末段恰好为 8 位十六进制(其后只剩 .pdf)则停止并入。歧义消解:对每处 + 合法的 10. 位置生成「前缀固定、尾部可延伸」的候选集合,优先取下划线 + 换成斜杠后能命中 md_norms 的最长解释;仍不中的取最短解释。 + """ + stem = filename[:-4] if filename.lower().endswith(".pdf") else filename + candidates: list[str] = [] + for pos in reversed([m.start() for m in re.finditer(r"10\.", stem)]): + segs = stem[pos + 3:].split("_") + if not segs or not DIGITS_RE.match(segs[0]): + continue # 首段不满足「10 + 4-9 位数字」,该 10. 位置无效 + parts = [segs[0]] + for i in range(1, len(segs)): + seg = segs[i] + if i == len(segs) - 1 and HEX8_RE.match(seg): + break # 8 位十六进制哈希(其后只剩 .pdf),停止并入 + if seg and seg[0].isalnum(): + parts.append(seg) + else: + break + for k in range(1, len(parts) + 1): + candidates.append(norm_doi("10." + "/".join(parts[:k]))) + if not candidates: + return "", "" + hits = [c for c in candidates if c in md_norms] + chosen = max(hits, key=len) if hits else min(candidates, key=len) + return chosen, chosen + + +def sha256_of(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def pdf_title_year(stem: str) -> tuple[str, str]: + """从文件名提取年份(首段 4 位数字)与标题(年份后、DOI 起始前)。""" + parts = stem.split("_") + year = parts[0] if len(parts) > 1 and re.fullmatch(r"\d{4}", parts[0]) else "" + title = "" + idx = stem.rfind("10.") + if idx > 0: + start = len(parts[0]) + 1 if year else 0 + title = stem[start:idx].strip("_").replace("_", " ").strip() + return title, year + + +def token_overlap(md_title: str, pdf_stem: str) -> int: + mt = set(re.findall(r"[a-z0-9]+", md_title.lower())) + pt = set(re.findall(r"[a-z0-9]+", pdf_stem.lower())) + return len(mt & pt) + + +def md_only_row(m: dict) -> dict: + return { + "doi": m["doi"], "doi_norm": m["doi_norm"], "md_path": m["md_path"], + "pdf_path": "", "status": "md_only", "title": m["title"], + "year": m["year"], "md_size": m["size"], "pdf_size": "", + "pdf_sha256": "", + } + + +def pdf_only_row(p: dict) -> dict: + return { + "doi": p["doi"], "doi_norm": p["doi_norm"], "md_path": "", + "pdf_path": p["pdf_path"], "status": "pdf_only", "title": p["title"], + "year": p["year"], "md_size": "", "pdf_size": p["size"], + "pdf_sha256": p["sha256"], + } + + +def main() -> int: + ap = argparse.ArgumentParser(description="水产库 MD 卡与 PDF 文件名 DOI 配对") + ap.add_argument("--source", required=True, help="aqua/source 目录") + ap.add_argument("--out", required=True, help="manifest.csv 输出路径(如 build/manifest.csv)") + args = ap.parse_args() + + src = Path(args.source) + md_dir = src / "题目摘要_MD" + pdf_dir = src / "PDF正文" + if not md_dir.is_dir() or not pdf_dir.is_dir(): + print(f"source 目录缺 题目摘要_MD 或 PDF正文: {src}", file=sys.stderr) + return 2 + + out_csv = Path(args.out) + out_csv.parent.mkdir(parents=True, exist_ok=True) + + # ---- MD 卡 ---- + md_list = [] + for f in sorted(md_dir.glob("*.md")): + text = f.read_text(encoding="utf-8", errors="replace") + doi = extract_md_doi(text) + title, year = md_metadata(text) + md_list.append({ + "doi": doi, + "doi_norm": norm_doi(doi) if doi else "", + "md_path": str(f.relative_to(src)), + "title": title, + "year": year, + "size": f.stat().st_size, + }) + md_norms = {m["doi_norm"] for m in md_list if m["doi_norm"]} + + # ---- PDF(sha256 并行) ---- + pdf_paths = sorted(pdf_dir.glob("*.pdf")) + pdf_sha: dict[str, str] = {} + with ThreadPoolExecutor(max_workers=8) as ex: + for rel, digest in ex.map( + lambda f: (str(f.relative_to(src)), sha256_of(f)), pdf_paths + ): + pdf_sha[rel] = digest + + pdf_list = [] + for f in pdf_paths: + rel = str(f.relative_to(src)) + doi, doi_norm = decode_pdf_doi(f.name, md_norms) + title, year = pdf_title_year(f.stem) + pdf_list.append({ + "doi": doi, + "doi_norm": doi_norm, + "pdf_path": rel, + "title": title, + "year": year, + "size": f.stat().st_size, + "sha256": pdf_sha[rel], + }) + + # ---- 配对 ---- + md_by_norm: dict[str, list] = defaultdict(list) + for m in md_list: + if m["doi_norm"]: + md_by_norm[m["doi_norm"]].append(m) + pdf_by_norm: dict[str, list] = defaultdict(list) + for p in pdf_list: + if p["doi_norm"]: + pdf_by_norm[p["doi_norm"]].append(p) + + rows: list[dict] = [] + paired = 0 + conflict_groups = 0 + for norm in sorted(set(md_by_norm) | set(pdf_by_norm)): + g_md = md_by_norm.get(norm, []) + g_pdf = pdf_by_norm.get(norm, []) + if len(g_md) == 1 and len(g_pdf) == 1: + m, p = g_md[0], g_pdf[0] + rows.append({ + "doi": m["doi"], "doi_norm": norm, "md_path": m["md_path"], + "pdf_path": p["pdf_path"], "status": "paired", + "title": m["title"], "year": m["year"], "md_size": m["size"], + "pdf_size": p["size"], "pdf_sha256": p["sha256"], + }) + paired += 1 + elif g_md and g_pdf: + conflict_groups += 1 + # DOI 一对多:按「文件名年份_标题前缀相似」择一配对,其余记独立行 + paired_md: set[str] = set() + remaining_pdf = list(g_pdf) + for m in g_md: + if not remaining_pdf: + break + scored = [ + (token_overlap(m["title"], Path(p["pdf_path"]).stem), p) + for p in remaining_pdf + ] + best_score, best_p = max(scored, key=lambda t: t[0]) + if best_score <= 0: + break + remaining_pdf.remove(best_p) + paired_md.add(m["md_path"]) + rows.append({ + "doi": m["doi"], "doi_norm": norm, "md_path": m["md_path"], + "pdf_path": best_p["pdf_path"], "status": "paired", + "title": m["title"], "year": m["year"], "md_size": m["size"], + "pdf_size": best_p["size"], "pdf_sha256": best_p["sha256"], + }) + paired += 1 + for m in g_md: + if m["md_path"] not in paired_md: + rows.append(md_only_row(m)) + for p in remaining_pdf: + rows.append(pdf_only_row(p)) + else: + for m in g_md: + rows.append(md_only_row(m)) + for p in g_pdf: + rows.append(pdf_only_row(p)) + + # 无 DOI 的 MD 卡 / 不可解码的 PDF(无法定位 10.x) + for m in md_list: + if not m["doi_norm"]: + rows.append(md_only_row(m)) + for p in pdf_list: + if not p["doi_norm"]: + rows.append(pdf_only_row(p)) + + md_only = sum(1 for r in rows if r["status"] == "md_only") + pdf_only = sum(1 for r in rows if r["status"] == "pdf_only") + undecodable = sum(1 for p in pdf_list if not p["doi_norm"]) + # 三类合计 == md_count + pdf_count - paired(paired 行内 md+pdf 各计 1) + assert paired + md_only + pdf_only == len(md_list) + len(pdf_list) - paired, \ + f"census 算术自检失败: paired={paired} md_only={md_only} pdf_only={pdf_only}" + census = { + "md_count": len(md_list), + "pdf_count": len(pdf_list), + "paired": paired, + "md_only": md_only, + "pdf_only": pdf_only, + "undecodable": undecodable, + "doi_conflicts": conflict_groups, + "generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "source_dir": str(src), + } + + rows.sort(key=lambda r: (r["doi_norm"], r["md_path"] or r["pdf_path"])) + tmp_csv = out_csv.with_name(out_csv.name + ".tmp") + with tmp_csv.open("w", encoding="utf-8", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=CSV_HEADER) + w.writeheader() + w.writerows(rows) + tmp_csv.replace(out_csv) + + census_path = out_csv.with_name("census.json") + tmp_census = census_path.with_name("census.json.tmp") + with tmp_census.open("w", encoding="utf-8") as fh: + json.dump(census, fh, ensure_ascii=False, indent=2) + tmp_census.replace(census_path) + + print(f"md={len(md_list)} pdf={len(pdf_list)} " + f"paired={paired} md_only={md_only} pdf_only={pdf_only} " + f"undecodable={undecodable} doi_conflicts={conflict_groups}") + print(f"manifest: {out_csv}") + print(f"census: {census_path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tooling/aqua_kb/qc_sample.py b/tooling/aqua_kb/qc_sample.py new file mode 100755 index 0000000..815db69 --- /dev/null +++ b/tooling/aqua_kb/qc_sample.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""qc_sample.py — 抽取 manifest.csv 三种状态样本并做可核验的规则检查。 + +用法: qc_sample.py [--manifest build/manifest.csv] [--source aqua/source] + [--n 20] [--seed 42] + +规则(任一不通过 exit 1): + paired: md_path、pdf_path 都存在;md 文本 grep 到 doi_norm(斜杠或下划线形态 + 任一命中);PDF 文件名归一化后等于该 doi_norm。 + md_only: pdf_path 为空;PDF正文 目录下无 ** 与 + *<斜杠换成下划线>* 文件;DOI汇总.txt 中 grep 到该 DOI(两种形态)。 + pdf_only: md_path 为空;题目摘要_MD 目录下 grep 不到该 DOI(两种形态)。 + +只依赖标准库。 +""" +from __future__ import annotations + +import argparse +import csv +import random +import re +import sys +from pathlib import Path + + +def norm_doi(d: str) -> str: + """DOI 归一化(与 pair_aqua_sources 同规则)。""" + return d.strip().lower().rstrip(".,;)]") + + +def slug_form(doi_norm: str) -> str: + """斜杠换下划线 + 点换下划线:文件名 DOI 段的下划线形态。""" + return doi_norm.replace("/", "_").replace(".", "_") + + +def grep_pdf_norm(filename: str, doi_norm: str) -> bool: + """PDF 文件名归一化后是否等于 doi_norm(复用 pair 的解码算法,取等值判定)。""" + stem = filename[:-4] if filename.lower().endswith(".pdf") else filename + cands: list[str] = [] + for pos in reversed([m.start() for m in re.finditer(r"10\.", stem)]): + segs = stem[pos + 3:].split("_") + if not segs or not re.fullmatch(r"\d{4,9}", segs[0]): + continue + parts = [segs[0]] + for i in range(1, len(segs)): + seg = segs[i] + if i == len(segs) - 1 and re.fullmatch(r"[0-9a-fA-F]{8}", seg): + break + if seg and seg[0].isalnum(): + parts.append(seg) + else: + break + for k in range(1, len(parts) + 1): + cands.append(norm_doi("10." + "/".join(parts[:k]))) + return doi_norm in cands + + +def grep_file_text(f: Path, needle_slash: str, needle_under: str) -> bool: + if not f.is_file(): + return False + text = f.read_text(encoding="utf-8", errors="replace") + return needle_slash in text or needle_under in text + + +def main() -> int: + ap = argparse.ArgumentParser(description="manifest 状态样本质量抽检") + ap.add_argument("--manifest", default="build/manifest.csv") + ap.add_argument("--source", default="aqua/source") + ap.add_argument("--n", type=int, default=20) + ap.add_argument("--seed", type=int, default=42) + args = ap.parse_args() + + manifest = Path(args.manifest) + src = Path(args.source) + md_dir = src / "题目摘要_MD" + pdf_dir = src / "PDF正文" + doi_txt = src / "DOI汇总.txt" + + rows = list(csv.DictReader(manifest.open(encoding="utf-8", newline=""))) + groups: dict[str, list[dict]] = {"paired": [], "md_only": [], "pdf_only": []} + for r in rows: + if r["status"] in groups: + groups[r["status"]].append(r) + + rng = random.Random(args.seed) + samples = {st: rng.sample(groups[st], min(args.n, len(groups[st]))) for st in groups} + + def check_md_text(doi_norm: str, md_path: str) -> bool: + text = (src / md_path).read_text(encoding="utf-8", errors="replace") + return doi_norm in text or slug_form(doi_norm) in text + + def check_md_dir_grep(doi_norm: str) -> bool: + n_slash = doi_norm + n_under = slug_form(doi_norm) + for f in md_dir.glob("*.md"): + t = f.read_text(encoding="utf-8", errors="replace") + if n_slash in t or n_under in t: + return True + return False + + fails = 0 + checked: dict[str, int] = {} + + print("== qc_sample ==") + print(f"manifest={manifest} rows={len(rows)} " + f"(paired={len(groups['paired'])} md_only={len(groups['md_only'])} " + f"pdf_only={len(groups['pdf_only'])}) seed={args.seed} n={args.n}\n") + + for st in ("paired", "md_only", "pdf_only"): + print(f"--- {st} (抽 {len(samples[st])} 条) ---") + for r in samples[st]: + dn = r["doi_norm"]; md = r["md_path"]; pdf = r["pdf_path"] + ok = True + if st == "paired": + if not (md and pdf and (src / md).is_file() and (src / pdf).is_file()): + ok = False + elif not check_md_text(dn, md): + ok = False + elif not grep_pdf_norm(Path(pdf).name, dn): + ok = False + elif st == "md_only": + if pdf: + ok = False + else: + has_glob = any( + pdf_dir.glob("*" + slug_form(dn) + "*") + ) or any( + pdf_dir.glob("*" + dn.replace("/", "_") + "*") + ) + if has_glob: + ok = False + elif not grep_file_text(doi_txt, dn, slug_form(dn)): + ok = False + else: # pdf_only + if md: + ok = False + elif check_md_dir_grep(dn): + ok = False + if not ok: + fails += 1 + print(f"[{'PASS' if ok else 'FAIL'}] {st:8s} " + f"doi={dn[:46]:46s} md={Path(md).name if md else '-':20s} " + f"pdf={Path(pdf).name if pdf else '-'}") + checked[st] = len(samples[st]) + + print("\n== 结果 ==") + print(f"checked={sum(checked.values())} fails={fails}") + for st in ("paired", "md_only", "pdf_only"): + print(f" {st}: {checked[st]} 条") + print("PASS" if fails == 0 else "FAIL") + return 1 if fails else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tooling/aqua_kb/render_active_library_env.sh b/tooling/aqua_kb/render_active_library_env.sh new file mode 100755 index 0000000..938b17a --- /dev/null +++ b/tooling/aqua_kb/render_active_library_env.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# 渲染 active_library.env:读 <库目录>/manifest.yaml,输出五触点 env。 +# 用法: render_active_library_env.sh <输出文件> +# config 类路径:manifest 值为相对路径(如 configs/rag/entity_alias.yaml)时渲染为 $QA_ROOT/<值>; +# 为空时回退 $QA_ROOT/configs/rag/<缺省文件名>。registry_path 为空则整行省略。 +set -euo pipefail +MANIFEST="$1"; OUT="$2" +release_id="$(awk -F': *' '/^release_id:/ {gsub(/[" ]/, "", $2); print $2; exit}' "$MANIFEST")" +collection="$(awk -F': *' '/^qdrant_collection:/ {gsub(/[" ]/, "", $2); print $2; exit}' "$MANIFEST")" +alias_cfg="$(awk -F': *' '/^alias_config:/ {gsub(/[" ]/, "", $2); print $2; exit}' "$MANIFEST")" +tax_cfg="$(awk -F': *' '/^taxonomy_config:/ {gsub(/[" ]/, "", $2); print $2; exit}' "$MANIFEST")" +registry="$(awk -F': *' '/^registry_path:/ {gsub(/[" ]/, "", $2); print $2; exit}' "$MANIFEST")" +[ -n "$release_id" ] || { echo "manifest 缺 release_id" >&2; exit 1; } +tmp="$OUT.tmp" +{ + echo "NORMALIZED_RELEASE_ID=$release_id" + echo "V15_QDRANT_COLLECTION=${collection:-$release_id}" + [ -n "$alias_cfg" ] && echo "QA_V1_ALIAS_CONFIG=\$QA_ROOT/$alias_cfg" || echo "QA_V1_ALIAS_CONFIG=\$QA_ROOT/configs/rag/entity_alias.yaml" + [ -n "$tax_cfg" ] && echo "QA_V1_TAXONOMY_CONFIG=\$QA_ROOT/$tax_cfg" || echo "QA_V1_TAXONOMY_CONFIG=\$QA_ROOT/configs/rag/relation_taxonomy.yaml" + [ -n "$registry" ] && echo "QA_PAPER_METADATA_REGISTRY=\$QA_ROOT/$registry" +} > "$tmp" +mv "$tmp" "$OUT" diff --git a/tooling/build_all.sh b/tooling/build_all.sh index daec6b5..58a0db5 100755 --- a/tooling/build_all.sh +++ b/tooling/build_all.sh @@ -8,6 +8,25 @@ umask 027 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" . "$SCRIPT_DIR/tooling.env" + +# ---- 0. 库身份(kb-libraries/)——不枚举库名,任意含 manifest.yaml 的目录均可 ---- +RAG_LIBRARY="${RAG_LIBRARY:-plant}" +rest_args=() +for arg in "$@"; do + case "$arg" in + --library=*) RAG_LIBRARY="${arg#--library=}" ;; + --library) RAG_LIBRARY="__next__" ;; + *) if [ "$RAG_LIBRARY" = "__next__" ]; then RAG_LIBRARY="$arg"; else rest_args+=("$arg"); fi ;; + esac +done +[ "$RAG_LIBRARY" = "__next__" ] && { echo "build_all: --library 缺值" >&2; exit 2; } +LIBRARY_DIR="$PROJECT_ROOT/kb-libraries/$RAG_LIBRARY" +[ -d "$LIBRARY_DIR" ] || LIBRARY_DIR="$RAG_LIBRARY" # 允许传任意目录路径 +[ -f "$LIBRARY_DIR/manifest.yaml" ] || { echo "build_all: 库目录 $LIBRARY_DIR 缺 manifest.yaml" >&2; exit 2; } +grep -qE '^release_id:' "$LIBRARY_DIR/manifest.yaml" || { echo "build_all: manifest.yaml 缺 release_id" >&2; exit 2; } +echo "库身份: $RAG_LIBRARY ($LIBRARY_DIR)" +set -- "${rest_args[@]}" + VERSION="${1:-$VERSION}" BUILD_ROOT="$SCRIPT_DIR/build/$VERSION" RELEASE_ROOT="$BUILD_ROOT/install" @@ -127,6 +146,14 @@ for t in qa methodology legacy-rag; do cp "$PROJECT_ROOT/agent/$t/env/prod.env.example" "$RELEASE_ROOT/$t/env/prod.env.example" 2>/dev/null || true done cp "$PROJECT_ROOT/sdk/.env.example" "$RELEASE_ROOT/sdk/.env.example" 2>/dev/null || true +# ---- 库身份装包:manifest + configs + active_library.env ---- +mkdir -p "$RELEASE_ROOT/install/kb" "$RELEASE_ROOT/qa/env" +cp "$LIBRARY_DIR/manifest.yaml" "$RELEASE_ROOT/install/kb/manifest.yaml" +if [ -d "$LIBRARY_DIR/configs" ]; then + cp "$LIBRARY_DIR"/configs/*.yaml "$RELEASE_ROOT/qa/configs/rag/" 2>/dev/null || true +fi +bash "$PROJECT_ROOT/tooling/aqua_kb/render_active_library_env.sh" \ + "$LIBRARY_DIR/manifest.yaml" "$RELEASE_ROOT/qa/env/active_library.env" # KB 包(定版后) [ -f "$BUILD_ROOT/knowledge-$VERSION.tar" ] && cp "$BUILD_ROOT/knowledge-$VERSION.tar"* "$RELEASE_ROOT/" # 部署脚本