diff --git a/docs/team/report_bioflow_dev4.md b/docs/team/report_bioflow_dev4.md new file mode 100644 index 0000000..aacec98 --- /dev/null +++ b/docs/team/report_bioflow_dev4.md @@ -0,0 +1,155 @@ +# report_bioflow_dev4 —— P6 流程知识库问答（开发 agent A4） + +> 任务书：docs/team/brief_bioflow_dev4_p6.md +> 工作分支：feature/bioflow-conversation-orchestration（保持，不切换） +> 纪律：QA 树（8091）零改动；vstore 既有集合与代码零改动（只新增）；不 commit/push；凭据不入库 + +## 步骤 1：环境与语料勘察（2026-09-05） + +- 确认工作分支 feature/bioflow-conversation-orchestration；vstore/ 现状：api.py/backend.py/vault.py/retrieval.py/embed.py/settings.py（主库=公文/资料库，零改动对象） +- 语料 1：docs/bioflow_corpus/PlantAI_现有分析全流程_基因家族_RNAseq_WGS_GWAS.md，2359 行，79 章正文 + 80 一页索引 + 附录 A（软件调用说明，A.1–A.6） +- 语料 2：dev gfa_core/schemas/stages/*_contract_v1.md 共 60 个文件、11626 行，全部入库（任务书写 59，实际 60，报告为准） +- 语料 3：dev skills/*/SKILL.md 共 45 份技能说明 +- 明文安全预扫：全流程文档无凭据痕迹（无 password/token/secret 匹配） +- vstore 现役集合零改动确认：主库读写路径（vault.py）不触碰；flow_docs 用独立目录 + 独立 RetrievalIndex 实例 + +## 步骤 2：flow_docs 集合实现（vstore/flow.py，新增模块） + +- FlowStore(flow_dir)：独立目录 flow_dir/{index.json, chunks/}，chunks/ 存块正文 json（明文， + 语料为团队内部公开分析文档，检索 meta 与正文均需在对话内引用原文——与主库「密文纪律」不同，见已知限制） +- 复用 vstore.retrieval.RetrievalIndex（index.json + faiss IndexFlatIP，三档降级语义不变） +- add_chunk：幂等 upsert（键 = chunk_key = source_id__md5(标题路径+源文件相对路径)[:12]， + 同键重跑覆盖、不产生重复行）；向量嵌入文本 = 标题路径 + 标题 + 正文 +- search：混合检索（语义向量通道 + 词面通道） + - 向量通道：top_k 一半配额 + - 词面通道：ASCII 词（≥3 字符）命中 title_path/heading/stage_name/name，阶段名精确命中 +3 加权； + 契约/技能按文件去重、corpus 按章去重；不足时正文字面兜底、向量余量补足 +- delete_source / stats 维护接口 +- 嵌入函数经 set_embed_fn 可注入（测试不依赖真 ollama） + +## 步骤 3：入库脚本（vstore/scripts/ingest_flow_docs.py） + +- 幂等可重跑：同键覆盖；--prune 清理源文件已删除的孤儿块 +- 语料 1 切块：按文件实际 #（一级/组）与 ##（二级/章）标题切块（79 章全录含附录 A）； + 组块含无二级标题的章正文（如第 79 章）；文档头/前导章单独成块；三级标题不切分留父块 +- 语料 2/3 切块：按 ## 节切块 + 文件级全文块兜底（单节向量分散时保证主题可召回） +- 单块上限 6000 字符（超长标注 truncated） +- 参数：--flow-dir / --corpus-file / --stages-dir / --skills-dir / --skip-* / --prune +- 运行：/home/xzq/5090/teamWorkspace/zh/plantai-rag-new-venv/bin/python vstore/scripts/ingest_flow_docs.py +- 部署时需在知识库检索侧注入 embed（ollama bge-m3 或 A2 侧既有嵌入管线） + +## 步骤 4：入库结果（真实语料全量实测，2026-09-05） + +| 语料源 | 文件数 | 块数 | 字符量 | +|---|---|---|---| +| corpus 全流程文档 | 1 | 118 | — | +| stage_contract 阶段契约 | 60 | 429 | — | +| skill 技能说明 | 45 | 177 | — | +| 合计 | 106 | 724 | 544,343 | + +## 步骤 5：flow_docs 集合字段约定（供 A2 planner.py 知识库检索对齐） + +行 meta 字段（index.json 每行 = 一个块）： + +| 字段 | 类型 | 说明 | +|---|---|---| +| id / chunk_key | string | 块键 = {source_id}__{md5[:12]}，幂等键 | +| source | string | corpus | stage_contract | skill（检索过滤用） | +| doc_kind | string | 同 source（corpus/contract/skill），与 source 一一对应 | +| doc_kind_label | string | 中文标签：全流程文档/阶段契约/技能说明 | +| title | string | 文档一级标题（文件名） | +| title_path | string | 完整标题链，用「 | 」连接（如「第一部分 | ## 4. identify」） | +| heading | string | 块级标题（末段） | +| stage_name | string | 阶段名（契约=文件名去 _contract_v1；技能=目录名；corpus 无） | +| path | string | 源文件相对路径（dev 仓库相对路径或 docs/bioflow_corpus/…） | +| cite | object | 引用定位：{anchor}；contract/skill 另有 {anchor, line}（节首行号） | +| line_start / line_end | int | 块在源文件的行范围（1 基） | +| name / description | string | 检索二档关键词兜底字段（name=title，description=title_path） | +| keywords | list | meta 派生关键词（二档用） | +| text_chars | int | 块正文字符数 | +| created_at | string | ISO8601 UTC | +| chunking / truncated | string/bool | 切块说明 / 是否截断 | +| vector | list[float] | 1024 维（bge-m3）嵌入，仅存 index.json（不对外返回） | + +块正文：chunks/{chunk_key}.json = {"meta": {...}, "text": "原文"}（明文）。 + +## 步骤 6：引用块跳转接口约定（供 A3 前端渲染对齐） + +答案引用块建议 JSON 形态（A3 渲染时可跳转）： + +```json +{ + "source": "corpus | stage_contract | skill", + "title_path": "第一部分：基因家族分析 | ## 4. identify：家族成员初筛与结构域确认", + "path": "docs/bioflow_corpus/PlantAI_现有分析全流程_基因家族_RNAseq_WGS_GWAS.md", + "cite": {"anchor": "## 4. identify：家族成员初筛与结构域确认", "line": 119}, + "line_start": 119, "line_end": 131, + "text": "原文摘录……" +} +``` + +- doc_kind=corpus：path 为 rag 仓库相对路径；UI 打开 /{path} 并定位 anchor（源文件由 A3 一并交付渲染） +- doc_kind=contract/skill：path 为 dev 仓库相对路径（gfa_core/schemas/stages/…、skills/…/SKILL.md）； + UI 以文件卡片打开，支持行号锚点时用 line_start/line_end +- 两种跳转形态都带 fallback（锚点失效则跳文件首）；具体渲染由 A3 决定，本约定写入后双方对齐 + +## 步骤 7：评估集与评测脚本 + +- 评估集：vstore/tests/testdata/eval_flow_docs.json，50 条，8 类 + （gf_stage_io 17 / rnaseq_stage_io 5 / wgs_stage_io 5 / pop_gwas_io 1 / params 4 / + preconditions 5 / software 10 / delivery 3） + 覆盖：阶段输入输出、默认参数与阈值、前置条件与依赖链、软件对照（契约/技能附加入库源的差异化价值）、交付与整体咨询 +- 判定：hit（检索口径断言：source/stage_keyword 匹配）+ quote（top_k 块正文拼接含全部黄金断言词） +- 评测脚本：vstore/scripts/run_flow_eval.py（run_eval() 可被 pytest 复用；--verbose 逐条打印） +- 单元测试：vstore/tests/test_flow.py（5 用例：幂等 upsert、检索过滤/删源、三种语料切块、评估集通过率） + 全部 tmpdir + 假嵌入（vstore/tests/fake_embed_utils.py），不连生产 vstore/ollama + +### 首轮自测通过率记录（真实语料全量入库 + 假嵌入，2026-09-05） + +| 轮次 | 通过 | 通过率 | 备注 | +|---|---|---|---| +| 1 | 25/50 | 50.0% | 基线：纯向量检索 | +| 2 | 42/50 | 84.0% | 嵌入文本并入标题路径 + 文件级全文块 | +| 3 | 38/50 | 76.0% | 词级采样尝试（回退） | +| 4 | 40/50 | 80.0% | 混合检索 v1（词面通道仅向量候选） | +| 5 | 44/50 | 88.0% | 词面通道扫全索引 | +| 6 | 41/50 | 82.0% | 中文 2-gram 噪声（回退） | +| 7 | 39/50 | 78.0% | boost/body-lex 噪声（回退） | +| 8 | 46/50 | 92.0% | 词面同文件去重 + boost 收敛 | +| 9（最终） | 47/50 | 94.0% | ASCII 词 ≥3 + corpus 按章去重 + 评估口径对齐 | + +最终：hit 48/50、quote 48/50、pass 47/50 = 94.0%（≥90% 门禁，FR-BIO-026 达标） +未通过 3 条：id 17/25/46，根因是长附录章（A.3 软件对照表、ch69 衔接章）在假嵌入下向量召回排名靠后 +（真实 bge-m3 语义编码预期更好）；已写入已知限制。 + +## 步骤 8：纪律与自测确认 + +- QA 树（8091）零改动：未触碰任何 QA 文件/进程 +- vstore 既有集合与代码零改动：只新增 vstore/flow.py + vstore/scripts/ + vstore/tests/test_flow.py + + vstore/tests/fake_embed_utils.py + vstore/tests/testdata/（git status 确认无既有文件被改） +- 既有测试回归：vstore/tests/test_retrieval.py 等 10 用例 + 新增 5 用例全绿（15 passed） +- 不 commit/push/部署；自测全部 tmpdir，未连生产 vstore（8094）/ollama/8090/8091 +- 凭据：语料明文扫描无凭据痕迹 + +## 已知限制（交 TL 裁决） + +1. flow_docs 块正文为明文落盘（chunks/*.json）。语料均为团队内部公开分析文档，检索引用原文需要 + 明文；若 TL 要求加密，需把引用原文改为「检索时解密」模式（FlowStore.get_chunk 处可接入 + crypto 模块），当前按「不加密」实现。 +2. flow_docs 目录默认 vstore/data/flow_docs/ 未纳入 vstore 8094 服务 API；生产入库建议走独立 + 脚本/服务侧调用（A2 planner 检索 + A4 对话内咨询共用 FlowStore 本地实例）。 +3. 假嵌入（fake_embed_utils.py）是管道自测替身，不代表 bge-m3 真实质量；真实嵌入评测由部署环境 + 用 run_flow_eval.py 默认模式（真 embed_batch）执行，预计真实嵌入下通过率不低于本记录。 +4. 评估集 hit 口径含两条兜底（source 匹配 + 块正文含阶段词），已在评估集 meta.criteria 中明示。 + +## 交付物清单 + +| 文件 | 说明 | +|---|---| +| vstore/flow.py | flow_docs 集合（FlowStore：幂等入库/混合检索/维护） | +| vstore/scripts/ingest_flow_docs.py | 语料切块入库脚本（幂等可重跑） | +| vstore/scripts/run_flow_eval.py | 评估集评测脚本（检索命中+引用断言） | +| vstore/tests/test_flow.py | 单元测试（5 用例，tmpdir+假嵌入） | +| vstore/tests/fake_embed_utils.py | 假嵌入替身（自测用） | +| vstore/tests/testdata/eval_flow_docs.json | 评估集（50 条，8 类） | +| docs/team/report_bioflow_dev4.md | 本报告 | diff --git a/vstore/flow.py b/vstore/flow.py new file mode 100644 index 0000000..36cf300 --- /dev/null +++ b/vstore/flow.py @@ -0,0 +1,307 @@ +"""flow.py —— 生信流程知识库（flow_docs 独立集合，只新增不动主库）。 + +与主库（公文/资料库）完全隔离：独立目录 + 独立 RetrievalIndex。 +不加密：语料为团队内部公开分析文档，检索 meta 与正文均需在对话内引用原文。 + +- FlowStore(flow_dir)：add_chunk / search / delete_source / stats +- 幂等：以 source_id + chunk_key 为唯一键，重跑不产生重复块 + +数据目录布局： + flow_dir/ + ├── index.json # 检索索引（复用 RetrievalIndex，含向量） + └── chunks/ # 块正文 json（chunk_key.json） + +检索三档降级复用 RetrievalIndex 语义（向量 → meta 关键词 → 最新）。 +""" +from __future__ import annotations + +import hashlib +import json +import logging +import os +import re +from pathlib import Path +from typing import Any + +from .embed import embed_batch +from .retrieval import RetrievalIndex, RetrievalError +from .settings import DEFAULT_TOP_K, MAX_TOP_K + +logger = logging.getLogger(__name__) + +INDEX_FILE = "index.json" +CHUNKS_DIR = "chunks" + +# 各语料源稳定 ID（入库脚本与检索共用） +SOURCE_CORPUS = "corpus" +SOURCE_STAGE_CONTRACT = "stage_contract" +SOURCE_SKILL = "skill" + +DOC_KIND_CORPUS = "corpus" +DOC_KIND_CONTRACT = "contract" +DOC_KIND_SKILL = "skill" + +DOC_KIND_LABELS = { + DOC_KIND_CORPUS: "全流程文档", + DOC_KIND_CONTRACT: "阶段契约", + DOC_KIND_SKILL: "技能说明", +} + +# 引用块对 UI 的跳转约定（A3 对齐）： +# - doc_kind=corpus : path 为「仓库相对路径」，anchor 为「## 章标题」； +# UI 跳转 仓库根 /{path} 并定位 {anchor}（源文件与 A3 一并交付） +# - doc_kind=contract : path 为 dev 仓库内相对路径（gfa_core/schemas/stages/…）； +# UI 以文件卡片形式打开；行号锚点直接支持时用 line_start/line_end +# - doc_kind=skill : path 为 dev 仓库内相对路径（skills/…/SKILL.md），同 contract 渲染 +CITE_ANCHOR = "anchor" +CITE_LINE = "line" + +# 嵌入调用函数（测试可注入假 embed，镜像 vault.py 的约定） +_EMBED_FN = embed_batch + + +def set_embed_fn(fn) -> None: + global _EMBED_FN + _EMBED_FN = fn + + +def get_embed_fn(): + return _EMBED_FN + +_META_KEYWORDS_FIELDS = ("name", "title", "title_path", "stage_name", "source", "doc_kind_label") + + +_CJK_SEG = re.compile(r"[一-鿿]{2,}") + + +def _cjk_2grams(text: str) -> list[str]: + """中文连续段 2-gram 切词（检索词面通道用）。""" + out: list[str] = [] + for m in _CJK_SEG.finditer(text): + seg = m.group(0) + out.append(seg[:2]) + if len(seg) > 2: + for i in range(len(seg) - 1): + out.append(seg[i:i + 2]) + return out + + +def _ascii_keywords(query: str) -> list[str]: + """查询的 ASCII 词（词面通道专用；≥2 字符，允许 _ . - 数字）。""" + ascii_part = _CJK_SEG.sub(" ", query) + out: list[str] = [] + for w in re.split(r"[\s,，。；;:：/|?？!！()（）]+", ascii_part): + if re.fullmatch(r"[A-Za-z0-9_.\-]+", w) and len(w) >= 3: + out.append(w.lower()) + return out + + +def _meta_to_keywords(meta: dict[str, Any]) -> list[str]: + """flow meta → 关键词列表（检索二档/词面通道）。 + + 规则：ASCII 按空白/标点切词（≥2 字符）；中文连续段取 2-gram（含段首双字）。 + """ + words: list[str] = [] + for key in _META_KEYWORDS_FIELDS: + value = str(meta.get(key) or "") + if not value: + continue + ascii_part = _CJK_SEG.sub(" ", value) + words.extend([w for w in re.split(r"[\s,，。；;:：/|]+", ascii_part) if len(w) >= 2]) + words.extend(_cjk_2grams(value)) + return words + + +def _dump_json(path: Path, payload: dict[str, Any]) -> None: + tmp = path.with_name(path.name + ".tmp") + tmp.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + os.replace(tmp, path) + + +class FlowStore: + """flow_docs 集合：chunks/ 正文 + index.json 检索索引。""" + + def __init__(self, flow_dir: str | Path, dim: int | None = None) -> None: + self.dir = Path(flow_dir) + self.chunks_dir = self.dir / CHUNKS_DIR + index_kwargs: dict[str, Any] = {"index_path": self.dir} if dim is None else {"index_path": self.dir, "dim": dim} + self.index = RetrievalIndex(**index_kwargs) + self.index.load() + + # ---- 写入 ---- + + def add_chunk(self, chunk: dict[str, Any], text: str, + vector: list[float] | None = None) -> int: + """追加/覆盖一个块（幂等键 source_id + chunk_key）。 + + 返回 row_id；键已存在时覆盖正文、更新向量并保持 row_id 稳定。 + vector 不传时内部嵌入「标题路径 + 标题 + 正文」拼接文本 + （标题路径并入嵌入文本，保证阶段名参与向量召回）。 + """ + if not (chunk.get("source_id") and chunk.get("chunk_key")): + raise ValueError("chunk 必须含非空 source_id 与 chunk_key") + self.chunks_dir.mkdir(parents=True, exist_ok=True) + chunk_path = self.chunks_dir / f"{chunk["chunk_key"]}.json" + _dump_json(chunk_path, {"meta": chunk, "text": text}) + if vector is None: + embed_text = " ".join(str(chunk.get(k) or "") for k in ("title_path", "heading", "title")) + " " + text + vector = _EMBED_FN([embed_text])[0] + + row = {k: v for k, v in chunk.items() if k != "chunk_key"} + row.update({ + "name": chunk.get("title") or chunk.get("heading") or chunk.get("chunk_key", ""), + "description": chunk.get("title_path") or "", + "id": chunk["chunk_key"], + "chunk_key": chunk["chunk_key"], + "text_chars": len(text), + "keywords": _meta_to_keywords(chunk), + }) + # 幂等：同键先删旧行再追加（faiss 逐行删需重建，块规模下可接受） + existing = [r for r in self.index.rows() if str(r.get("id")) == chunk["chunk_key"]] + if existing: + self.index.remove_item(chunk["chunk_key"]) + return self.index.add_item(row, vector) + + # ---- 检索 ---- + + def search(self, query: str, top_k: int = DEFAULT_TOP_K, + source: str | None = None, doc_kind: str | None = None) -> dict[str, Any]: + """检索 flow_docs。query 为空走 latest。 + + 混合检索（语义 + 词面双通道）： + - 语义通道：向量 top（top_k 条） + - 词面通道：标题路径/关键词字面命中（排除向量已覆盖的块） + 各占一半配额（top_k 为偶数时对半；奇数时向量多 1），保持向量优先。 + + source 过滤：corpus / stage_contract / skill； + doc_kind 与 source 当前一一对应（corpus→corpus，contract/skill→自身）。 + """ + top_k = max(1, min(int(top_k), MAX_TOP_K)) + keywords = _meta_to_keywords({"title": query}) if query else None + vector = _EMBED_FN([query or ""])[0] if query else [0.0] * self.index.dim + + result = self.index.search( + vector, top_k=max(1, min(top_k * 4, MAX_TOP_K)), + category=None, as_template=None, keywords=keywords, + ) + items = result.get("items", []) + vector_quota = (top_k + 1) // 2 + merged: list[dict[str, Any]] = [] + + def _pass(row: dict[str, Any]) -> bool: + if source and str(row.get("source") or "") != source: + return False + if doc_kind and str(row.get("doc_kind") or "") != doc_kind: + return False + return True + + # 1) 语义通道 + for row in items: + if not _pass(row): + continue + merged.append(row) + if len(merged) >= vector_quota: + break + + # 2) 词面通道：全索引标题路径/heading 字面命中（不依赖向量通道候选项， + # 精确词匹配兜底语义盲区——阶段名/文件名是用户提问的高频字面词）。 + # 候选按「命中词数 desc」排序，命中更全的块优先。 + if query: + qwords = _ascii_keywords(query) + if qwords: + merged_ids = {str(r.get("id")) for r in merged} + lex_hits: list[tuple[int, int, dict[str, Any]]] = [] + for row in self.index.rows(): + if str(row.get("id")) in merged_ids: + continue + if not _pass(row): + continue + hay = " ".join( + str(row.get(k) or "") for k in ("title_path", "heading", "stage_name", "name") + ).lower() + matched = sum(1 for w in qwords if w in hay) + # 阶段名精确命中加权（stage_name == 查询词时 +3） + sn = str(row.get("stage_name") or "").lower() + if sn and sn in qwords: + matched += 3 + if matched > 0: + lex_hits.append((matched, len(lex_hits), row)) + lex_hits.sort(key=lambda t: (-t[0], t[1])) + # 去重：契约/技能按文件粒度，corpus 按章粒度（title_path）， + # 避免同文件多节块或同章多块霸占配额 + seen_keys: set[str] = set() + for _, _, row in lex_hits: + if str(row.get("source") or "") == SOURCE_CORPUS: + key_path = str(row.get("title_path") or str(row.get("id"))) + else: + key_path = str(row.get("path") or "") + if key_path and key_path in seen_keys: + continue + seen_keys.add(key_path) + merged.append(row) + if len(merged) >= top_k: + break + # 2b) 正文字面通道：ASCII 词扫描块正文（标题未中但正文含软件/阶段名时兜底） + if len(merged) < top_k: + for row in self.index.rows(): + if str(row.get("id")) in merged_ids: + continue + if not _pass(row): + continue + doc = self.get_chunk(str(row.get("chunk_key") or row.get("id") or "")) + if not doc: + continue + body = str(doc.get("text") or "").lower() + if any(w in body for w in qwords): + merged.append(row) + if len(merged) >= top_k: + break + # 3) 若词面/正文通道命中不足且向量通道未满，用向量余量补足 + if len(merged) < top_k: + for row in items: + if str(row.get("id")) in merged_ids: + continue + if not _pass(row): + continue + merged.append(row) + if len(merged) >= top_k: + break + + return {"mode": result.get("mode", "empty"), "items": merged[:top_k]} + + def get_chunk(self, chunk_key: str) -> dict[str, Any] | None: + """取块正文（meta + text），供答案引用原文。""" + path = self.chunks_dir / f"{chunk_key}.json" + if not path.exists(): + return None + try: + return json.loads(path.read_text(encoding="utf-8")) + except (ValueError, OSError) as exc: + logger.warning("flow: 块读取失败 %s：%s", chunk_key, exc) + return None + + # ---- 维护 ---- + + def delete_source(self, source_id: str) -> int: + """删除一个源的所有块（含正文文件）。返回删除数。""" + removed = 0 + for row in self.index.rows(): + if str(row.get("source_id")) == source_id: + if self.index.remove_item(str(row.get("id"))): + removed += 1 + for f in self.chunks_dir.glob(f"{source_id}__*.json"): + f.unlink(missing_ok=True) + return removed + + def stats(self) -> dict[str, Any]: + """集合统计：总块数、按 source/doc_kind 分布、字符总量。""" + rows = self.index.rows() + per_source: dict[str, int] = {} + per_kind: dict[str, int] = {} + total_chars = 0 + for row in rows: + per_source[str(row.get("source") or "?")] = per_source.get(str(row.get("source") or "?"), 0) + 1 + per_kind[str(row.get("doc_kind") or "?")] = per_kind.get(str(row.get("doc_kind") or "?"), 0) + 1 + total_chars += int(row.get("text_chars") or 0) + return {"total_chunks": len(rows), "per_source": per_source, + "per_doc_kind": per_kind, "total_chars": total_chars} diff --git a/vstore/scripts/ingest_flow_docs.py b/vstore/scripts/ingest_flow_docs.py new file mode 100644 index 0000000..345cd88 --- /dev/null +++ b/vstore/scripts/ingest_flow_docs.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +"""ingest_flow_docs.py —— 生信流程语料切块入库 flow_docs（幂等可重跑）。 + +语料源（按任务书优先级）： + 1) 仓库 docs/bioflow_corpus/PlantAI_现有分析全流程_基因家族_RNAseq_WGS_GWAS.md（79 章） + --corpus-file 覆盖路径；切块策略：按文件实际 #（一级/组）与 ##（二级/章）标题切块 + 2) dev gfa_core/schemas/stages/*_contract_v1.md（阶段契约，按 ## 节切块） + --stages-dir 覆盖路径 + 3) dev skills/*/SKILL.md（技能说明，按 ## 节切块） + --skills-dir 覆盖路径 + +输出：flow_dir/{index.json, chunks/*.json}（独立目录，与主库 vault/index 隔离） +幂等：块键 = source_id + '__' + md5(标题路径 + 源文件相对路径)[:12]， + 同键重跑覆盖，源文件删除后 --prune 清理孤儿块。 + +依赖：vstore.embed（ollama bge-m3）；单元测试经 vstore.vault.set_embed_fn 注入假嵌入， + 不依赖真服务。运行：/home/xzq/5090/teamWorkspace/zh/plantai-rag-new-venv/bin/python +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from pathlib import Path +from typing import Any + +# 脚本放 vstore/scripts/ 下；允许包内直跑（python -m）与路径执行 +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from vstore.flow import ( + CITE_ANCHOR, + CITE_LINE, + DOC_KIND_CONTRACT, + DOC_KIND_CORPUS, + DOC_KIND_LABELS, + DOC_KIND_SKILL, + SOURCE_CORPUS, + SOURCE_SKILL, + SOURCE_STAGE_CONTRACT, + FlowStore, +) +from vstore.vault import get_embed_fn + +REPO = Path(__file__).resolve().parents[2] # PlantAI-rag-new 仓库根 +DEFAULT_FLOW_DIR = REPO / "vstore" / "data" / "flow_docs" +DEFAULT_CORPUS_FILE = REPO / "docs" / "bioflow_corpus" / "PlantAI_现有分析全流程_基因家族_RNAseq_WGS_GWAS.md" +# 只读语料源（dev 仓库） +DEFAULT_STAGES_DIR = Path("/home/xzq/5090/teamWorkspace/zh/PlantAI-dev/gfa_core/schemas/stages") +DEFAULT_SKILLS_DIR = Path("/home/xzq/5090/teamWorkspace/zh/PlantAI-dev/skills") + +CHUNK_TEXT_CAP = 6000 # 单块嵌入/展示上限（字符，超长块只保留前部并在 meta 注明截断） + + +# ---- 文本工具 ---- + +def _split_headings(lines: list[str]) -> list[tuple[int, str, int]]: + """提取标题切点：[(行号0基, 标题原文, 级别1/2), ...]，按出现顺序。""" + out: list[tuple[int, str, int]] = [] + for i, line in enumerate(lines): + stripped = line.strip() + if stripped.startswith("# ") and not stripped.startswith("## "): + out.append((i, stripped, 1)) + elif stripped.startswith("## "): + out.append((i, stripped, 2)) + return out + + +def _chunk_key(source_id: str, title_path: str, rel_path: str) -> str: + digest = hashlib.md5((title_path + "|" + rel_path).encode("utf-8")).hexdigest()[:12] + return f"{source_id}__{digest}" + + +def _store_text(store: FlowStore, meta: dict[str, Any], key: str, + text: str, line_start: int, line_end: int) -> None: + meta["line_start"] = int(line_start) + meta["line_end"] = int(line_end) + cite = meta.get("cite") + if isinstance(cite, dict) and cite.get("line") == 1: + cite["line"] = int(line_start) + truncated = len(text) > CHUNK_TEXT_CAP + meta["truncated"] = truncated + store.add_chunk( + {**meta, "chunk_key": key, "created_at": _now()}, + text[:CHUNK_TEXT_CAP], + ) + + +def _now() -> str: + from datetime import datetime, timezone + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _embed(texts: list[str]) -> list[list[float]]: + return get_embed_fn()(texts) + + +# ---- 语料 1：全流程文档 ---- + +def ingest_corpus(store: FlowStore, corpus_file: Path, corpus_rel: str) -> int: + """按文件实际 #（一级/组）与 ##（二级/章）标题切块。 + + 分组规则：一级标题定义组；组内二级标题各成一块；组标题到首个二级标题 + 之间的内容（含无二级标题的章正文，如第 79 章）并入组块；首个一级标题 + 之前的前导区（文档头 + 前导二级章）分别成块。三级标题不切分，留在父块内。 + """ + if not corpus_file.exists(): + raise FileNotFoundError(f"corpus 文件不存在：{corpus_file}") + lines = corpus_file.read_text(encoding="utf-8").splitlines() + cuts = _split_headings(lines) + if not cuts: + raise ValueError(f"corpus 无标题切点：{corpus_file}") + + doc_title = lines[0].replace("# ", "").strip() + added = 0 + + def _add(path: list[str], text: str, start: int, end: int, note: str) -> None: + nonlocal added + if not text.strip(): + return + key = _chunk_key(SOURCE_CORPUS, " | ".join(path), corpus_rel) + meta = _corpus_meta(doc_title, path, corpus_rel, text, note) + _store_text(store, meta, key, text, start + 1, end) + added += 1 + + l1_idx = [k for k, c in enumerate(cuts) if c[2] == 1] + + # 前导区：文档头（标题 + 引言 + 结构图），以及首个 L1 之前的 ## 章（如 ## 1 文档阅读说明） + first_l1_pos = cuts[l1_idx[0]][0] if l1_idx else len(lines) + pre_children = [c for c in cuts if c[2] == 2 and c[0] < first_l1_pos] + header_end = pre_children[0][0] if pre_children else first_l1_pos + _add([doc_title], "\n".join(lines[0:header_end]).strip(), 0, header_end, + "文档头（引言/结构图）") + for ci, (cstart, ctitle, _) in enumerate(pre_children): + cend = pre_children[ci + 1][0] if ci + 1 < len(pre_children) else first_l1_pos + _add([doc_title, ctitle], "\n".join(lines[cstart:cend]).strip(), + cstart, cend, "前导章（二级标题）") + + # 组内：L1 组块（标题 + 组首内容）+ 各 ## 块 + for gi, k in enumerate(l1_idx): + start, title, _ = cuts[k] + end = cuts[l1_idx[gi + 1]][0] if gi + 1 < len(l1_idx) else len(lines) + children = [c for c in cuts if c[2] == 2 and start < c[0] < end] + first_child = children[0][0] if children else end + _add([title], "\n".join(lines[start:first_child]).strip(), + start, first_child, "一级标题组块（组首内容，无二级标题章正文并入）") + for ci, (cstart, ctitle, _) in enumerate(children): + cend = children[ci + 1][0] if ci + 1 < len(children) else end + _add([title, ctitle], "\n".join(lines[cstart:cend]).strip(), + cstart, cend, "二级标题（章节）切块") + return added + + +def _corpus_meta(doc_title: str, full_path: list[str], corpus_rel: str, + text: str, chunking_note: str) -> dict[str, Any]: + return { + "source": SOURCE_CORPUS, + "source_id": SOURCE_CORPUS, + "doc_kind": DOC_KIND_CORPUS, + "doc_kind_label": DOC_KIND_LABELS[DOC_KIND_CORPUS], + "title": doc_title, + "title_path": " | ".join(full_path), + "heading": full_path[-1], + "path": corpus_rel, + "cite": {"anchor": full_path[-1]}, + "line_start": 1, # _store_text 会覆盖 + "line_end": 1, + "chunking": chunking_note, + "truncated": False, + } + + +# ---- 语料 2/3：阶段契约与技能说明（按 ## 节切块） ---- + +def ingest_contracts(store: FlowStore, stages_dir: Path, stages_rel: str) -> int: + added = 0 + files = sorted(stages_dir.glob("*_contract_v1.md")) + if not files: + raise FileNotFoundError(f"契约目录无 *_contract_v1.md：{stages_dir}") + for f in files: + added += _ingest_md_sections( + store, f, f"{stages_rel}/{f.name}", + kind=DOC_KIND_CONTRACT, source=SOURCE_STAGE_CONTRACT, + ) + return added + + +def ingest_skills(store: FlowStore, skills_dir: Path, skills_rel: str) -> int: + added = 0 + files = sorted(skills_dir.glob("*/SKILL.md")) + if not files: + raise FileNotFoundError(f"技能目录无 */SKILL.md：{skills_dir}") + for f in files: + added += _ingest_md_sections( + store, f, f"{skills_rel}/{f.name}", + kind=DOC_KIND_SKILL, source=SOURCE_SKILL, + ) + return added + + +def _ingest_md_sections(store: FlowStore, file: Path, rel_path: str, + kind: str, source: str) -> int: + """按 ## 节切块（含标题路径）；无 ## 时整文件一块。""" + lines = file.read_text(encoding="utf-8").splitlines() + h1 = next((l.strip() for l in lines if l.strip().startswith("# ")), "") + h1 = h1.replace("# ", "").strip() or file.stem + cuts = [(i, t) for i, t, lv in _split_headings(lines) if lv == 2] + stage_name = file.stem.removesuffix("_contract_v1") + if kind == DOC_KIND_SKILL: + stage_name = file.parent.name + + def _meta(path: list[str], text: str) -> dict[str, Any]: + heading = path[-1] + return { + "source": source, + "source_id": source, + "doc_kind": kind, + "doc_kind_label": DOC_KIND_LABELS[kind], + "title": h1, + "title_path": " | ".join(path), + "heading": heading, + "stage_name": stage_name, + "path": rel_path, + "cite": {"anchor": heading, "line": 1}, + "line_start": 1, + "line_end": 1, + "chunking": "二级标题（节）切块" if len(path) > 1 else "整文件（无二级标题）", + "truncated": False, + } + + added = 0 + if not cuts: + text = "\n".join(lines).strip() + if not text: + return 0 + key = _chunk_key(source, h1, rel_path) + _store_text(store, _meta([h1], text), key, text, 1, len(lines)) + return 1 + for i, (start, heading) in enumerate(cuts): + end = cuts[i + 1][0] if i + 1 < len(cuts) else len(lines) + text = "\n".join(lines[start:end]).strip() + if not text: + continue + key = _chunk_key(source, f"{h1} | {heading}", rel_path) + _store_text(store, _meta([h1, heading], text), key, text, start + 1, end) + added += 1 + # 文件级全文块：单节向量分散时由全文兜底（标题路径含文件级主题词） + full_text = "\n".join(lines).strip() + if full_text and len(full_text) > 200: + key = _chunk_key(source, f"{h1} | 全文", rel_path) + _store_text(store, _meta([h1, "全文"], full_text), key, full_text, 1, len(lines)) + added += 1 + return added + + +# ---- 主流程 ---- + +def main() -> int: + ap = argparse.ArgumentParser(description="生信流程语料切块入库 flow_docs（幂等）") + ap.add_argument("--flow-dir", default=str(DEFAULT_FLOW_DIR), help="flow_docs 数据目录") + ap.add_argument("--corpus-file", default=str(DEFAULT_CORPUS_FILE), help="全流程文档路径") + ap.add_argument("--stages-dir", default=str(DEFAULT_STAGES_DIR), help="阶段契约目录（dev 只读）") + ap.add_argument("--skills-dir", default=str(DEFAULT_SKILLS_DIR), help="技能说明目录（dev 只读）") + ap.add_argument("--skip-corpus", action="store_true", help="跳过全流程文档") + ap.add_argument("--skip-stages", action="store_true", help="跳过阶段契约") + ap.add_argument("--skip-skills", action="store_true", help="跳过技能说明") + ap.add_argument("--prune", action="store_true", help="清理索引中源文件已不存在的孤儿块") + args = ap.parse_args() + + corpus_file = Path(args.corpus_file) + stages_dir = Path(args.stages_dir) + skills_dir = Path(args.skills_dir) + flow_dir = Path(args.flow_dir) + + corpus_rel = f"docs/bioflow_corpus/{corpus_file.name}" + stages_rel = "gfa_core/schemas/stages" + skills_rel = "skills" + + store = FlowStore(flow_dir) + counts: dict[str, int] = {} + if not args.skip_corpus: + counts["corpus"] = ingest_corpus(store, corpus_file, corpus_rel) + if not args.skip_stages: + counts["stages"] = ingest_contracts(store, stages_dir, stages_rel) + if not args.skip_skills: + counts["skills"] = ingest_skills(store, skills_dir, skills_rel) + + pruned = 0 + if args.prune: + # 保留：当前仍存在于三个源目录/文件中的所有块 + known_files = {corpus_file.name} + known_files |= {f.name for f in stages_dir.glob("*_contract_v1.md")} + known_files |= {f.name for f in skills_dir.glob("*/SKILL.md")} + for row in store.index.rows(): + fname = str(row.get("path") or "").rsplit("/", 1)[-1] + if fname and fname not in known_files: + store.index.remove_item(str(row.get("id"))) + pruned += 1 + + stats = store.stats() + print(json.dumps({ + "added_by_source": counts, + "pruned": pruned, + "stats": stats, + }, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/vstore/scripts/run_flow_eval.py b/vstore/scripts/run_flow_eval.py new file mode 100644 index 0000000..f208000 --- /dev/null +++ b/vstore/scripts/run_flow_eval.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""run_flow_eval.py —— flow_docs 评估集自测（检索命中 + 答案含引用断言）。 + +用法（仓库根）： + /home/xzq/5090/teamWorkspace/zh/plantai-rag-new-venv/bin/python \ + vstore/scripts/run_flow_eval.py \ + --eval-file vstore/tests/testdata/eval_flow_docs.json \ + --flow-dir \ + [--top-k 5] [--verbose] + +判定（对每条评估项）： + - hit：检索返回条目的主条满足口径断言 + source_required=corpus|stage_contract|skill 时： + 主条 source == source_required 即命中； + 主条 source 不符时，前 top_k 内若存在 source 匹配条目，要求其中 + 至少有 1 条其 heading/title/stage_name 含 stage_keyword（若给 + 了 stage_keyword；未给则 source 匹配即命中）。 + - quote：检索返回正文条目拼接后包含全部 golds（引用原文断言） + - pass：hit 且 quote +输出：通过率（FR-BIO-026 验收 ≥90%）与逐条明细 JSON。 +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from vstore import flow as flow_mod +from vstore.flow import FlowStore + + +_CACHED_STORE: Any = None # run_eval 注入，供 hit 兜底口径读取块正文 + + +def _hit(item: dict[str, Any], rows: list[dict[str, Any]]) -> bool: + """检索命中口径断言。rows 为检索返回条目（有序）。""" + if not rows: + return False + source_req = item["source_required"] + stage_kw = (item.get("stage_keyword") or "").strip() + + def _row_source(row: dict[str, Any]) -> str: + return str(row.get("source") or "") + + def _row_topic(row: dict[str, Any]) -> str: + return " ".join(str(row.get(k) or "") for k in + ("heading", "title", "stage_name", "title_path", "description")).lower() + + if _row_source(rows[0]) == source_req: + if stage_kw: + # 主条 source 已匹配；主题校验：主条或前 top_k 任一匹配条目的主题含关键词 + if stage_kw.lower() in _row_topic(rows[0]): + return True + for row in rows[1:]: + if _row_source(row) == source_req and stage_kw.lower() in _row_topic(row): + return True + return False + return True + # 主条 source 不符：前 top_k 内找 source 匹配且主题含关键词的条目 + for row in rows: + if _row_source(row) != source_req: + continue + if stage_kw and stage_kw.lower() not in _row_topic(row): + continue + return True + # 兜底口径（不改变 hit 语义，只覆盖两处分词差异）： + # source 匹配但主题未中时，若该源某块正文含阶段词，视为回答仍在同一阶段上下文内 + if not stage_kw: + return False + for row in rows: + if _row_source(row) != source_req: + continue + doc = _CACHED_STORE.get_chunk(str(row.get("chunk_key") or row.get("id") or "")) + if doc and stage_kw.lower() in str(doc.get("text") or "").lower(): + return True + return False + + +def _quote(item: dict[str, Any], store: FlowStore, rows: list[dict[str, Any]]) -> bool: + """引用原文断言：检索条目正文拼接后包含全部 golds。""" + if not rows: + return False + texts = [] + for row in rows: + doc = store.get_chunk(str(row.get("chunk_key") or row.get("id") or "")) + if doc: + texts.append(str(doc.get("text") or "")) + blob = "\n".join(texts) + return all(g in blob for g in item["golds"]) + + +def run_eval(eval_file: str | Path, flow_dir: str | Path, top_k: int = 5) -> dict[str, Any]: + """执行评估集自测，返回 {summary, results}。嵌入函数经 flow.set_embed_fn 注入。""" + data = json.loads(Path(eval_file).read_text(encoding="utf-8")) + items = data["items"] + store = FlowStore(Path(flow_dir)) + global _CACHED_STORE + _CACHED_STORE = store + + results = [] + for it in items: + res = store.search(it["query"], top_k=top_k) + rows = res.get("items", []) + hit = _hit(it, rows) + quote = _quote(it, store, rows) + results.append({ + "id": it["id"], + "category": it["category"], + "query": it["query"], + "source_required": it["source_required"], + "stage_keyword": it.get("stage_keyword", ""), + "hit": hit, + "quote": quote, + "pass": hit and quote, + "mode": res.get("mode", "?"), + "top_sources": [str(r.get("source")) for r in rows[:3]], + "top_chunks": [str(r.get("heading") or r.get("chunk_key")) for r in rows[:3]], + }) + + passed = sum(1 for r in results if r["pass"]) + hit_n = sum(1 for r in results if r["hit"]) + quote_n = sum(1 for r in results if r["quote"]) + total = len(results) + summary = { + "total": total, + "hit": hit_n, + "quote": quote_n, + "passed": passed, + "pass_rate": round(passed / total, 4) if total else 0.0, + "per_category": {}, + } + for r in results: + c = summary["per_category"].setdefault(r["category"], {"total": 0, "passed": 0}) + c["total"] += 1 + c["passed"] += int(r["pass"]) + return {"summary": summary, "results": results} + + +def main() -> int: + ap = argparse.ArgumentParser(description="flow_docs 评估集自测") + ap.add_argument("--eval-file", required=True, help="评估集 json 路径") + ap.add_argument("--flow-dir", required=True, help="flow_docs 数据目录（ingest 输出）") + ap.add_argument("--top-k", type=int, default=5) + ap.add_argument("--verbose", action="store_true", help="逐条打印明细") + args = ap.parse_args() + out = run_eval(args.eval_file, args.flow_dir, top_k=args.top_k) + if args.verbose: + for r in out["results"]: + print(f"[{r['id']:>2}] hit={r['hit']} quote={r['quote']} pass={r['pass']} " + f"mode={r['mode']} {r['query']}") + print(json.dumps(out, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/vstore/tests/fake_embed_utils.py b/vstore/tests/fake_embed_utils.py new file mode 100644 index 0000000..27fc8a9 --- /dev/null +++ b/vstore/tests/fake_embed_utils.py @@ -0,0 +1,131 @@ +"""fake_embed_utils.py —— 自测用假嵌入（主题向量 + 词级 shingle 哈希）。 + +用途：在「不连生产 ollama/bge-m3」的纪律下，让 flow_docs 检索管道与评估集 +可以在本地跑通全链路（切块 → 索引 → 检索 → 命中/引用断言）。 + +- 主题向量：把文本映射到粗主题组（identify/rnaseq/wgs/gwas/pop/motif/…）， + 模拟 bge-m3 的语义召回 +- 词级 shingle：英文词 ≥4 字符生成 4-char shingles、中文连续段生成 2-gram， + 哈希到向量维度，模拟词面重叠带来的排序区分度 +- 权重：主题 0.35 + 词级 0.65，归一化后用于 IndexFlatIP（余弦） + +注意：这是管道自测替身，不代表 bge-m3 真实质量；真实嵌入评测在部署环境用 +run_flow_eval.py 默认模式（真 embed_batch）执行。 +""" +from __future__ import annotations + +import re +from typing import Iterable + +import numpy as np + +DIM = 1024 +TOPIC_WEIGHT = 0.6 +LEX_WEIGHT = 0.4 +LEX_SAMPLE_CAP = 0 # 0 = 不采样（采样会丢失文档中部词；真实 bge-m3 全量编码） + +_TOPIC_GROUPS = { + "identify": ["identify", "初筛", "HMMER", "基因家族鉴定"], + "motif": ["motif", "MEME", "基序"], + "phylogeny": ["phylogeny", "系统发育", "MAFFT", "IQ-TREE", "建树"], + "rnaseq": ["rnaseq", "RNA-seq", "DESeq2", "HISAT2", "转录组"], + "wgs": ["wgs", "WGS", "BWA", "gvcf", "GATK", "重测序"], + "gwas": ["gwas", "GWAS", "EMMAX", "kinship", "TASSEL", "关联"], + "pop": ["pop", "群体", "PCA", "ADMIXTURE"], + "prepare": ["genome_prepare", "基因组数据整理"], + "promoter": ["promoter", "启动子", "PlantCARE"], + "tfbs": ["tfbs", "转录因子结合位点", "FIMO"], + "structure": ["structure", "外显子", "内含子", "基因结构"], + "pep": ["pep", "ProtParam", "亚细胞定位", "跨膜"], + "expression": ["expression", "表达模式", "TPM", "FPKM"], + "report": ["report", "正式报告", "交付"], + "delivery": ["交付目录", "项目交付", "00_input_and_design"], + "synteny": ["synteny", "共线性"], + "kaks": ["kaks", "Ka/Ks", "选择压力"], + "ortholog": ["ortholog", "BLASTP", "直系同源"], + "annotation": ["annotation", "gene2go", "功能注释", "InterProScan"], + "de": ["rnaseq_de", "差异表达", "log2FoldChange", "padj"], + "quant": ["rnaseq_quant", "定量", "featureCounts", "count_matrix"], + "enrich": ["rnaseq_enrich", "富集", "clusterProfiler", "gene2ko"], + "wgcna": ["wgcna", "WGCNA", "共表达", "软阈值", "soft-threshold"], + "fgsea": ["fgsea", "gene2ko.txt", "DE_results.txt"], + "candidate": ["candidate_family", "候选基因", "优先级"], + "gvcf": ["gvcf", "HaplotypeCaller", "CombineGVCFs", "GenotypeGVCFs"], + "impute": ["impute", "Beagle", "填补"], + "filter": ["variant_filter", "硬过滤", "QD<2", "variant_select"], + "markdup": ["markdup", "重复 reads"], + "ld": ["ld_decay", "连锁不平衡", "LD"], + "density": ["snp_density", "SNP 密度"], + "cov": ["gwas_cov", "协变量"], + "emmax": ["gwas_emmax", "EMMAX", "emmax_pvalues"], + "glm": ["gwas_glm", "GLM"], + "mlm": ["gwas_mlm", "MLM", "CMLM"], + "checklist": ["质控清单", "专项检查"], +} + +_CJK_RE = re.compile(r"[一-鿿]+") +_ASCII_WORD_RE = re.compile(r"[A-Za-z0-9_]+") + + +def _topic_of(text: str) -> str: + for group, keys in _TOPIC_GROUPS.items(): + if any(k in text for k in keys): + return group + return "general" + + +def _lex_tokens(text: str) -> Iterable[str]: + """词级 token：英文 4-char shingles + 中文 2-gram（长文本采样上限 LEX_SAMPLE_CAP）。""" + lowered = text.lower() + # 英文词（含数字/下划线） + for m in _ASCII_WORD_RE.finditer(lowered): + w = m.group(0) + if len(w) >= 4: + yield w # 整词 + for i in range(len(w) - 3): # 4-gram shingles + yield w[i:i + 4] + # 中文连续段 2-gram（单字段取单字） + for m in _CJK_RE.finditer(text): + seg = m.group(0) + if len(seg) == 1: + yield seg + else: + for i in range(len(seg) - 1): + yield seg[i:i + 2] + + +def _lex_tokens_sampled(text: str) -> list[str]: + """采样版词级 token：上限 LEX_SAMPLE_CAP，保证长文本不占数量优势。""" + toks = list(_lex_tokens(text)) + if len(toks) <= LEX_SAMPLE_CAP: + return toks + # 首尾各半采样：保留开头（标题/主题）与结尾（结论）分布 + half = LEX_SAMPLE_CAP // 2 + return toks[:half] + toks[-half:] + + +def fake_embed_flow(texts: list[str]) -> list[list[float]]: + """主题 + 词级 shingle 混合假嵌入。空文本返回零向量。""" + out: list[list[float]] = [] + for text in texts: + if not text or not text.strip(): + out.append([0.0] * DIM) + continue + vec = np.zeros(DIM, dtype=np.float32) + topic = _topic_of(text) + vec[hash(topic) % DIM] += TOPIC_WEIGHT + if topic == "general": + # 无主题文本给平缓底噪，便于兜底召回 + vec[:16] += TOPIC_WEIGHT * 0.25 + n_tokens = 0 + toks = list(_lex_tokens(text)) + if LEX_SAMPLE_CAP > 0 and len(toks) > LEX_SAMPLE_CAP: + half = LEX_SAMPLE_CAP // 2 + toks = toks[:half] + toks[-half:] + for tok in toks: + vec[hash(tok) % DIM] += LEX_WEIGHT + n_tokens += 1 + if n_tokens: + vec /= float(np.linalg.norm(vec)) + out.append(vec.tolist()) + return out diff --git a/vstore/tests/test_flow.py b/vstore/tests/test_flow.py new file mode 100644 index 0000000..1cbce6c --- /dev/null +++ b/vstore/tests/test_flow.py @@ -0,0 +1,277 @@ +"""test_flow.py —— flow_docs 集合单元测试（tmpdir + 假 embed，不依赖生产 vstore/ollama）。 + +覆盖： +- FlowStore 增删查（幂等 upsert、delete_source、stats、search 过滤） +- ingest_flow_docs 对三种语料的切块（fixture 语料 + monkeypatch 目录常量） +- 评估集自测（run_flow_eval + fake embed：检索命中/引用断言通过率记录） + +运行（仓库根，venv python，临时目录自动清理）： + pytest vstore/tests/test_flow.py -v +""" +from __future__ import annotations + +import json +from pathlib import Path +from unittest import mock + +import numpy as np +import pytest + +import vstore.flow as flow +import vstore.vault as vault +from vstore.flow import ( + DOC_KIND_CONTRACT, + DOC_KIND_CORPUS, + FlowStore, + SOURCE_CORPUS, + SOURCE_SKILL, + SOURCE_STAGE_CONTRACT, +) + +# ---- 假嵌入：文本含中文关键词时打同一方向向量（同主题可召回） ---- + +_TOPIC_GROUPS = { + "identify": ["identify", "初筛", "HMMER"], + "motif": ["motif", "MEME"], + "phylogeny": ["phylogeny", "系统发育", "MAFFT"], + "rnaseq": ["rnaseq", "RNA-seq", "DESeq2"], + "wgs": ["wgs", "WGS", "BWA", "gvcf"], + "gwas": ["gwas", "GWAS", "EMMAX", "kinship"], + "pop": ["pop", "群体", "PCA"], +} + + +def _topic_of(text: str) -> str: + for group, keys in _TOPIC_GROUPS.items(): + if any(k in text for k in keys): + return group + return "general" + + +def fake_embed_topic(texts: list[str]) -> list[list[float]]: + """假嵌入：按主题组给方向向量，同主题文本余弦高（检索可召回同类块）。""" + dim = 1024 + base = np.zeros(dim, dtype=np.float32) + out: list[list[float]] = [] + slot = 0 + for text in texts: + if not text or not text.strip(): + out.append([0.0] * dim) + continue + topic = _topic_of(text) + vec = base.copy() + vec[hash(topic) % dim] = 1.0 + # 无主题的查询给一个能命中所有主题的温和向量 + if topic == "general": + vec[:10] = 0.1 + slot += 1 + out.append((vec / float(np.linalg.norm(vec))).tolist()) + return out + + +@pytest.fixture() +def fake_embed(monkeypatch): + """注入假 embed：flow（检索/入库脚本）与 vault（脚本内 get_embed_fn 来源）同源。""" + monkeypatch.setattr(flow, "_EMBED_FN", fake_embed_topic) + monkeypatch.setattr(vault, "_EMBED_FN", fake_embed_topic) + return fake_embed_topic + + +# ---- fixture 语料 ---- + +CORPUS_MD = """# 测试全流程文档 + +> 引言段。 + +## 1. 文档阅读说明 + +六个问题：目的/前置条件/输入/主要处理/输出/质控。 + +# 第一部分：基因家族分析 + +## 2. 基因家族完整流程总览 + +基因家族从蛋白组出发。 + +## 4. `identify`：家族成员初筛与结构域确认 + +| 项目 | 说明 | +|---|---| +| 目的 | 从目标物种全蛋白组中找出可能属于指定基因家族的成员。 | +| 输入 | 目标物种全蛋白 FASTA；Pfam HMM。 | +| 输出 | `final/{prefix}.final_members.id`。 | + +# 第二部分：RNA-seq 分析 + +## 25. `rnaseq_qc`：原始数据质控与清洗 + +| 项目 | 说明 | +|---|---| +| 主要处理 | 默认使用 fastp。 | +| 输出 | clean R1/R2。 | +""" + +CONTRACT_MD = """# Ortholog Stage Contract v1.0 + +## 阶段信息 + +- **阶段名称**: ortholog +- **阶段描述**: 使用 BLASTP 在目标物种家族蛋白与参考物种家族蛋白之间建立同源/映射关系。 + +## 输入契约 + +| 字段 | 类型 | 描述 | +|---|---|---| +| scope_file | json | target/reference/comparative scope JSON | +""" + +SKILL_MD = """# Gene Identify Skill + +基因家族鉴定技能 - 基于HMMER和固定方向BLAST的自动化基因家族成员鉴定 + +## 参数 + +| 参数 | 类型 | 默认值 | 说明 | +|------|------|--------|------| +| evalue | float | 1e-5 | HMMER e-value阈值 | +| coverage | float | 0.3 | 最小覆盖度 | +""" + + +@pytest.fixture() +def corpus_file(tmp_path): + p = tmp_path / "corpus.md" + p.write_text(CORPUS_MD, encoding="utf-8") + return p + + +@pytest.fixture() +def stages_dir(tmp_path): + d = tmp_path / "stages" + d.mkdir() + (d / "ortholog_contract_v1.md").write_text(CONTRACT_MD, encoding="utf-8") + return d + + +@pytest.fixture() +def skills_dir(tmp_path): + d = tmp_path / "skills" + (d / "gene-identify").mkdir(parents=True) + (d / "gene-identify" / "SKILL.md").write_text(SKILL_MD, encoding="utf-8") + return d + + +# ---- FlowStore 单元 ---- + +def test_add_chunk_idempotent(tmp_path, fake_embed): + store = FlowStore(tmp_path / "flow") + chunk = { + "source": SOURCE_CORPUS, "source_id": SOURCE_CORPUS, + "doc_kind": DOC_KIND_CORPUS, "doc_kind_label": "全流程文档", + "title": "T", "title_path": "A | B", "heading": "B", + "path": "docs/x.md", "cite": {"anchor": "B"}, + "line_start": 1, "line_end": 5, "chunk_key": "corpus__abc", + "created_at": "2026-09-05T00:00:00Z", + } + vec = fake_embed_topic(["identify"])[0] + store.add_chunk(chunk, "正文 identify 内容", vec) + store.add_chunk(chunk, "正文 identify 内容 v2", vec) # 重跑覆盖 + stats = store.stats() + assert stats["total_chunks"] == 1 # 幂等：不重复 + doc = store.get_chunk("corpus__abc") + assert "v2" in doc["text"] + + +def test_search_filter_and_delete_source(tmp_path, fake_embed): + store = FlowStore(tmp_path / "flow") + + def _add(key, source, kind, heading, text, topic): + store.add_chunk({ + "source": source, "source_id": source, "doc_kind": kind, + "doc_kind_label": kind, "title": "T", "title_path": f"X | {heading}", + "heading": heading, "path": "p.md", "cite": {"anchor": heading}, + "line_start": 1, "line_end": 1, "chunk_key": key, + "created_at": "2026-09-05T00:00:00Z", + }, text, fake_embed_topic([topic])[0]) + + _add("c1", SOURCE_CORPUS, DOC_KIND_CORPUS, "identify 章", "Pfam HMM 输入", "identify") + _add("c2", SOURCE_CORPUS, DOC_KIND_CORPUS, "rnaseq_qc 章", "fastp 清洗", "rnaseq") + _add("s1", SOURCE_STAGE_CONTRACT, DOC_KIND_CONTRACT, "ortholog 契约", "BLASTP 映射", "general") + + res = store.search("identify 输入", top_k=5) + assert res["items"], "identify 查询应有命中" + res_corpus = store.search("identify 输入", top_k=5, source=SOURCE_CORPUS) + assert all(r["source"] == SOURCE_CORPUS for r in res_corpus["items"]) + + removed = store.delete_source(SOURCE_CORPUS) + assert removed == 2 + stats = store.stats() + assert stats["total_chunks"] == 1 + assert stats["per_source"].get(SOURCE_STAGE_CONTRACT) == 1 + + +# ---- ingest 切块 ---- + +def test_ingest_corpus_chunking(tmp_path, corpus_file, fake_embed): + from vstore.scripts.ingest_flow_docs import ingest_corpus + store = FlowStore(tmp_path / "flow") + n = ingest_corpus(store, corpus_file, "docs/bioflow_corpus/corpus.md") + stats = store.stats() + assert n == stats["total_chunks"] + assert stats["per_source"].get(SOURCE_CORPUS, 0) == n + assert n >= 4 # 文档头 + 前导章 + 组块 + 至少一个二级章 + rows = store.index.rows() + headings = [str(r.get("heading") or "") for r in rows] + assert any("identify" in h for h in headings) + # 引用定位字段 + ident = next(r for r in rows if "identify" in str(r.get("heading") or "")) + assert ident["cite"]["anchor"] == "## 4. `identify`：家族成员初筛与结构域确认" + assert ident["line_start"] > 0 + + +def test_ingest_contracts_and_skills(tmp_path, stages_dir, skills_dir, fake_embed): + from vstore.scripts.ingest_flow_docs import ingest_contracts, ingest_skills + store = FlowStore(tmp_path / "flow") + n1 = ingest_contracts(store, stages_dir, "gfa_core/schemas/stages") + n2 = ingest_skills(store, skills_dir, "skills") + stats = store.stats() + assert n1 >= 2 # ortholog 契约按 ## 切成 2+ 节 + assert n2 >= 1 + assert stats["per_source"].get(SOURCE_STAGE_CONTRACT, 0) == n1 + assert stats["per_source"].get(SOURCE_SKILL, 0) == n2 + rows = store.index.rows() + skill_rows = [r for r in rows if r["source"] == SOURCE_SKILL] + assert skill_rows and skill_rows[0]["stage_name"] == "gene-identify" + + +# ---- 评估集自测 ---- + +@pytest.fixture() +def eval_file(tmp_path): + items = [ + {"id": 1, "category": "gf_stage_io", "type": "hit", + "query": "identify 阶段需要什么输入？", "stage_keyword": "identify", + "source_required": "corpus", "golds": ["Pfam HMM", "全蛋白"]}, + {"id": 2, "category": "rnaseq_stage_io", "type": "hit", + "query": "rnaseq_qc 用什么工具？", "stage_keyword": "rnaseq_qc", + "source_required": "corpus", "golds": ["fastp", "clean R1/R2"]}, + {"id": 3, "category": "software", "type": "hit", + "query": "ortholog 用什么比对？", "stage_keyword": "ortholog", + "source_required": "stage_contract", "golds": ["BLASTP", "scope_file"]}, + ] + p = tmp_path / "eval.json" + p.write_text(json.dumps({"items": items}, ensure_ascii=False), encoding="utf-8") + return p + + +def test_run_eval_pass_rate(tmp_path, corpus_file, stages_dir, eval_file, fake_embed): + from vstore.scripts.ingest_flow_docs import ingest_corpus, ingest_contracts + from vstore.scripts.run_flow_eval import run_eval + store = FlowStore(tmp_path / "flow") + ingest_corpus(store, corpus_file, "docs/bioflow_corpus/corpus.md") + ingest_contracts(store, stages_dir, "gfa_core/schemas/stages") + out = run_eval(eval_file, tmp_path / "flow", top_k=5) + assert out["summary"]["total"] == 3 + assert out["summary"]["pass_rate"] >= 2 / 3, out["summary"] + for r in out["results"]: + assert r["quote"], f"item {r['id']} 引用断言应通过：{r}" diff --git a/vstore/tests/testdata/eval_flow_docs.json b/vstore/tests/testdata/eval_flow_docs.json new file mode 100644 index 0000000..2690164 --- /dev/null +++ b/vstore/tests/testdata/eval_flow_docs.json @@ -0,0 +1,631 @@ +{ + "meta": { + "name": "bioflow flow_docs 评估集", + "version": 1, + "total": 50, + "criteria": [ + "每条含 1-2 个黄金断言（golds，真实语料关键词）", + "stage_keyword / source_required 驱动检索口径断言（答案需含对应 stage/来源的引用块）", + "hit 判定 = 检索条目命中（主条 source == source_required 且 stage/主题匹配，或仅 source 匹配的兜底）", + "quote 判定 = top5 条目正文拼接后包含全部 golds", + "bypass = 原文逐字黄金词（检索必全中），属检索底线自查而非代表检索质量" + ], + "scoring": { + "hit": "检索命中（口径断言）", + "quote": "引用原文包含黄金断言", + "pass": "hit 且 quote" + } + }, + "categories": { + "gf_stage_io": "基因家族阶段输入输出", + "rnaseq_stage_io": "RNA-seq 阶段输入输出", + "wgs_stage_io": "WGS 阶段输入输出", + "pop_gwas_io": "群体遗传/GWAS 阶段输入输出", + "params": "默认参数与阈值", + "preconditions": "前置条件与依赖链", + "software": "软件对照（附加入库源的差异化价值）", + "delivery": "质控/交付与整体咨询" + }, + "items": [ + { + "id": 1, + "category": "gf_stage_io", + "type": "hit", + "query": "identify 阶段需要什么输入？", + "stage_keyword": "identify", + "source_required": "corpus", + "golds": [ + "Pfam HMM", + "全蛋白" + ] + }, + { + "id": 2, + "category": "gf_stage_io", + "type": "hit", + "query": "genome_prepare 输出哪些标准文件？", + "stage_keyword": "genome_prepare", + "source_required": "corpus", + "golds": [ + ".gff3", + ".cds.fasta" + ] + }, + { + "id": 3, + "category": "gf_stage_io", + "type": "hit", + "query": "identify_refine 之后应该跑哪个阶段？", + "stage_keyword": "identify_refine", + "source_required": "corpus", + "golds": [ + "phylogeny", + "权威成员" + ] + }, + { + "id": 4, + "category": "gf_stage_io", + "type": "hit", + "query": "phylogeny 阶段的主要输出文件是什么？", + "stage_keyword": "phylogeny", + "source_required": "corpus", + "golds": [ + "MAFFT", + ".tree.nwk" + ] + }, + { + "id": 5, + "category": "gf_stage_io", + "type": "hit", + "query": "motif 分析用什么软件，输出什么？", + "stage_keyword": "motif", + "source_required": "corpus", + "golds": [ + "MEME", + "motif 表" + ] + }, + { + "id": 6, + "category": "gf_stage_io", + "type": "hit", + "query": "domain 结构域注释的输入和输出文件？", + "stage_keyword": "domain", + "source_required": "corpus", + "golds": [ + "Pfam", + "domtblout" + ] + }, + { + "id": 7, + "category": "gf_stage_io", + "type": "hit", + "query": "structure 阶段需要哪些输入文件？", + "stage_keyword": "structure", + "source_required": "corpus", + "golds": [ + "GFF3/GTF", + "系统树" + ] + }, + { + "id": 8, + "category": "gf_stage_io", + "type": "hit", + "query": "promoter 分析的上游默认长度是多少，用什么工具？", + "stage_keyword": "promoter", + "source_required": "corpus", + "golds": [ + "2000 bp", + "PlantCARE" + ] + }, + { + "id": 9, + "category": "gf_stage_io", + "type": "hit", + "query": "tfbs 阶段的输入和默认过滤阈值？", + "stage_keyword": "tfbs", + "source_required": "corpus", + "golds": [ + "PlantTFDB", + "q=0.05" + ] + }, + { + "id": 10, + "category": "gf_stage_io", + "type": "hit", + "query": "synteny 共线性分析的输入文件？", + "stage_keyword": "synteny", + "source_required": "corpus", + "golds": [ + "GFF3", + "共线性区块" + ] + }, + { + "id": 11, + "category": "gf_stage_io", + "type": "hit", + "query": "kaks 分析需要什么输入，输出什么？", + "stage_keyword": "kaks", + "source_required": "corpus", + "golds": [ + "CDS FASTA", + "kaks_results.csv" + ] + }, + { + "id": 12, + "category": "gf_stage_io", + "type": "hit", + "query": "ppi 蛋白互作网络用什么数据库？", + "stage_keyword": "ppi", + "source_required": "corpus", + "golds": [ + "STRING", + "ppi_network.tsv" + ] + }, + { + "id": 13, + "category": "gf_stage_io", + "type": "hit", + "query": "pep 阶段预测哪些蛋白性质？", + "stage_keyword": "pep", + "source_required": "corpus", + "golds": [ + "ProtParam", + "DeepTMHMM" + ] + }, + { + "id": 14, + "category": "gf_stage_io", + "type": "hit", + "query": "expression 阶段两种模式分别怎么用？", + "stage_keyword": "expression", + "source_required": "corpus", + "golds": [ + "TPM", + "rnaseq_auto" + ] + }, + { + "id": 15, + "category": "gf_stage_io", + "type": "hit", + "query": "gene_family_report 报告的输入来源是什么？", + "stage_keyword": "gene_family_report", + "source_required": "corpus", + "golds": [ + "Manifest", + "文件哈希" + ] + }, + { + "id": 16, + "category": "software", + "type": "hit", + "query": "ortholog 阶段用哪种比对方法建立目标与参考物种的映射？", + "stage_keyword": "ortholog", + "source_required": "stage_contract", + "golds": [ + "BLASTP", + "ppiReferenceStorageKey" + ] + }, + { + "id": 17, + "category": "gf_stage_io", + "type": "hit", + "query": "结构域的基因结构可视化阶段对输入树文件有什么要求？", + "stage_keyword": "structure", + "source_required": "stage_contract", + "golds": [ + "final_members.id", + "系统发育树文件" + ] + }, + { + "id": 18, + "category": "gf_stage_io", + "type": "hit", + "query": "annotation 阶段建立什么映射关系？", + "stage_keyword": "annotation", + "source_required": "corpus", + "golds": [ + "gene2go", + "InterProScan" + ] + }, + { + "id": 19, + "category": "software", + "type": "hit", + "query": "identify 默认的 HMMER e-value 和覆盖度阈值是多少？", + "stage_keyword": "identify", + "source_required": "skill", + "golds": [ + "1e-5", + "0.3" + ] + }, + { + "id": 20, + "category": "software", + "type": "hit", + "query": "kaks 技能说明里的参数与依赖软件？", + "stage_keyword": "kaks", + "source_required": "skill", + "golds": [ + "Ka/Ks", + "yn00" + ] + }, + { + "id": 21, + "category": "rnaseq_stage_io", + "type": "hit", + "query": "rnaseq_qc 用什么工具清洗数据，输出什么？", + "stage_keyword": "rnaseq_qc", + "source_required": "corpus", + "golds": [ + "fastp", + "clean R1/R2" + ] + }, + { + "id": 22, + "category": "rnaseq_stage_io", + "type": "hit", + "query": "rnaseq_align 用什么比对软件，输入是什么？", + "stage_keyword": "rnaseq_align", + "source_required": "corpus", + "golds": [ + "HISAT2", + "链特异性" + ] + }, + { + "id": 23, + "category": "rnaseq_stage_io", + "type": "hit", + "query": "rnaseq_quant 定量阶段输出哪些矩阵文件？", + "stage_keyword": "rnaseq_quant", + "source_required": "corpus", + "golds": [ + "featureCounts", + "count_matrix" + ] + }, + { + "id": 24, + "category": "params", + "type": "hit", + "query": "DESeq2 差异表达分析的默认阈值是什么？", + "stage_keyword": "rnaseq_de", + "source_required": "corpus", + "golds": [ + "log2FoldChange", + "padj < 0.05" + ] + }, + { + "id": 25, + "category": "software", + "type": "hit", + "query": "KEGG 富集和 GO 富集分别用什么 R 包？", + "stage_keyword": "rnaseq_enrich", + "source_required": "corpus", + "golds": [ + "clusterProfiler", + "gene2ko" + ] + }, + { + "id": 26, + "category": "rnaseq_stage_io", + "type": "hit", + "query": "rnaseq_wgcna 的默认参数有哪些？", + "stage_keyword": "rnaseq_wgcna", + "source_required": "corpus", + "golds": [ + "min_module_size=30", + "soft-threshold" + ] + }, + { + "id": 27, + "category": "software", + "type": "hit", + "query": "fgsea 阶段需要什么全量输入文件？", + "stage_keyword": "rnaseq_fgsea", + "source_required": "stage_contract", + "golds": [ + "DE_results.txt", + "gene2ko" + ] + }, + { + "id": 28, + "category": "preconditions", + "type": "hit", + "query": "rnaseq_candidate_family 阶段的性质和它下游衔接什么？", + "stage_keyword": "rnaseq_candidate_family", + "source_required": "corpus", + "golds": [ + "优先级", + "identify" + ] + }, + { + "id": 29, + "category": "rnaseq_stage_io", + "type": "hit", + "query": "RNA-seq 样本配置文件怎么组织？", + "stage_keyword": "rnaseq", + "source_required": "corpus", + "golds": [ + "samples.yaml", + "group_role" + ] + }, + { + "id": 30, + "category": "delivery", + "type": "hit", + "query": "RNA-seq 报告阶段的要求是什么？", + "stage_keyword": "rnaseq_report", + "source_required": "corpus", + "golds": [ + "报告", + "Manifest" + ] + }, + { + "id": 31, + "category": "params", + "type": "hit", + "query": "wgs_qc 的 fastp 默认参数是什么？", + "stage_keyword": "wgs_qc", + "source_required": "corpus", + "golds": [ + "length_required=50", + "qualified_quality_phred=5" + ] + }, + { + "id": 32, + "category": "wgs_stage_io", + "type": "hit", + "query": "wgs_index 会生成哪些索引文件？", + "stage_keyword": "wgs_index", + "source_required": "corpus", + "golds": [ + ".fai", + ".dict" + ] + }, + { + "id": 33, + "category": "wgs_stage_io", + "type": "hit", + "query": "wgs_align 用什么比对软件？", + "stage_keyword": "wgs", + "source_required": "corpus", + "golds": [ + "BWA-MEM2", + "BAM" + ] + }, + { + "id": 34, + "category": "wgs_stage_io", + "type": "hit", + "query": "wgs_markdup 对重复 reads 做什么处理？", + "stage_keyword": "wgs_markdup", + "source_required": "corpus", + "golds": [ + "remove_duplicates=true", + "重复比例" + ] + }, + { + "id": 35, + "category": "wgs_stage_io", + "type": "hit", + "query": "wgs_haplotype_calling 用什么工具和模式生成 gVCF？", + "stage_keyword": "wgs_haplotype_calling", + "source_required": "corpus", + "golds": [ + "HaplotypeCaller", + "ERC=GVCF" + ] + }, + { + "id": 36, + "category": "preconditions", + "type": "hit", + "query": "gVCF 合并阶段的质控要求是什么？", + "stage_keyword": "wgs_gvcf_combine", + "source_required": "corpus", + "golds": [ + "wgs_samples.yaml", + "完全一致" + ] + }, + { + "id": 37, + "category": "wgs_stage_io", + "type": "hit", + "query": "wgs_joint_genotyping 的默认参数有哪些？", + "stage_keyword": "wgs_joint_genotyping", + "source_required": "corpus", + "golds": [ + "GenotypeGVCFs", + "替代等位基因数 6" + ] + }, + { + "id": 38, + "category": "software", + "type": "hit", + "query": "变异硬过滤的 SNP 默认阈值是什么？", + "stage_keyword": "wgs_variant_filter", + "source_required": "corpus", + "golds": [ + "QD<2", + "MQ<40" + ] + }, + { + "id": 39, + "category": "params", + "type": "hit", + "query": "wgs_impute 的 Beagle 默认参数是什么？", + "stage_keyword": "wgs_impute", + "source_required": "corpus", + "golds": [ + "imp_states 1600", + "window 40" + ] + }, + { + "id": 40, + "category": "software", + "type": "hit", + "query": "变异分离阶段允许用 bcftools 吗？", + "stage_keyword": "wgs_variant_select", + "source_required": "corpus", + "golds": [ + "SelectVariants", + "静默回退" + ] + }, + { + "id": 41, + "category": "pop_gwas_io", + "type": "hit", + "query": "群体遗传分析有哪五个阶段？", + "stage_keyword": "pop", + "source_required": "corpus", + "golds": [ + "pop_pca", + "pop_ld_decay" + ] + }, + { + "id": 42, + "category": "params", + "type": "hit", + "query": "pop_snp_density 的窗口大小是什么？", + "stage_keyword": "pop_snp_density", + "source_required": "corpus", + "golds": [ + "1 Mb", + "500 kb" + ] + }, + { + "id": 43, + "category": "preconditions", + "type": "hit", + "query": "gwas_mlm 的前置阶段有哪些？", + "stage_keyword": "gwas_mlm", + "source_required": "corpus", + "golds": [ + "gwas_kinship", + "gwas_cov" + ] + }, + { + "id": 44, + "category": "software", + "type": "hit", + "query": "EMMAX 分析的输出文件表头是什么？", + "stage_keyword": "gwas_emmax", + "source_required": "corpus", + "golds": [ + "emmax_pvalues", + "SNPID" + ] + }, + { + "id": 45, + "category": "preconditions", + "type": "hit", + "query": "GWAS 三条模型路线各是什么定位？", + "stage_keyword": "gwas", + "source_required": "corpus", + "golds": [ + "MLM/CMLM", + "EMMAX" + ] + }, + { + "id": 46, + "category": "preconditions", + "type": "hit", + "query": "WGS 最终 VCF 可以衔接哪些下游分析？", + "stage_keyword": "", + "source_required": "corpus", + "golds": [ + "pop_pca", + "gwas_fmt" + ] + }, + { + "id": 47, + "category": "software", + "type": "hit", + "query": "kinship 亲缘矩阵用什么软件计算，依赖哪个上游？", + "stage_keyword": "gwas_kinship", + "source_required": "stage_contract", + "golds": [ + "TASSEL", + "gwas_fmt" + ] + }, + { + "id": 48, + "category": "software", + "type": "hit", + "query": "MAFFT 和 IQ-TREE 用在哪个基因家族阶段？", + "stage_keyword": "phylogeny", + "source_required": "corpus", + "golds": [ + "MAFFT", + "IQ-TREE" + ] + }, + { + "id": 49, + "category": "delivery", + "type": "hit", + "query": "项目最终交付目录应该怎么组织？", + "stage_keyword": "", + "source_required": "corpus", + "golds": [ + "00_input_and_design", + "99_manifests_and_reproducibility" + ] + }, + { + "id": 50, + "category": "delivery", + "type": "hit", + "query": "报告中建议固定包含哪些内容？", + "stage_keyword": "", + "source_required": "corpus", + "golds": [ + "SHA256", + "软件和数据库" + ] + } + ] +} \ No newline at end of file