From d54419f040480e793105687df0790d0c1d48c60c Mon Sep 17 00:00:00 2001 From: admin Date: Fri, 11 Sep 2026 17:17:25 +0800 Subject: [PATCH] =?UTF-8?q?fix(r4-28v4):=20chat=20=E5=80=99=E9=80=89?= =?UTF-8?q?=E8=85=90=E7=83=82=E6=A0=B9=E6=B2=BB=E2=80=94=E2=80=94Qdrant=20?= =?UTF-8?q?=E6=9D=A1=E7=9B=AE=20paper=5Fid=20=E6=B4=BE=E7=94=9F=20DOI/PMCI?= =?UTF-8?q?D=20=E5=9B=9E=E5=A1=AB=E5=BC=95=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 根因(TL 实测 + 只读探针确认): 175 归一化库 Neo4j ASSERTS 关系 13040841 条全无 evidence 属性 (_ONE_HOP Cypher 也不返回 evidence);而 175 无 paper metadata registry, Qdrant payload 无 pmid/pmcid/doi 键 -> chat 模式 public_citation 过滤 把有正文的 qdrant 条目(120 条)全滤掉,只留无 evidence 的 neo4j 图断言 (136 条)-> BGE 复刻 _document 模板输入 evidence 空 -> logit=-3.07, sigmoid=0.044 与审计 bge 0.020-0.047 完全吻合 -> 0.25 final 门全拒 sources=[]。176 正常是因为配了 registry 回填 pmid。 数字链:dedup 244 -> public_citation kept 136(全图断言)-> pre_rerank 160 -> cluster 136 代表 -> rerank 全垃圾分。 修复:item_from_payload 引用字段回填优先级 payload 显式 > registry > paper_id 派生(doi_from_paper_id/pmcid_from_paper_id + doi: 前缀形态)。 r4 v1/v2/v3 的阈值语义不变;search 免 rerank 路径不受影响。 Co-Authored-By: Claude Fable 5 --- .../rag/qa_v1_1_73/direct_flow_retriever.py | 26 ++++++++++++++-- .../qa/src/rag/qa_v1_1_73/test_score_gate.py | 30 +++++++++++++++++++ 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/agent/qa/src/rag/qa_v1_1_73/direct_flow_retriever.py b/agent/qa/src/rag/qa_v1_1_73/direct_flow_retriever.py index 8ae9ceb..a77f868 100644 --- a/agent/qa/src/rag/qa_v1_1_73/direct_flow_retriever.py +++ b/agent/qa/src/rag/qa_v1_1_73/direct_flow_retriever.py @@ -3,6 +3,7 @@ from __future__ import annotations from concurrent.futures import ThreadPoolExecutor import json import os +import re from pathlib import Path from typing import Any, Dict, Iterable, List, Tuple @@ -16,6 +17,7 @@ from src.rag.normalized_release import ( ) from .schemas import EvidenceItem, RetrievalHypothesis +from .soft_gate_policy import doi_from_paper_id, pmcid_from_paper_id ALLOWED_QDRANT_COLLECTIONS = { "plant_srt_direct_v1_bge_m3_20260702", @@ -36,6 +38,12 @@ def _norm(value: Any) -> str: return " ".join(text.split()) +# doi: 前缀形态 paper_id 的 DOI 全文校验（回改 v4，2026-09-11） +_DOI_LITERAL = re.compile( + r"10\.\d{4,9}/[A-Za-z0-9][A-Za-z0-9._;()/:+\-]*" +) + + def load_paper_metadata_registry(path: Path) -> Dict[str, Dict[str, Any]]: records: Dict[str, Dict[str, Any]] = {} if not path.is_file(): @@ -342,6 +350,20 @@ class DirectFlowRetriever: metadata = payload.get("metadata") if isinstance(payload.get("metadata"), dict) else {} paper_id = _text(payload.get("paper_id") or payload.get("best_paper_id")) registry_metadata = getattr(self, "paper_metadata", {}).get(paper_id, {}) + # 缺陷 #28 回改 v4（2026-09-11）：175 无 paper metadata registry， + # Qdrant payload 又无 pmid/pmcid/doi 键——chat 的 public_citation + # 过滤会把有正文的 qdrant 条目全滤掉，只剩无 evidence 的 neo4j + # 归一化图断言（bge 全 0.02-0.047），final 门 0.25 全拒 sources=[]。 + # 从 paper_id 派生 DOI/PMCID 回填引用字段（payload 显式字段 > + # registry > paper_id 派生）。 + _derived_doi = doi_from_paper_id(paper_id) + _derived_pmcid = pmcid_from_paper_id(paper_id) + # doi: 前缀形态（如 doi:10.1002/ps.6526）直接剥前缀（DOI 已带斜杠， + # doi_from_paper_id 不覆盖此形态）。 + if not _derived_doi and paper_id.lower().startswith("doi:"): + _candidate = paper_id[4:].strip() + if _DOI_LITERAL.fullmatch(_candidate): + _derived_doi = _candidate return EvidenceItem( evidence_id="", kb="V16/V15", @@ -352,8 +374,8 @@ class DirectFlowRetriever: paper_id=paper_id, title=_text(payload.get("title") or payload.get("paper_title") or metadata.get("title") or registry_metadata.get("title")), pmid=_text(payload.get("pmid") or payload.get("pubmed_id") or metadata.get("pmid") or registry_metadata.get("pmid")), - pmcid=_text(payload.get("pmcid") or metadata.get("pmcid") or registry_metadata.get("pmcid")), - doi=_text(payload.get("doi") or metadata.get("doi") or registry_metadata.get("doi")), + pmcid=_text(payload.get("pmcid") or metadata.get("pmcid") or registry_metadata.get("pmcid") or _derived_pmcid), + doi=_text(payload.get("doi") or metadata.get("doi") or registry_metadata.get("doi") or _derived_doi), species=_text(payload.get("species") or metadata.get("species")), section=_text(payload.get("section") or metadata.get("section")), output_layer=_text(payload.get("output_layer") or "main_kg"), diff --git a/agent/qa/src/rag/qa_v1_1_73/test_score_gate.py b/agent/qa/src/rag/qa_v1_1_73/test_score_gate.py index 6fb5f76..4e58a8f 100644 --- a/agent/qa/src/rag/qa_v1_1_73/test_score_gate.py +++ b/agent/qa/src/rag/qa_v1_1_73/test_score_gate.py @@ -294,6 +294,36 @@ def test_direct_flow_retriever_threshold_passthrough(): assert r.qdrant.calls[-1]["score_threshold"] is None +def test_item_from_payload_derives_citation_from_paper_id(): + """回改 v4 根因单测：Qdrant payload 无 pmid/pmcid/doi 时从 paper_id + 派生 DOI——chat 模式 public_citation 过滤不得把有正文的 qdrant 条目 + 全滤掉（此前 rerank 输入全是无 evidence 的 neo4j 图断言，bge 全 + 0.02-0.047，final 门 0.25 全拒 sources=[]）。""" + import rag.qa_v1_1_73.direct_flow_retriever as dfr_mod + + r = dfr_mod.DirectFlowRetriever.__new__(dfr_mod.DirectFlowRetriever) + r.paper_metadata = {} + # 内部 DOI 形态 + it = r.item_from_payload( + {"paper_id": "10.3390_f12050650", "source": "s", "relation": "r", + "target": "t", "evidence": "e"}, + "qdrant_vector", "q", + ) + assert it.doi == "10.3390/f12050650" + assert it.retrieval_methods == ["qdrant_vector"] + # doi: 前缀形态 + it2 = r.item_from_payload( + {"paper_id": "doi:10.1002/ps.6526", "source": "s", "relation": "r", + "target": "t", "evidence": "e"}, + "qdrant_vector", "q", + ) + assert it2.doi == "10.1002/ps.6526" + # 派生出的 DOI 能通过 public_citation 校验（过滤不再误杀） + from rag.qa_v1_1_73.citation_contract import public_citation + cite = public_citation(evidence_id="", pmid="", pmcid="", doi=it.doi) + assert cite == "[DOI: 10.3390/f12050650]" + + def test_env_overrides(monkeypatch): monkeypatch.setenv("QA_MIN_VECTOR_SCORE", "0.5") assert score_gate._env_value("QA_MIN_VECTOR_SCORE", 0.45) == 0.5