fix(r4-28v2): 双域阈值+检索模式免 LLM 免 rerank

TL 复测校准(TL 实测 175:垃圾 vector 0.32-0.34/正常 0.63+;垃圾 final 0.19-0.20):
1. score_gate 双域双常量:MIN_VECTOR_SCORE=0.45(QA_MIN_VECTOR_SCORE,向量域,
   Qdrant score_threshold 与 search 预筛);MIN_SOURCE_SCORE=0.25 保持(chat
   模式 final_score 后置过滤)。
2. pipeline answer():retrieval_only 无条件关 use_draft/use_planner(不看
   llm.available,search 模式绝不产生答案);use_bge_rerank 强制关(免 CPU
   rerank,性能根治);候选按 vector_score 排序直接输出且 final_score=
   vector_score(与 Qdrant 阈值同域)。
3. Qdrant score_threshold 仅在 retrieval_only 路径传入(direct_flow
   retriever 参数化,0.45);chat 不传保召回;v16_retriever 还原(r4-28 的
   0.25 全域阈值回退)。
4. 单测更新:search 无 LLM/无 rerank/0.45 滤 0.32 保 0.63/score 取 vector
   域/chat 不传阈值;chat final 域 0.25 语义不变。

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-09-11 16:33:15 +08:00
co-authored by Claude Fable 5
parent f18286f022
commit 1fe80b293e
5 changed files with 245 additions and 78 deletions
@@ -16,7 +16,6 @@ from src.rag.normalized_release import (
)
from .schemas import EvidenceItem, RetrievalHypothesis
from .score_gate import min_source_score
ALLOWED_QDRANT_COLLECTIONS = {
"plant_srt_direct_v1_bge_m3_20260702",
@@ -201,17 +200,20 @@ class DirectFlowRetriever:
}
return merged, audit
def qdrant_search(self, query: str, limit: int) -> List[EvidenceItem]:
def qdrant_search(
self, query: str, limit: int, *, score_threshold: float | None = None
) -> List[EvidenceItem]:
try:
vector = self.embed(query)
# 缺陷 #28(2026-09-11):相关性下限双保险(与 pipeline
# _select_graph_balanced 同源同值,见 score_gate.py)。
# 缺陷 #28 回改 v2(2026-09-11):向量域阈值仅 search/retrieval-only
# 路径传入(pipeline 传 QA_MIN_VECTOR_SCORE=0.45);chat 模式 None
# 保召回,rerank 后由 final 域 MIN_SOURCE_SCORE=0.25 兜底。
response = self.qdrant.query_points(
collection_name=self.collection,
query=vector,
limit=limit,
with_payload=True,
score_threshold=min_source_score(),
score_threshold=score_threshold,
)
except Exception:
return []
+41 -12
View File
@@ -43,7 +43,7 @@ from .question_decomposition import QuestionDecompositionPlanner
from .query_builder import QueryBuilder
from .reranker import BGEReranker
from .schemas import EvidenceItem, HypothesisEntity, RetrievalHypothesis
from .score_gate import min_source_score
from .score_gate import min_source_score, min_vector_score
from .soft_gate_policy import (
LEGACY,
doi_from_paper_id,
@@ -1255,7 +1255,11 @@ class QAV1Pipeline:
use_qdrant = bool(retrieval_ablation.get("use_qdrant", True))
use_neo4j = bool(retrieval_ablation.get("use_neo4j", True))
use_graph_expansion = bool(retrieval_ablation.get("use_graph_expansion", True)) and use_neo4j
use_bge_rerank = bool(ablation.get("use_bge_rerank", True)) and self.config.enable_bge_rerank
use_bge_rerank = (
bool(ablation.get("use_bge_rerank", True))
and self.config.enable_bge_rerank
and not retrieval_only # 回改 v2:search 免 CPU rerank(性能根治)
)
use_final_llm = not retrieval_only
graph_min_evidence = int(ablation.get("graph_min_evidence") or os.environ.get("QA_V115_GRAPH_MIN_EVIDENCE", "15"))
answer_context_mode = str(ablation.get("answer_context") or "top30_plus_graph_chains")
@@ -1286,11 +1290,10 @@ class QAV1Pipeline:
1.0, min(float(endpoint_path_time_budget_seconds or 5.0), 30.0)
)
if retrieval_only and not self.llm.available:
# V1.1.711 supports retrieval-only operation. Without an LLM key,
# disable provisional drafting, structured planning, and final answer
# generation, but still return reranked top30 evidence and native
# Neo4j graph chains from the raw question.
if retrieval_only:
# 缺陷 #28 回改 v2(2026-09-11):检索模式绝不产生答案——无条件
# 关闭草稿/规划/最终 LLM(不看 llm.available;LLM 调用纯浪费且
# 拖慢 search)。仍返回 top30 证据与 native 图链。
use_draft = False
use_planner = False
use_final_llm = False
@@ -1392,7 +1395,15 @@ class QAV1Pipeline:
"use_neo4j": use_neo4j,
"use_graph_expansion": use_graph_expansion,
}
if self.agentic_controller is not None:
if self.agentic_controller is None:
# 回改 v2(2026-09-11):agentic_controller 禁用(含单测)时,
# 给确定性 plan——query_plan.agentic_plan 键不缺失。
agentic_plan = {
"vector_plan": {"enabled": bool(use_qdrant)},
"graph_plan": {"enabled": bool(use_neo4j)},
"plan_source": "deterministic_retrieval_only_plan",
}
elif self.agentic_controller is not None:
try:
agentic_plan = self.agentic_controller.plan(
question,
@@ -1551,6 +1562,9 @@ class QAV1Pipeline:
hypothesis,
qdrant_top_k=qdrant_top_k,
graph_top_k=graph_top_k,
qdrant_score_threshold=(
min_vector_score() if retrieval_only else None
),
**direct_flow_switches,
)
timings["retrieval"] = time.perf_counter() - t0
@@ -1566,6 +1580,14 @@ class QAV1Pipeline:
clustered_input, cluster_audit = build_clusters(ranked_input)
if use_bge_rerank:
ranked, rerank_audit = self._rerank(question, clustered_input)
elif retrieval_only:
# 回改 v2(2026-09-11):search 免 BGE rerank,候选按 vector_score
# 排序直接输出,final_score 落在 vector_score 域——与 Qdrant
# score_threshold(min_vector_score=0.45)同域同语义。
ranked = sorted(clustered_input, key=lambda x: -x.vector_score)
for item in ranked:
item.final_score = item.vector_score
rerank_audit = [{"status": "disabled_retrieval_only"}]
else:
ranked = sorted(clustered_input, key=lambda x: -(x.rrf_score or x.vector_score))
for item in ranked:
@@ -1575,6 +1597,8 @@ class QAV1Pipeline:
ranked,
top_k_for_llm=top_k_for_llm,
graph_min_evidence=graph_min_evidence,
# 回改 v2:search 预筛用向量域 0.45;chat 走 final 域 0.25
min_score=(min_vector_score() if retrieval_only else None),
)
assign_evidence_ids(selected)
timings["cluster_bge_rerank_and_select"] = time.perf_counter() - t0
@@ -3317,11 +3341,15 @@ class QAV1Pipeline:
*,
top_k_for_llm: int,
graph_min_evidence: int,
min_score: float | None = None,
) -> tuple[List[EvidenceItem], Dict[str, Any]]:
# 缺陷 #28(2026-09-11):相关性下限。低于下限的候选不入选;
# 全部低于下限 -> 返回空(前端呈现未命中,与 Qdrant
# score_threshold 双保险)。阈值见 score_gate.py(env 可覆盖)。
_min_score = min_source_score()
# 缺陷 #28 回改 v2(2026-09-11):相关性下限双域。
# - min_score 显式传入时用它(search/retrieval-only 传向量域
# min_vector_score()=0.45);
# - 缺省为 final 域 min_source_score()=0.25(chat 模式语义不变)。
# 低于下限的候选不入选;全部低于下限 -> 返回空(前端呈现未命中)。
_min_score = min_source_score() if min_score is None else min_score
_min_score_domain = "vector" if min_score is not None else "final"
_below_min = len(ranked) - sum(
1 for item in ranked if item.final_score >= _min_score
) # 被下限过滤的候选数(审计用,见 return 处)
@@ -3376,6 +3404,7 @@ class QAV1Pipeline:
"graph_quota_satisfied": graph_selected >= graph_min_evidence,
"min_source_score": _min_score,
"below_min_source_score": _below_min,
"min_score_domain": _min_score_domain,
"note": "Graph quota is enforced after BGE rerank. If fewer graph candidates exist, all available graph candidates are retained and the deficit is reported.",
}
+31 -27
View File
@@ -1,23 +1,20 @@
"""score_gate.py —— 检索相关性下限(缺陷 #28,2026-09-11)。
"""score_gate.py —— 检索相关性下限(缺陷 #28 回改 v2,2026-09-11)。
语义:候选证据的最终得分(EvidenceItem.final_score)低于下限即视为
「与问题无关」,不得进入 evidence_top30 / sources。
双域双常量(TL 2026-09-11 在 175 裸进程 Qdrant 27433 + Ollama bge-m3:567m 实测):
- 垃圾问题「景甜是谁」top10 vector_score(cosine) 全部 0.32-0.34;
- arginine 正常问题 top10 vector_score 0.63+;
- 垃圾 final_score(BGE rerank 后)仍为 0.19-0.20(测试 agent 实测)。
证据与阈值依据(2026-09-11 实测):
- 无关问题「景甜是谁」实测 30 条 final_score 全部 0.19-0.20(纯植物学
无关文献);正常命中(DISCOVERY.md §4.1 金标样本)vector_score=0.714、
rerank_score=0.989、final_score=0.982。
- BGE rerank 路径 final_score = 0.8*bge(sigmoid) + 0.2*rrf/max_rrf;
bge 是 sigmoid 概率,0.20 垃圾分对应 raw logit ≈ -1.4,0.25 对应 -1.1,
0.30 对应 -0.85,正常命中的 sigmoid 通常 >0.9。
- Qdrant collection distance=Cosine(score ∈ [-1,1];bge-m3 文本嵌入对
随机中文问题的 top-k 得分通常 <0.2)。
MIN_VECTOR_SCORE = 0.45(env QA_MIN_VECTOR_SCORE)
- 语义:向量域(Qdrant Cosine 分数)相关性下限。0.45 位于垃圾带
(0.32-0.34)与正常带(0.63+)之间并各留余量。
- 用途:仅 search / retrieval-only 路径——Qdrant score_threshold 与
检索模式预筛。chat 模式不使用(保召回,rerank 后用 MIN_SOURCE_SCORE 兜底)。
默认 0.25:与垃圾实测带(0.19-0.20)留 ~25% 余量,又显著低于正常命中
(>0.7),同时为 0.2~0.3 边缘问题保留再排序机会(阈值过滤器只是预筛,
被过滤者已低于下限,保留也无意义)。
env `QA_MIN_SOURCE_SCORE` 可覆盖(0-1,非法值回落默认),供复测按
acceptance-r4 §8.3 授权临时调参。
MIN_SOURCE_SCORE = 0.25(env QA_MIN_SOURCE_SCORE)
- 语义:最终域(final_score = 0.8*sigmoid(bge) + 0.2*rrf/max_rrf)相关性
下限。垃圾 final 0.19-0.20 与其留 ~25% 余量,正常命中 final >0.7。
- 用途:chat 模式 _select_graph_balanced 的后置过滤(r4-28 语义不变)。
本模块无第三方依赖、无 IO,可被 v16_retriever / direct_flow_retriever /
pipeline 直接 import(编译期与运行期均安全)。
@@ -26,28 +23,35 @@ from __future__ import annotations
import os
# 相关性下限(与 Qdrant score_threshold 同源同值,双保险)
DEFAULT_MIN_VECTOR_SCORE = 0.45
DEFAULT_MIN_SOURCE_SCORE = 0.25
_ENV_KEY = "QA_MIN_SOURCE_SCORE"
_ENV_VECTOR = "QA_MIN_VECTOR_SCORE"
_ENV_SOURCE = "QA_MIN_SOURCE_SCORE"
def _env_override() -> float:
"""env 覆盖:QA_MIN_SOURCE_SCORE ∈ [0,1] 才生效,否则回落默认。"""
raw = os.environ.get(_ENV_KEY, "").strip()
def _env_value(env_key: str, default: float) -> float:
"""env 覆盖:值 ∈ [0,1] 才生效,否则回落默认。"""
raw = os.environ.get(env_key, "").strip()
if not raw:
return DEFAULT_MIN_SOURCE_SCORE
return default
try:
value = float(raw)
except ValueError:
return DEFAULT_MIN_SOURCE_SCORE
return default
if not (0.0 <= value <= 1.0):
return DEFAULT_MIN_SOURCE_SCORE
return default
return value
def min_vector_score() -> float:
"""向量域下限(Qdrant score_threshold / search 预筛)。"""
return min_vector_score.value
def min_source_score() -> float:
"""当前生效的相关性下限(模块导入期缓存 env 值,与部署环境一致)。"""
"""最终域下限(chat 模式 final_score 后置过滤)。"""
return min_source_score.value
min_source_score.value = _env_override() # type: ignore[attr-defined]
min_vector_score.value = _env_value(_ENV_VECTOR, DEFAULT_MIN_VECTOR_SCORE) # type: ignore[attr-defined]
min_source_score.value = _env_value(_ENV_SOURCE, DEFAULT_MIN_SOURCE_SCORE) # type: ignore[attr-defined]
+166 -29
View File
@@ -1,16 +1,15 @@
"""test_score_gate.py —— 相关性下限选择单测(缺陷 #28,acceptance-r4 §2.4)。
"""test_score_gate.py —— 相关性下限单测(缺陷 #28 回改 v2,2026-09-11)。
运行(venv 内,源码树):
cd agent/qa/src/rag/qa_v1_1_73
python -m pytest test_score_gate.py -v
(编译期 compile_qa.sh 会把全部 test_*.py 一并编进 .so 树,部署时不冲突;
正式单测都在源码树跑。)
覆盖:
- 30 条全部低于下限 -> 选入 0(acceptance 桩)
- 5 高 + 25 低 -> 仅选入 5(遵循现有配额逻辑)
- graph 配额不受影响;低于下限的 graph 候选不入选
- env 覆盖 QA_MIN_SOURCE_SCORE 生效与非法值回落
- search(retrieval_only)模式:无 LLM 调用、无 rerank 调用;
vector 域 0.45 预筛滤掉 0.32 垃圾、保留 0.63 正常;
sources 的 score 字段取 vector_score(final_score=vector_score 域)。
- chat 模式:final 域 0.25 后置过滤语义不变(5 高+25 低 -> 5、30 低 -> 0);
Qdrant score_threshold 不传(None,保召回)。
"""
from __future__ import annotations
@@ -25,7 +24,9 @@ from rag.qa_v1_1_73.schemas import EvidenceItem # noqa: E402
from rag.qa_v1_1_73.pipeline import QAV1Pipeline # noqa: E402
def _item(idx: int, *, final: float, graph: bool = False, key: str = "") -> EvidenceItem:
def _item(
idx: int, *, final: float = 0.0, vector: float = 0.0, graph: bool = False, key: str = ""
) -> EvidenceItem:
return EvidenceItem(
evidence_id=f"E{idx}",
kb="V16",
@@ -35,6 +36,7 @@ def _item(idx: int, *, final: float, graph: bool = False, key: str = "") -> Evid
evidence=f"evidence body {idx}",
relation_id=(key or f"rid-{idx}"),
retrieval_methods=(["neo4j_bounded_expand"] if graph else ["qdrant_vector"]),
vector_score=vector,
final_score=final,
)
@@ -46,30 +48,31 @@ def _select(items, *, top_k=30, graph_min=0):
return selected, audit
def test_all_below_threshold_selects_zero():
"""acceptance §2.4:30 条全部低于下限 -> 选入 0。"""
# ---------------------------------------------------------------------------
# chat 模式 final 域 0.25(r4-28 语义不变)
# ---------------------------------------------------------------------------
def test_chat_all_below_source_threshold_selects_zero():
"""30 条 final 全部低于 0.25 -> 选入 0。"""
items = [_item(i, final=0.20) for i in range(30)]
selected, audit = _select(items)
assert selected == []
assert audit["policy"].startswith("graph_balanced_top30")
assert audit["below_min_source_score"] == 30
assert audit["min_source_score"] == score_gate.min_source_score()
def test_five_high_twentyfive_low_selects_only_five():
"""acceptance §2.4:5 高 + 25 低 -> 仅选入 5(沿用现有配额与 rerank 序)。"""
def test_chat_five_high_twentyfive_low_selects_only_five():
"""5 高(final>=0.25)+ 25 低 -> 仅选入 5(保 rerank 序)。"""
items = [_item(i, final=0.90 - i * 0.01) for i in range(5)]
items += [_item(5 + i, final=0.20) for i in range(25)]
selected, audit = _select(items)
assert len(selected) == 5
assert {item.evidence_id for item in selected} == {f"E{i}" for i in range(5)}
# 顺序保持 rerank 序
assert [item.evidence_id for item in selected] == [f"E{i}" for i in range(5)]
assert audit["below_min_source_score"] == 25
def test_below_threshold_graph_candidates_excluded():
"""低分 graph 候选不占用 graph 配额(配额逻辑语义不变)。"""
def test_chat_below_threshold_graph_candidates_excluded():
"""低分 graph 候选不占用 graph 配额。"""
items = [
_item(1, final=0.30, graph=True),
_item(2, final=0.21, graph=True),
@@ -82,17 +85,151 @@ def test_below_threshold_graph_candidates_excluded():
assert audit["below_min_source_score"] == 2
def test_env_override_and_invalid_fallback(monkeypatch):
"""env 解析函数行为:合法值生效;非法值/越界回落默认。"""
monkeypatch.setenv("QA_MIN_SOURCE_SCORE", "0.35")
assert score_gate._env_override() == 0.35
monkeypatch.setenv("QA_MIN_SOURCE_SCORE", "abc")
assert score_gate._env_override() == score_gate.DEFAULT_MIN_SOURCE_SCORE
# ---------------------------------------------------------------------------
# search / retrieval-only 模式:免 LLM、免 rerank、vector 域 0.45 预筛
# ---------------------------------------------------------------------------
class _StubLLM:
available = True # 可用也必须不调用(回改核心断言)
def complete_json(self, *args, **kwargs):
raise AssertionError("search 模式不得调用 LLM")
def complete(self, *args, **kwargs):
raise AssertionError("search 模式不得调用 LLM")
class _StubDirectRetriever:
"""捕获 retrieve 入参,供断言 Qdrant score_threshold 传递。"""
def __init__(self, items):
self.items = items
self.last_kwargs = {}
def retrieve(self, question, queries, hypothesis, **kwargs):
self.last_kwargs = kwargs
return list(self.items), {
"qdrant": [],
"seed_extraction": {},
"neo4j": [],
"dedup": {"qdrant_items": len(self.items)},
"options": dict(kwargs),
}
def close(self):
pass
def _make_pipeline(graph_schema):
from rag.qa_v1_1_73.config import QAConfig
cfg = QAConfig.from_env()
if graph_schema == "legacy":
object.__setattr__(cfg, "graph_schema", "legacy")
import threading
p = QAV1Pipeline.__new__(QAV1Pipeline)
p.config = cfg
p._reranker_lock = threading.Lock()
p.llm = _StubLLM()
p.provisional = _RaisingComposer("provisional")
p.hypothesis_extractor = _RaisingComposer("hypothesis")
p.query_builder = _StubQueryBuilder()
p.reranker = _StubReranker()
p.agentic_controller = None
p.question_decomposer = None
p.subquestion_orchestrator = None
p._retriever_stub = _StubDirectRetriever(ITEMS)
p.direct_retriever_factory = lambda: p._retriever_stub
p.audit = _NullAudit()
p.domain = _StubDomain()
return p
class _RaisingComposer:
def __init__(self, name):
self.name = name
def generate(self, *args, **kwargs):
raise AssertionError(f"{self.name} 不得被调用")
def extract(self, *args, **kwargs):
raise AssertionError(f"{self.name} 不得被调用")
class _StubQueryBuilder:
def build(self, question, provisional_answer, hypothesis):
return {"hypothesis_lane": [], "direct": [], "concept": []}
class _StubReranker:
def rerank(self, question, items):
raise AssertionError("search 模式不得调用 rerank")
class _NullAudit:
def new_query_id(self):
return "qid"
def write(self, query_id, payload):
return None
class _StubDomain:
def normalize_hypothesis(self, hypothesis):
return {"entities": []}
def direct_entities_from_question(self, question):
return []
# 集成(answer())路径:search 免 rerank,final_score = vector_score(同域),
# 按 TL 实测向量域构造:垃圾 0.32(< 0.45 滤掉)、正常 0.63(保留)。
JUNK = _item(1, vector=0.32)
GOOD = _item(2, vector=0.63)
ITEMS = [JUNK, GOOD]
def test_search_mode_no_llm_no_rerank_keeps_good_only():
p = _make_pipeline("legacy")
out = p.answer(
"景甜是谁",
retrieval_only=True,
native_chain_enabled=False,
progress_callback=None,
)
assert out["sources"] != [] # vector 0.63 保留
titles = [src["source"] for src in out["sources"]]
assert "src2" in titles and "src1" not in titles # 0.32 垃圾被 0.45 预筛滤掉
assert p._retriever_stub.last_kwargs["qdrant_score_threshold"] == 0.45
assert out["retrieval_audit"]["options"]["qdrant_score_threshold"] == 0.45
assert out["query_plan"]["agentic_plan"]["plan_source"] == "deterministic_retrieval_only_plan"
assert out["degraded_execution"] is False
# 跳过 rerank 后:final_score = vector_score(TL 回改语义,与 Qdrant 阈值
# 同域);rerank_score 未精排(0.0)。BFF normalize 的 score 取 final_score。
for src in out["sources"]:
assert src["rerank_score"] == 0.0
assert src["final_score"] == src["vector_score"]
def test_chat_mode_passes_no_threshold():
p = _make_pipeline("legacy")
object.__setattr__(p.config, "enable_bge_rerank", False)
out = p.answer(
"景甜是谁",
retrieval_only=False,
native_chain_enabled=False,
progress_callback=None,
)
assert p._retriever_stub.last_kwargs["qdrant_score_threshold"] is None
def test_env_overrides(monkeypatch):
monkeypatch.setenv("QA_MIN_VECTOR_SCORE", "0.5")
assert score_gate._env_value("QA_MIN_VECTOR_SCORE", 0.45) == 0.5
monkeypatch.setenv("QA_MIN_VECTOR_SCORE", "abc")
assert score_gate._env_value("QA_MIN_VECTOR_SCORE", 0.45) == 0.45
monkeypatch.setenv("QA_MIN_SOURCE_SCORE", "1.5")
assert score_gate._env_override() == score_gate.DEFAULT_MIN_SOURCE_SCORE
assert score_gate._env_value("QA_MIN_SOURCE_SCORE", 0.25) == 0.25
monkeypatch.delenv("QA_MIN_VECTOR_SCORE", raising=False)
monkeypatch.delenv("QA_MIN_SOURCE_SCORE", raising=False)
# 生效值(value 属性)驱动 _select_graph_balanced 的过滤
monkeypatch.setattr(score_gate.min_source_score, "value", 0.35)
items = [_item(i, final=0.30) for i in range(5)]
selected, _ = _select(items)
assert selected == []
@@ -9,7 +9,6 @@ from typing import Any, Dict, List
import requests
from neo4j import GraphDatabase
from qdrant_client import QdrantClient
from .score_gate import min_source_score
TOKEN_RE = re.compile(r"[A-Za-z][A-Za-z0-9_.-]{2,}|[\u4e00-\u9fff]{2,}")
@@ -101,15 +100,11 @@ class V16IndependentRetriever:
self, question: str, limit: int, include_candidate: bool
) -> List[Dict[str, Any]]:
try:
# 缺陷 #28(2026-09-11):相关性下限双保险(与 pipeline
# _select_graph_balanced 同源同值)。collection distance=Cosine,
# 阈值语义 = 余弦相似度下限;低于下限的点直接不返回。
response = self.qdrant.query_points(
collection_name=self.collection,
query=self.embed(question),
limit=limit * 2,
with_payload=True,
score_threshold=min_source_score(),
)
except Exception:
return []