jerryandClaude Fable 5
1e94de7464
- BFF 薄适配层（FastAPI，8090）：QA 代理/SSE 中继/Neo4j 直连/抽取代理/会话持久化 - Vue3 前端：五相位检索流水线、文献来源与证据链、知识图谱、提取任务四 tab - 四档响应式（桌面/笔记本/平板/手机）、深浅主题 - 81 个 FR 全部实现，测试追溯矩阵 81/81，后端 121 测/前端 156 测 - 真实凭据已脱敏，运行时配置在 backend/.env（不入 git） Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
269 lines
11 KiB
Python
269 lines
11 KiB
Python
"""normalize.py —— 上游响应 → 契约形态的归一化器（API.md §7/§8/§9）。
|
||
|
||
真实上游键以 DISCOVERY.md §4.1 为准；别名规则按 API.md 逐条实现。
|
||
本模块是纯函数集合（不发起 IO），供 qa 路由（A2）与测试共用。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
from typing import Any
|
||
|
||
|
||
def pick(obj: dict, *names: str, default=None):
|
||
"""取第一个存在且非 None 的键（归一化核心工具）。保留 PM 脚手架语义。"""
|
||
for n in names:
|
||
if isinstance(obj, dict) and n in obj and obj[n] is not None:
|
||
return obj[n]
|
||
return default
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# §8 Source 归一化（缺失字段键必须存在，值为 null）
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def normalize_source(item: dict[str, Any] | None) -> dict[str, Any]:
|
||
"""单个 source item → API.md §8 形态（所有键存在，缺失为 null）。
|
||
|
||
别名映射（按 API.md §8 精确实现）：
|
||
- content|evidence|text|snippet → snippet
|
||
- doc_id|paper_id|pubmed_id|id|chunk_id → paper_id（chunk_id 单独保留）
|
||
- final_score|score|confidence|rerank_score → score（真实键 final_score）
|
||
- title|paper_title|source → title（真实 item 的 source 是断言主体实体名；
|
||
title 存在时优先取 title，两者并存时 source 落入 metadata.source_entity）
|
||
- url|doi_url → url；只有 doi 时派生 url = https://doi.org/{doi}
|
||
- evidence_id|id → id；relation_id|assertion_id → metadata.assertion_id
|
||
- output_layer / claim_scope / cluster_id / support_count / section /
|
||
species / vector_score / rrf_score / rerank_score / retrieval_methods /
|
||
retrieval_queries / cluster_member_* 原样透传（或进 metadata）
|
||
"""
|
||
if not isinstance(item, dict):
|
||
item = {}
|
||
|
||
source = pick(item, "source") # 断言主体实体名（上游语义）
|
||
title = pick(item, "title", "paper_title")
|
||
|
||
doi = pick(item, "doi")
|
||
url = pick(item, "url", "doi_url")
|
||
if url is None and doi:
|
||
url = f"https://doi.org/{doi}"
|
||
|
||
metadata: dict[str, Any] = {}
|
||
if isinstance(pick(item, "metadata"), dict):
|
||
metadata.update(item["metadata"])
|
||
if source is not None and title is not None and source != title:
|
||
metadata.setdefault("source_entity", source)
|
||
for key in (
|
||
"vector_score", "rrf_score", "rerank_score", "kb",
|
||
"retrieval_methods", "retrieval_queries",
|
||
"cluster_member_count", "cluster_member_assertion_ids",
|
||
"cluster_member_paper_ids", "cluster_member_pmids",
|
||
"output_layer", "claim_scope", "species",
|
||
):
|
||
if key in item and item[key] is not None:
|
||
metadata.setdefault(key, item[key])
|
||
|
||
assertion_id = pick(item, "relation_id", "assertion_id")
|
||
if assertion_id is not None:
|
||
metadata.setdefault("assertion_id", assertion_id)
|
||
|
||
return {
|
||
"id": pick(item, "evidence_id", "id"),
|
||
"title": title,
|
||
"paper_id": pick(item, "doc_id", "paper_id", "pubmed_id", "id", "chunk_id"),
|
||
"chunk_id": pick(item, "chunk_id"),
|
||
"pmid": pick(item, "pmid"),
|
||
"pmcid": pick(item, "pmcid"),
|
||
"doi": doi,
|
||
"url": url,
|
||
"snippet": pick(item, "content", "evidence", "text", "snippet"),
|
||
"score": pick(item, "final_score", "score", "confidence", "rerank_score"),
|
||
"locator": pick(item, "locator"),
|
||
"section": pick(item, "section"),
|
||
"support_count": pick(item, "support_count"),
|
||
"type": pick(item, "type"),
|
||
"metadata": metadata,
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# §9 图谱：hop / chain / {nodes,edges,chains}
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def normalize_hop(hop: dict[str, Any]) -> dict[str, Any]:
|
||
"""hop 归一化：
|
||
from_name|source → source、to_name|target → target、
|
||
from_id|source_canonical_id → source 节点 id、to_id|target_canonical_id → target 节点 id、
|
||
relation 原样、source_type|target_type → source/target 节点 type、
|
||
paper_id / pmid / pmcid / doi 透传。
|
||
"""
|
||
if not isinstance(hop, dict):
|
||
hop = {}
|
||
return {
|
||
"source": pick(hop, "from_name", "source"),
|
||
"target": pick(hop, "to_name", "target"),
|
||
"source_id": pick(hop, "from_id", "source_canonical_id"),
|
||
"target_id": pick(hop, "to_id", "target_canonical_id"),
|
||
"source_type": pick(hop, "source_type"),
|
||
"target_type": pick(hop, "target_type"),
|
||
"relation": pick(hop, "relation"),
|
||
"assertion_id": pick(hop, "assertion_id"),
|
||
"paper_id": pick(hop, "paper_id"),
|
||
"pmid": pick(hop, "pmid"),
|
||
"pmcid": pick(hop, "pmcid"),
|
||
"doi": pick(hop, "doi"),
|
||
}
|
||
|
||
|
||
def normalize_chain_item(item: dict[str, Any]) -> dict[str, Any]:
|
||
"""单个 chain item（真实形态：chain_id / hops[] 等）→ {chain_id, hops}。"""
|
||
if not isinstance(item, dict):
|
||
item = {}
|
||
hops = [normalize_hop(h) for h in item.get("hops", []) if isinstance(h, dict)]
|
||
return {"chain_id": pick(item, "chain_id"), "hops": hops}
|
||
|
||
|
||
def extract_chains(raw: dict[str, Any] | None) -> list[dict[str, Any]]:
|
||
"""从可能的两类源提取 chain 列表：
|
||
- `native_graph_chains`：列表（真实形态，305 条）
|
||
- `graph_knowledge_chains`：dict 形态 `{mode, chains[], chain_count, …}`，取 `.chains`
|
||
两种形态都要能提取出 hop 列表。
|
||
"""
|
||
if not isinstance(raw, dict):
|
||
return []
|
||
chains = pick(raw, "native_graph_chains", "graph_knowledge_chains")
|
||
if isinstance(chains, dict):
|
||
chains = chains.get("chains")
|
||
if not isinstance(chains, list):
|
||
return []
|
||
return [normalize_chain_item(c) for c in chains]
|
||
|
||
|
||
def normalize_graph(raw: dict[str, Any] | None) -> dict[str, Any]:
|
||
"""从 `knowledge_graph` / `graph_edges` / `native_graph_chains`（列表形态）/
|
||
`graph_knowledge_chains`（dict 形态）提取并归一化为 {nodes, edges, chains}。
|
||
|
||
- 节点去重（按 id；id 缺失的 hop 跳过）；label 取 from_name/to_name、type 取
|
||
source_type/target_type。
|
||
- 边携带 chain_id 与 hop 序号（hop_index 缺失时按列表序补）。
|
||
- 链保持 chain_id + hops 列表。
|
||
"""
|
||
raw = raw if isinstance(raw, dict) else {}
|
||
chains = extract_chains(raw)
|
||
|
||
nodes: dict[str, dict[str, Any]] = {}
|
||
edges: list[dict[str, Any]] = []
|
||
|
||
def add_node(node_id: Any, name: Any, typ: Any) -> None:
|
||
if not node_id:
|
||
return
|
||
key = str(node_id)
|
||
if key not in nodes:
|
||
nodes[key] = {"id": node_id, "label": name, "type": typ}
|
||
|
||
for chain in chains:
|
||
for idx, hop in enumerate(chain["hops"]):
|
||
hop_index = hop.get("hop_index")
|
||
if hop_index is None:
|
||
hop_index = idx
|
||
# 节点 id 优先 canonical id；label 优先名字，退化到 id
|
||
src_id = pick(hop, "source_id", "source") or hop.get("source")
|
||
tgt_id = pick(hop, "target_id", "target") or hop.get("target")
|
||
src_label = hop.get("source") or src_id
|
||
tgt_label = hop.get("target") or tgt_id
|
||
add_node(src_id, src_label, hop.get("source_type"))
|
||
add_node(tgt_id, tgt_label, hop.get("target_type"))
|
||
if src_id and tgt_id:
|
||
edges.append({
|
||
"source": src_id,
|
||
"target": tgt_id,
|
||
"relation": hop.get("relation"),
|
||
"label": hop.get("relation"),
|
||
"chain_id": chain.get("chain_id"),
|
||
"hop": hop_index,
|
||
"paper_id": hop.get("paper_id"),
|
||
"pmid": hop.get("pmid"),
|
||
"pmcid": hop.get("pmcid"),
|
||
"doi": hop.get("doi"),
|
||
})
|
||
|
||
# 兼容显式 {nodes, edges} 形态（knowledge_graph / graph_edges 直接给边列表时）
|
||
explicit_nodes = raw.get("nodes")
|
||
explicit_edges = raw.get("edges")
|
||
if isinstance(explicit_nodes, list):
|
||
for n in explicit_nodes:
|
||
if isinstance(n, dict) and n.get("id"):
|
||
key = str(n["id"])
|
||
if key not in nodes:
|
||
nodes[key] = {
|
||
"id": n["id"],
|
||
"label": pick(n, "label"),
|
||
"type": pick(n, "type"),
|
||
}
|
||
if isinstance(explicit_edges, list):
|
||
edges.extend(
|
||
{
|
||
"source": pick(e, "source"),
|
||
"target": pick(e, "target"),
|
||
"relation": pick(e, "relation"),
|
||
"label": pick(e, "label", "relation"),
|
||
"chain_id": pick(e, "chain_id"),
|
||
"hop": pick(e, "hop"),
|
||
"paper_id": pick(e, "paper_id"),
|
||
"pmid": pick(e, "pmid"),
|
||
"pmcid": pick(e, "pmcid"),
|
||
"doi": pick(e, "doi"),
|
||
}
|
||
for e in explicit_edges
|
||
if isinstance(e, dict)
|
||
)
|
||
|
||
return {
|
||
"nodes": list(nodes.values()),
|
||
"edges": edges,
|
||
"chains": chains,
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# §7 归一化 QA 响应（/chat、/chat/stream complete、/search 共用）
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def normalize_qa_response(raw: dict[str, Any] | None) -> dict[str, Any]:
|
||
"""上游 31 键响应 → API.md §7 形态。
|
||
|
||
别名（真实键优先）：
|
||
- answer|response|message → answer
|
||
- sources|citations → sources；used_sources|references → used_sources
|
||
- query_plan|agentic_plan|retrieval_plan → retrieval_plan（真实键 query_plan）
|
||
- retrieval_notes|notes → retrieval_notes
|
||
- timings|timing → timing（真实键 timings）
|
||
- citation_validation|citation_map → citation_map（真实键 citation_validation）
|
||
- 图谱四源 → graph + chains（见 normalize_graph）
|
||
- success 透传；raw 保存原始完整响应
|
||
"""
|
||
raw = raw if isinstance(raw, dict) else {}
|
||
|
||
sources = [normalize_source(s) for s in (pick(raw, "sources", "citations") or [])]
|
||
used_sources = [
|
||
normalize_source(s) for s in (pick(raw, "used_sources", "references") or [])
|
||
]
|
||
graph = normalize_graph(raw)
|
||
retrieval_plan = pick(raw, "query_plan", "agentic_plan", "retrieval_plan")
|
||
|
||
return {
|
||
"query_id": pick(raw, "query_id"),
|
||
"version": pick(raw, "version"),
|
||
"mode": pick(raw, "mode", default="chat"),
|
||
"answer": pick(raw, "answer", "response", "message", default=""),
|
||
"source_count": pick(raw, "source_count", default=len(sources)),
|
||
"sources": sources,
|
||
"used_sources": used_sources,
|
||
"chains": graph["chains"],
|
||
"graph": {"nodes": graph["nodes"], "edges": graph["edges"], "chains": graph["chains"]},
|
||
"retrieval_plan": retrieval_plan,
|
||
"retrieval_notes": pick(raw, "retrieval_notes", "notes"),
|
||
"timing": pick(raw, "timings", "timing"),
|
||
"citation_map": pick(raw, "citation_validation", "citation_map"),
|
||
"success": pick(raw, "success", default=True),
|
||
"raw": raw,
|
||
}
|