269 lines
11 KiB
Python
269 lines
11 KiB
Python
|
"""normalize.py —— 上游响应 → 契约形态的归一化器（API.md §7/§8/§9）。
|
|||
|
|
|
|||
|
|
真实上游键以 DISCOVERY.md §4.1 为准；别名规则按 API.md 逐条实现。
|
|||
|
|
本模块是纯函数集合（不发起 IO），供 qa 路由（A2）与测试共用。
|
|||
|
|
"""
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
from typing import Any
|
|||
|
|
|
|||
|
|
|
|||
|
|
def pick(obj: dict, *names: str, default=None):
|
|||
|
|
"""取第一个存在且非 None 的键（归一化核心工具）。保留 PM 脚手架语义。"""
|
|||
|
|
for n in names:
|
|||
|
|
if isinstance(obj, dict) and n in obj and obj[n] is not None:
|
|||
|
|
return obj[n]
|
|||
|
|
return default
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
# §8 Source 归一化（缺失字段键必须存在，值为 null）
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
def normalize_source(item: dict[str, Any] | None) -> dict[str, Any]:
|
|||
|
|
"""单个 source item → API.md §8 形态（所有键存在，缺失为 null）。
|
|||
|
|
|
|||
|
|
别名映射（按 API.md §8 精确实现）：
|
|||
|
|
- content|evidence|text|snippet → snippet
|
|||
|
|
- doc_id|paper_id|pubmed_id|id|chunk_id → paper_id（chunk_id 单独保留）
|
|||
|
|
- final_score|score|confidence|rerank_score → score（真实键 final_score）
|
|||
|
|
- title|paper_title|source → title（真实 item 的 source 是断言主体实体名；
|
|||
|
|
title 存在时优先取 title，两者并存时 source 落入 metadata.source_entity）
|
|||
|
|
- url|doi_url → url；只有 doi 时派生 url = https://doi.org/{doi}
|
|||
|
|
- evidence_id|id → id；relation_id|assertion_id → metadata.assertion_id
|
|||
|
|
- output_layer / claim_scope / cluster_id / support_count / section /
|
|||
|
|
species / vector_score / rrf_score / rerank_score / retrieval_methods /
|
|||
|
|
retrieval_queries / cluster_member_* 原样透传（或进 metadata）
|
|||
|
|
"""
|
|||
|
|
if not isinstance(item, dict):
|
|||
|
|
item = {}
|
|||
|
|
|
|||
|
|
source = pick(item, "source") # 断言主体实体名（上游语义）
|
|||
|
|
title = pick(item, "title", "paper_title")
|
|||
|
|
|
|||
|
|
doi = pick(item, "doi")
|
|||
|
|
url = pick(item, "url", "doi_url")
|
|||
|
|
if url is None and doi:
|
|||
|
|
url = f"https://doi.org/{doi}"
|
|||
|
|
|
|||
|
|
metadata: dict[str, Any] = {}
|
|||
|
|
if isinstance(pick(item, "metadata"), dict):
|
|||
|
|
metadata.update(item["metadata"])
|
|||
|
|
if source is not None and title is not None and source != title:
|
|||
|
|
metadata.setdefault("source_entity", source)
|
|||
|
|
for key in (
|
|||
|
|
"vector_score", "rrf_score", "rerank_score", "kb",
|
|||
|
|
"retrieval_methods", "retrieval_queries",
|
|||
|
|
"cluster_member_count", "cluster_member_assertion_ids",
|
|||
|
|
"cluster_member_paper_ids", "cluster_member_pmids",
|
|||
|
|
"output_layer", "claim_scope", "species",
|
|||
|
|
):
|
|||
|
|
if key in item and item[key] is not None:
|
|||
|
|
metadata.setdefault(key, item[key])
|
|||
|
|
|
|||
|
|
assertion_id = pick(item, "relation_id", "assertion_id")
|
|||
|
|
if assertion_id is not None:
|
|||
|
|
metadata.setdefault("assertion_id", assertion_id)
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
"id": pick(item, "evidence_id", "id"),
|
|||
|
|
"title": title,
|
|||
|
|
"paper_id": pick(item, "doc_id", "paper_id", "pubmed_id", "id", "chunk_id"),
|
|||
|
|
"chunk_id": pick(item, "chunk_id"),
|
|||
|
|
"pmid": pick(item, "pmid"),
|
|||
|
|
"pmcid": pick(item, "pmcid"),
|
|||
|
|
"doi": doi,
|
|||
|
|
"url": url,
|
|||
|
|
"snippet": pick(item, "content", "evidence", "text", "snippet"),
|
|||
|
|
"score": pick(item, "final_score", "score", "confidence", "rerank_score"),
|
|||
|
|
"locator": pick(item, "locator"),
|
|||
|
|
"section": pick(item, "section"),
|
|||
|
|
"support_count": pick(item, "support_count"),
|
|||
|
|
"type": pick(item, "type"),
|
|||
|
|
"metadata": metadata,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
# §9 图谱：hop / chain / {nodes,edges,chains}
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
def normalize_hop(hop: dict[str, Any]) -> dict[str, Any]:
|
|||
|
|
"""hop 归一化：
|
|||
|
|
from_name|source → source、to_name|target → target、
|
|||
|
|
from_id|source_canonical_id → source 节点 id、to_id|target_canonical_id → target 节点 id、
|
|||
|
|
relation 原样、source_type|target_type → source/target 节点 type、
|
|||
|
|
paper_id / pmid / pmcid / doi 透传。
|
|||
|
|
"""
|
|||
|
|
if not isinstance(hop, dict):
|
|||
|
|
hop = {}
|
|||
|
|
return {
|
|||
|
|
"source": pick(hop, "from_name", "source"),
|
|||
|
|
"target": pick(hop, "to_name", "target"),
|
|||
|
|
"source_id": pick(hop, "from_id", "source_canonical_id"),
|
|||
|
|
"target_id": pick(hop, "to_id", "target_canonical_id"),
|
|||
|
|
"source_type": pick(hop, "source_type"),
|
|||
|
|
"target_type": pick(hop, "target_type"),
|
|||
|
|
"relation": pick(hop, "relation"),
|
|||
|
|
"assertion_id": pick(hop, "assertion_id"),
|
|||
|
|
"paper_id": pick(hop, "paper_id"),
|
|||
|
|
"pmid": pick(hop, "pmid"),
|
|||
|
|
"pmcid": pick(hop, "pmcid"),
|
|||
|
|
"doi": pick(hop, "doi"),
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def normalize_chain_item(item: dict[str, Any]) -> dict[str, Any]:
|
|||
|
|
"""单个 chain item（真实形态：chain_id / hops[] 等）→ {chain_id, hops}。"""
|
|||
|
|
if not isinstance(item, dict):
|
|||
|
|
item = {}
|
|||
|
|
hops = [normalize_hop(h) for h in item.get("hops", []) if isinstance(h, dict)]
|
|||
|
|
return {"chain_id": pick(item, "chain_id"), "hops": hops}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def extract_chains(raw: dict[str, Any] | None) -> list[dict[str, Any]]:
|
|||
|
|
"""从可能的两类源提取 chain 列表：
|
|||
|
|
- `native_graph_chains`：列表（真实形态，305 条）
|
|||
|
|
- `graph_knowledge_chains`：dict 形态 `{mode, chains[], chain_count, …}`，取 `.chains`
|
|||
|
|
两种形态都要能提取出 hop 列表。
|
|||
|
|
"""
|
|||
|
|
if not isinstance(raw, dict):
|
|||
|
|
return []
|
|||
|
|
chains = pick(raw, "native_graph_chains", "graph_knowledge_chains")
|
|||
|
|
if isinstance(chains, dict):
|
|||
|
|
chains = chains.get("chains")
|
|||
|
|
if not isinstance(chains, list):
|
|||
|
|
return []
|
|||
|
|
return [normalize_chain_item(c) for c in chains]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def normalize_graph(raw: dict[str, Any] | None) -> dict[str, Any]:
|
|||
|
|
"""从 `knowledge_graph` / `graph_edges` / `native_graph_chains`（列表形态）/
|
|||
|
|
`graph_knowledge_chains`（dict 形态）提取并归一化为 {nodes, edges, chains}。
|
|||
|
|
|
|||
|
|
- 节点去重（按 id；id 缺失的 hop 跳过）；label 取 from_name/to_name、type 取
|
|||
|
|
source_type/target_type。
|
|||
|
|
- 边携带 chain_id 与 hop 序号（hop_index 缺失时按列表序补）。
|
|||
|
|
- 链保持 chain_id + hops 列表。
|
|||
|
|
"""
|
|||
|
|
raw = raw if isinstance(raw, dict) else {}
|
|||
|
|
chains = extract_chains(raw)
|
|||
|
|
|
|||
|
|
nodes: dict[str, dict[str, Any]] = {}
|
|||
|
|
edges: list[dict[str, Any]] = []
|
|||
|
|
|
|||
|
|
def add_node(node_id: Any, name: Any, typ: Any) -> None:
|
|||
|
|
if not node_id:
|
|||
|
|
return
|
|||
|
|
key = str(node_id)
|
|||
|
|
if key not in nodes:
|
|||
|
|
nodes[key] = {"id": node_id, "label": name, "type": typ}
|
|||
|
|
|
|||
|
|
for chain in chains:
|
|||
|
|
for idx, hop in enumerate(chain["hops"]):
|
|||
|
|
hop_index = hop.get("hop_index")
|
|||
|
|
if hop_index is None:
|
|||
|
|
hop_index = idx
|
|||
|
|
# 节点 id 优先 canonical id；label 优先名字，退化到 id
|
|||
|
|
src_id = pick(hop, "source_id", "source") or hop.get("source")
|
|||
|
|
tgt_id = pick(hop, "target_id", "target") or hop.get("target")
|
|||
|
|
src_label = hop.get("source") or src_id
|
|||
|
|
tgt_label = hop.get("target") or tgt_id
|
|||
|
|
add_node(src_id, src_label, hop.get("source_type"))
|
|||
|
|
add_node(tgt_id, tgt_label, hop.get("target_type"))
|
|||
|
|
if src_id and tgt_id:
|
|||
|
|
edges.append({
|
|||
|
|
"source": src_id,
|
|||
|
|
"target": tgt_id,
|
|||
|
|
"relation": hop.get("relation"),
|
|||
|
|
"label": hop.get("relation"),
|
|||
|
|
"chain_id": chain.get("chain_id"),
|
|||
|
|
"hop": hop_index,
|
|||
|
|
"paper_id": hop.get("paper_id"),
|
|||
|
|
"pmid": hop.get("pmid"),
|
|||
|
|
"pmcid": hop.get("pmcid"),
|
|||
|
|
"doi": hop.get("doi"),
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
# 兼容显式 {nodes, edges} 形态（knowledge_graph / graph_edges 直接给边列表时）
|
|||
|
|
explicit_nodes = raw.get("nodes")
|
|||
|
|
explicit_edges = raw.get("edges")
|
|||
|
|
if isinstance(explicit_nodes, list):
|
|||
|
|
for n in explicit_nodes:
|
|||
|
|
if isinstance(n, dict) and n.get("id"):
|
|||
|
|
key = str(n["id"])
|
|||
|
|
if key not in nodes:
|
|||
|
|
nodes[key] = {
|
|||
|
|
"id": n["id"],
|
|||
|
|
"label": pick(n, "label"),
|
|||
|
|
"type": pick(n, "type"),
|
|||
|
|
}
|
|||
|
|
if isinstance(explicit_edges, list):
|
|||
|
|
edges.extend(
|
|||
|
|
{
|
|||
|
|
"source": pick(e, "source"),
|
|||
|
|
"target": pick(e, "target"),
|
|||
|
|
"relation": pick(e, "relation"),
|
|||
|
|
"label": pick(e, "label", "relation"),
|
|||
|
|
"chain_id": pick(e, "chain_id"),
|
|||
|
|
"hop": pick(e, "hop"),
|
|||
|
|
"paper_id": pick(e, "paper_id"),
|
|||
|
|
"pmid": pick(e, "pmid"),
|
|||
|
|
"pmcid": pick(e, "pmcid"),
|
|||
|
|
"doi": pick(e, "doi"),
|
|||
|
|
}
|
|||
|
|
for e in explicit_edges
|
|||
|
|
if isinstance(e, dict)
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
"nodes": list(nodes.values()),
|
|||
|
|
"edges": edges,
|
|||
|
|
"chains": chains,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
# §7 归一化 QA 响应（/chat、/chat/stream complete、/search 共用）
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
def normalize_qa_response(raw: dict[str, Any] | None) -> dict[str, Any]:
|
|||
|
|
"""上游 31 键响应 → API.md §7 形态。
|
|||
|
|
|
|||
|
|
别名（真实键优先）：
|
|||
|
|
- answer|response|message → answer
|
|||
|
|
- sources|citations → sources；used_sources|references → used_sources
|
|||
|
|
- query_plan|agentic_plan|retrieval_plan → retrieval_plan（真实键 query_plan）
|
|||
|
|
- retrieval_notes|notes → retrieval_notes
|
|||
|
|
- timings|timing → timing（真实键 timings）
|
|||
|
|
- citation_validation|citation_map → citation_map（真实键 citation_validation）
|
|||
|
|
- 图谱四源 → graph + chains（见 normalize_graph）
|
|||
|
|
- success 透传；raw 保存原始完整响应
|
|||
|
|
"""
|
|||
|
|
raw = raw if isinstance(raw, dict) else {}
|
|||
|
|
|
|||
|
|
sources = [normalize_source(s) for s in (pick(raw, "sources", "citations") or [])]
|
|||
|
|
used_sources = [
|
|||
|
|
normalize_source(s) for s in (pick(raw, "used_sources", "references") or [])
|
|||
|
|
]
|
|||
|
|
graph = normalize_graph(raw)
|
|||
|
|
retrieval_plan = pick(raw, "query_plan", "agentic_plan", "retrieval_plan")
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
"query_id": pick(raw, "query_id"),
|
|||
|
|
"version": pick(raw, "version"),
|
|||
|
|
"mode": pick(raw, "mode", default="chat"),
|
|||
|
|
"answer": pick(raw, "answer", "response", "message", default=""),
|
|||
|
|
"source_count": pick(raw, "source_count", default=len(sources)),
|
|||
|
|
"sources": sources,
|
|||
|
|
"used_sources": used_sources,
|
|||
|
|
"chains": graph["chains"],
|
|||
|
|
"graph": {"nodes": graph["nodes"], "edges": graph["edges"], "chains": graph["chains"]},
|
|||
|
|
"retrieval_plan": retrieval_plan,
|
|||
|
|
"retrieval_notes": pick(raw, "retrieval_notes", "notes"),
|
|||
|
|
"timing": pick(raw, "timings", "timing"),
|
|||
|
|
"citation_map": pick(raw, "citation_validation", "citation_map"),
|
|||
|
|
"success": pick(raw, "success", default=True),
|
|||
|
|
"raw": raw,
|
|||
|
|
}
|