初始提交

This commit is contained in:
potaser
2026-07-10 17:59:59 +08:00
commit 5bb98a937a
660 changed files with 871233 additions and 0 deletions
+30
View File
@@ -0,0 +1,30 @@
from api.main import app
from api.routes import router
from api.schemas import (
CaseCreate,
CaseResponse,
FeedbackRequest,
ProgressUpdate,
QueryRequest,
QueryResponse,
SessionCreate,
SessionResponse,
WebSocketMessage,
)
from api.websocket import ConnectionManager, manager
__all__ = [
"app",
"router",
"CaseCreate",
"CaseResponse",
"FeedbackRequest",
"ProgressUpdate",
"QueryRequest",
"QueryResponse",
"SessionCreate",
"SessionResponse",
"WebSocketMessage",
"ConnectionManager",
"manager",
]
+437
View File
@@ -0,0 +1,437 @@
"""
高级RAG API路由
集成查询扩展、实体链接、引用溯源、动态知识图谱、跨论文融合、Neo4j图算法
"""
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from typing import List, Dict, Optional
router = APIRouter()
# ========== 查询扩展API ==========
class QueryExpansionRequest(BaseModel):
query: str
num_expansions: int = 5
strategy: str = "expansion" # expansion, intent_focused, keyword_combination
class QueryExpansionResponse(BaseModel):
original: str
rewritten: str
alternatives: List[str]
intent: str
confidence: float
keywords: List[str]
synonyms: Dict[str, List[str]]
@router.post("/advanced/query/expand")
async def expand_query_endpoint(request: QueryExpansionRequest):
"""查询扩展端点"""
try:
from src.rag.advanced import QueryExpander, QueryRewriter
expander = QueryExpander()
expansion = expander.expand_query(request.query, request.num_expansions)
return QueryExpansionResponse(
original=request.query,
rewritten=expansion.expanded_queries[0] if expansion.expanded_queries else request.query,
alternatives=expansion.expanded_queries[1:] if len(expansion.expanded_queries) > 1 else [],
intent=expansion.intent.value,
confidence=expansion.intent_confidence,
keywords=expansion.keywords,
synonyms=expansion.synonyms
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"查询扩展失败: {str(e)}")
@router.post("/advanced/query/rewrite")
async def rewrite_query_endpoint(request: QueryExpansionRequest):
"""查询重写端点"""
try:
from src.rag.advanced import QueryRewriter
rewriter = QueryRewriter()
result = rewriter.rewrite(request.query, request.strategy)
return result
except Exception as e:
raise HTTPException(status_code=500, detail=f"查询重写失败: {str(e)}")
# ========== 实体链接API ==========
class EntityLinkingRequest(BaseModel):
text: str
class EntityLinkingResponse(BaseModel):
entities: List[Dict]
linked_count: int
total_count: int
@router.post("/advanced/entity/link")
async def entity_linking_endpoint(request: EntityLinkingRequest):
"""实体链接端点"""
try:
from src.rag.advanced import EntityLinker
linker = EntityLinker()
linked_entities = linker.link_entities(request.text)
entities = []
for le in linked_entities:
entity_info = {
"mention": le.mention.text,
"start_pos": le.mention.start_pos,
"end_pos": le.mention.end_pos,
"context": le.mention.context[:100],
"linked": le.kb_entity is not None,
"link_confidence": le.link_confidence,
"disambiguation_score": le.disambiguation_score,
}
if le.kb_entity:
entity_info.update({
"kb_id": le.kb_entity.entity_id,
"kb_name": le.kb_entity.name,
"kb_type": le.kb_entity.entity_type.value,
"kb_definition": le.kb_entity.definition,
"kb_aliases": le.kb_entity.aliases[:5],
})
entities.append(entity_info)
linked_count = sum(1 for e in entities if e["linked"])
return EntityLinkingResponse(
entities=entities,
linked_count=linked_count,
total_count=len(entities)
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"实体链接失败: {str(e)}")
# ========== 引用溯源API ==========
class CitationRequest(BaseModel):
answer: str
sources: List[Dict]
min_citations: int = 3
class CitationResponse(BaseModel):
answer: str
citations: List[Dict]
confidence: float
coverage_score: float
total_citations: int
@router.post("/advanced/citation/add")
async def add_citations_endpoint(request: CitationRequest):
"""添加引用溯源端点"""
try:
from src.rag.advanced import CitationIntegrator
integrator = CitationIntegrator()
result = integrator.integrate_citations(
request.answer,
request.sources,
request.min_citations
)
return CitationResponse(
answer=result["answer"],
citations=result["citations"],
confidence=result["confidence"],
coverage_score=result["coverage_score"],
total_citations=result["total_citations"]
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"添加引用失败: {str(e)}")
# ========== 动态知识图谱API ==========
class DynamicKGRequest(BaseModel):
sources: List[Dict]
class DynamicKGResponse(BaseModel):
entities: List[Dict]
relations: List[Dict]
total_entities: int
total_relations: int
conflicts: int
resolved: int
needs_review: int
graph_data: Dict
@router.post("/advanced/kg/build")
async def build_dynamic_kg_endpoint(request: DynamicKGRequest):
"""动态构建知识图谱端点"""
try:
from src.rag.advanced import DynamicKGBuilder
builder = DynamicKGBuilder()
result = builder.build_from_search_results(request.sources)
# 获取图谱数据
graph_data = builder.get_graph_data()
return DynamicKGResponse(
entities=result["entities"],
relations=result["relations"],
total_entities=result["total_entities"],
total_relations=result["total_relations"],
conflicts=result["conflicts"],
resolved=result["resolved"],
needs_review=result["needs_review"],
graph_data=graph_data
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"构建知识图谱失败: {str(e)}")
# ========== 跨论文融合API ==========
class CrossPaperFusionRequest(BaseModel):
papers: List[Dict]
class CrossPaperFusionResponse(BaseModel):
entities: List[Dict]
relations: List[Dict]
stats: Dict
@router.post("/advanced/fusion/cross-paper")
async def cross_paper_fusion_endpoint(request: CrossPaperFusionRequest):
"""跨论文知识融合端点"""
try:
from src.rag.advanced import CrossPaperFusion
fusion = CrossPaperFusion()
result = fusion.fuse_knowledge(request.papers)
return CrossPaperFusionResponse(
entities=result["entities"],
relations=result["relations"],
stats=result["stats"]
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"跨论文融合失败: {str(e)}")
@router.post("/advanced/fusion/provenance/{entity_id}")
async def get_provenance_endpoint(entity_id: str, request: CrossPaperFusionRequest):
"""获取实体溯源信息端点"""
try:
from src.rag.advanced import CrossPaperFusion
fusion = CrossPaperFusion()
fusion.fuse_knowledge(request.papers)
provenance = fusion.get_entity_provenance(entity_id)
return provenance
except Exception as e:
raise HTTPException(status_code=500, detail=f"获取溯源信息失败: {str(e)}")
# ========== Neo4j图算法API ==========
class ShortestPathRequest(BaseModel):
start_node: str
end_node: str
relationship_type: Optional[str] = None
class ShortestPathResponse(BaseModel):
status: str
path: Optional[Dict]
message: Optional[str]
@router.post("/advanced/graph/shortest-path")
async def shortest_path_endpoint(request: ShortestPathRequest):
"""最短路径分析端点"""
try:
from src.rag.advanced import GraphAlgorithmAPI
api = GraphAlgorithmAPI()
result = api.shortest_path_endpoint(
request.start_node,
request.end_node,
request.relationship_type
)
return ShortestPathResponse(**result)
except Exception as e:
raise HTTPException(status_code=500, detail=f"最短路径分析失败: {str(e)}")
@router.get("/advanced/graph/communities")
async def community_detection_endpoint():
"""社区发现端点"""
try:
from src.rag.advanced import GraphAlgorithmAPI
api = GraphAlgorithmAPI()
result = api.community_detection_endpoint()
return result
except Exception as e:
raise HTTPException(status_code=500, detail=f"社区发现失败: {str(e)}")
class CentralityRequest(BaseModel):
algorithm: str = "degree" # degree, betweenness, pagerank
top_k: int = 10
@router.post("/advanced/graph/centrality")
async def centrality_analysis_endpoint(request: CentralityRequest):
"""中心性分析端点"""
try:
from src.rag.advanced import GraphAlgorithmAPI
api = GraphAlgorithmAPI()
result = api.centrality_analysis_endpoint(request.algorithm, request.top_k)
return result
except Exception as e:
raise HTTPException(status_code=500, detail=f"中心性分析失败: {str(e)}")
@router.get("/advanced/graph/stats")
async def graph_stats_endpoint():
"""图统计端点"""
try:
from src.rag.advanced import GraphAlgorithmAPI
api = GraphAlgorithmAPI()
result = api.graph_stats_endpoint()
return result
except Exception as e:
raise HTTPException(status_code=500, detail=f"获取图统计失败: {str(e)}")
# ========== 综合RAG API ==========
class AdvancedRAGRequest(BaseModel):
query: str
enable_expansion: bool = True
enable_entity_linking: bool = True
enable_citation: bool = True
enable_dynamic_kg: bool = True
top_k: int = 5
class AdvancedRAGResponse(BaseModel):
query: str
expanded_query: Optional[str]
answer: str
entities: List[Dict]
citations: List[Dict]
knowledge_graph: Optional[Dict]
confidence: float
@router.post("/advanced/rag")
async def advanced_rag_endpoint(request: AdvancedRAGRequest):
"""高级RAG综合端点"""
try:
from src.rag.advanced import (
QueryExpander,
EntityLinker,
CitationIntegrator,
DynamicKGBuilder
)
# 1. 查询扩展
expanded_query = request.query
if request.enable_expansion:
expander = QueryExpander()
expansion = expander.expand_query(request.query, num_expansions=3)
expanded_query = expansion.expanded_queries[0] if expansion.expanded_queries else request.query
# 2. 实体链接
entities = []
if request.enable_entity_linking:
linker = EntityLinker()
linked = linker.link_entities(request.query)
entities = [
{
"mention": le.mention.text,
"linked": le.kb_entity is not None,
"kb_name": le.kb_entity.name if le.kb_entity else None,
"kb_type": le.kb_entity.entity_type.value if le.kb_entity else None,
"confidence": le.link_confidence
}
for le in linked
]
# 3. 模拟检索结果(实际应该调用检索器)
mock_sources = [
{
"doc_id": f"PMID{10000000 + i}",
"content": f"Sample content for {request.query}",
"score": 0.9 - i * 0.1,
"chunk_id": f"chunk_{i}",
"metadata": {"title": f"Paper {i}", "year": "2023"}
}
for i in range(request.top_k)
]
# 4. 生成答案
answer = f"基于检索到的{len(mock_sources)}条文献,关于'{request.query}'的分析如下..."
# 5. 添加引用
citations = []
if request.enable_citation:
integrator = CitationIntegrator()
citation_result = integrator.integrate_citations(answer, mock_sources, min_citations=3)
answer = citation_result["answer"]
citations = citation_result["citations"]
# 6. 构建动态知识图谱
kg_data = None
if request.enable_dynamic_kg:
builder = DynamicKGBuilder()
kg_result = builder.build_from_search_results(mock_sources)
kg_data = builder.get_graph_data()
return AdvancedRAGResponse(
query=request.query,
expanded_query=expanded_query if request.enable_expansion else None,
answer=answer,
entities=entities,
citations=citations,
knowledge_graph=kg_data,
confidence=0.85
)
except Exception as e:
import traceback
traceback.print_exc()
raise HTTPException(status_code=500, detail=f"高级RAG处理失败: {str(e)}")
+35
View File
@@ -0,0 +1,35 @@
"""Maintained FastAPI entrypoint for the PlantAI-rag cleanup baseline."""
from __future__ import annotations
from fastapi import FastAPI
from api.kg_routes import router as kg_router
from api.methodology_routes import router as methodology_router
def create_app() -> FastAPI:
app = FastAPI(
title="PlantAI-rag API",
description="Maintained API surface for the PlantAI-rag open-source cleanup baseline.",
version="0.1.0",
)
app.include_router(kg_router, prefix="/api/v1")
app.include_router(methodology_router, prefix="/api/v1")
@app.get("/")
async def root() -> dict[str, str]:
return {
"name": "PlantAI-rag",
"status": "ready",
"runtime_surface": "maintained_baseline",
}
@app.get("/health")
async def health() -> dict[str, str]:
return {"status": "healthy"}
return app
app = create_app()
+71
View File
@@ -0,0 +1,71 @@
"""Case-specific deterministic QA routes.
These endpoints expose validated reproduction-case packages before they are
merged into the general RAG chat route. They do not call an LLM.
"""
from functools import lru_cache
from pathlib import Path
from typing import Any
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel, Field
from src.rag.unified.case_qa import CasePackageQAEngine
PROJECT_ROOT = Path(__file__).resolve().parents[1]
FT_SPL_CASE_PACKAGE = PROJECT_ROOT / "data" / "ft_spl_reproduction_case_package.json"
router = APIRouter()
class CaseQARequest(BaseModel):
query: str = Field(..., min_length=1)
class CaseQAResponse(BaseModel):
query: str
intent: str
answer: str
citations: list[str]
facts: dict[str, Any]
class CaseStatusResponse(BaseModel):
case_id: str
package_path: str
package_exists: bool
supported: bool
@lru_cache(maxsize=1)
def get_ft_spl_engine() -> CasePackageQAEngine:
if not FT_SPL_CASE_PACKAGE.exists():
raise FileNotFoundError(str(FT_SPL_CASE_PACKAGE))
return CasePackageQAEngine(FT_SPL_CASE_PACKAGE)
@router.get("/ft-spl/status", response_model=CaseStatusResponse)
async def ft_spl_case_status():
return CaseStatusResponse(
case_id="ft_spl_reproduction_job_1777615271527",
package_path=str(FT_SPL_CASE_PACKAGE),
package_exists=FT_SPL_CASE_PACKAGE.exists(),
supported=FT_SPL_CASE_PACKAGE.exists(),
)
@router.post("/ft-spl/qa", response_model=CaseQAResponse)
async def ft_spl_case_qa(request: CaseQARequest):
try:
response = get_ft_spl_engine().answer(request.query)
except FileNotFoundError as exc:
raise HTTPException(status_code=503, detail=f"Case package missing: {exc}") from exc
return CaseQAResponse(
query=response.query,
intent=response.intent,
answer=response.answer,
citations=response.citations,
facts=response.facts,
)
+935
View File
@@ -0,0 +1,935 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Hybrid (Qdrant + Neo4j) QA routes for the V15 SRT knowledge base.
This route uses V15HybridRetriever for vector (Qdrant) and graph (Neo4j)
retrieval. The old SQLite-only path is preserved in the .bak file.
"""
from __future__ import annotations
import json
import os
import re
import asyncio
import urllib.request
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from fastapi import APIRouter, HTTPException, Query
from pydantic import BaseModel, Field
from src.rag.v15_hybrid_retriever import V15HybridRetriever
PROJECT_ROOT = Path(__file__).resolve().parent.parent
DEFAULT_KB_DIR = PROJECT_ROOT / "data" / "knowledge_base" / "v15_srt_kb_20260525"
router = APIRouter(prefix="/kb-qa", tags=["V15 SRT KB QA"])
_llm_manager = None
_result_chunk_cache: Dict[str, List[Dict[str, str]]] = {}
STOPWORDS = {
"what",
"which",
"where",
"when",
"who",
"how",
"does",
"did",
"are",
"was",
"were",
"the",
"and",
"for",
"with",
"from",
"into",
"about",
"that",
"this",
"these",
"those",
"gene",
"genes",
"protein",
"proteins",
"family",
"tell",
"me",
"is",
"of",
"in",
"to",
"a",
"an",
}
LAYER_FILTERS = {
"current_findings": (
"r.output_layer = 'main_kg' "
"AND COALESCE(r.claim_scope, '') NOT IN ('background_context', 'cited_prior_work') "
"AND COALESCE(r.candidate_status, '') != 'candidate'"
),
"main_kg": (
"r.output_layer = 'main_kg' "
"AND COALESCE(r.claim_scope, '') NOT IN ('background_context', 'cited_prior_work') "
"AND COALESCE(r.candidate_status, '') != 'candidate'"
),
"methods": "r.output_layer = 'method_relations' AND COALESCE(r.candidate_status, '') != 'candidate'",
"method_relations": "r.output_layer = 'method_relations' AND COALESCE(r.candidate_status, '') != 'candidate'",
"datasets": "r.output_layer = 'dataset_metadata' AND COALESCE(r.candidate_status, '') != 'candidate'",
"dataset_metadata": "r.output_layer = 'dataset_metadata' AND COALESCE(r.candidate_status, '') != 'candidate'",
"prior_work": "r.output_layer = 'background_relations'",
"background_relations": "r.output_layer = 'background_relations'",
"candidate": "(r.output_layer = 'candidate_review' OR r.candidate_status = 'candidate')",
"candidate_review": "(r.output_layer = 'candidate_review' OR r.candidate_status = 'candidate')",
"all": "1 = 1",
}
RELATION_FAMILY_ALIASES = {
"baseline_expression": [
"expressed_in",
"highly_expressed_during",
"only_expressed_during",
"differentially_expressed_under",
"upregulated_in",
"downregulated_in",
"downregulated_during",
"peak_at",
"not_induced_under",
],
"up_expression": ["upregulated_in", "induced_in", "increased_in"],
"down_expression": ["downregulated_in", "downregulated_during", "repressed_in", "decreased_in"],
"differential_expression": ["differentially_expressed_under"],
"expression_peak": ["peak_at", "reached_maximum_at"],
"not_induced": ["not_induced_under"],
"identification": ["identified_in", "identified_from"],
"member_count": ["has_member_count", "contains_gene_family_members"],
"classification": ["member_of", "classified_into", "divided_into", "separated_into", "categorized_into"],
"domain": ["contains_domain", "has_domain"],
"cis_element": ["contains_cis_element", "has_cis_element", "has_cis_elements"],
"motif": ["contains_motif"],
"localization": ["localizes_to"],
"interaction": ["interacts_with"],
"binding": ["binds_to"],
"targeting": ["targets"],
"homology": ["clustered_with", "homologous_to"],
"synteny": ["syntenic_with", "has_collinear_pairs_with", "collinear_with"],
"regulation": ["regulates", "positively_regulates", "negatively_regulates"],
"effect": ["promotes", "enhances", "overexpression_enhances", "overexpression_increases", "overexpression_improves"],
"function": ["plays_role_in", "play_roles_in", "involved_in", "participates_in"],
"identity": ["is_a"],
}
class KBQARequest(BaseModel):
question: str = Field(..., description="Question to answer from the V15 SRT KB")
top_k: int = Field(8, ge=1, le=50)
layer: str = Field(
"auto",
description=(
"auto/current_findings/methods/datasets/prior_work/"
"candidate/all"
),
)
include_evidence: bool = True
include_candidate_fallback: bool = False
use_llm: bool = True
llm_provider: Optional[str] = None
llm_model: Optional[str] = None
anchor_entity: Optional[str] = Field(
None,
description="Optional structured retrieval hint, e.g. GmP450 genes",
)
relation_family: Optional[str] = Field(
None,
description="Optional question/relation family hint, e.g. baseline_expression",
)
preferred_relations: List[str] = Field(
default_factory=list,
description="Optional exact relation hints for structured retrieval",
)
preferred_papers: List[str] = Field(
default_factory=list,
description="Optional paper_id hints for benchmark/debug QA; used as boost, not as a hard filter",
)
class KBQASource(BaseModel):
relation_id: str
paper_id: str = ""
pubmed_id: str = ""
p_source: str = ""
source_result_path: str = ""
chunk_id: str = ""
chunk_hint: str = ""
evidence_key: str = ""
source: str
source_type: str = ""
relation: str
target: str
target_type: str = ""
evidence: str = ""
section: str = ""
source_label: str = ""
claim_scope: str = ""
output_layer: str = ""
candidate_status: str = ""
confidence: str = ""
score: float = 0.0
class KBQAResponse(BaseModel):
question: str
answer: str
selected_layer: str
layer_reason: str
source_count: int
paper_source_count: int = 0
sources: List[KBQASource]
kb_path: str
answer_mode: str = "extractive"
llm_used: bool = False
notes: List[str] = []
def _kb_dir() -> Path:
configured = os.environ.get("V15_SRT_KB_DIR")
return Path(configured).expanduser() if configured else DEFAULT_KB_DIR
def get_llm_manager():
"""Lazy-load the existing project LLM manager for answer synthesis."""
global _llm_manager
if _llm_manager is not None:
return _llm_manager
try:
import yaml
from src.llm.manager import LLMManager
config_path = PROJECT_ROOT / "config" / "llm.yaml"
if config_path.exists():
config = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
else:
config = {"providers": {}, "default": {}}
# KB-QA must be able to use DeepSeek even when the shared llm.yaml was
# created before the DeepSeek provider was added to the project.
config.setdefault("providers", {}).setdefault(
"deepseek",
{
"name": "DeepSeek",
"api_base": os.environ.get("DEEPSEEK_BASE_URL", "https://api.deepseek.com"),
"models": [
{"name": "deepseek-v4-flash", "context_length": 128000},
{"name": "deepseek-v4-pro", "context_length": 128000},
{"name": "deepseek-chat", "context_length": 64000},
],
"env_key": "DEEPSEEK_API_KEY",
},
)
provider = os.environ.get("V15_KB_QA_LLM_PROVIDER")
model = os.environ.get("V15_KB_QA_LLM_MODEL")
if provider:
config.setdefault("default", {})["provider"] = provider
if model:
config.setdefault("default", {})["model"] = model
if os.environ.get("V15_KB_QA_LLM_MAX_TOKENS"):
config.setdefault("default", {})["max_tokens"] = int(os.environ["V15_KB_QA_LLM_MAX_TOKENS"])
if os.environ.get("V15_KB_QA_LLM_TEMPERATURE"):
config.setdefault("default", {})["temperature"] = float(os.environ["V15_KB_QA_LLM_TEMPERATURE"])
_llm_manager = LLMManager(config)
return _llm_manager
except Exception as exc:
print(f"[KB-QA] LLM manager init failed: {exc}")
return None
def _classify_layer(question: str, requested_layer: str) -> Tuple[str, str]:
requested = (requested_layer or "auto").strip().lower()
if requested != "auto":
if requested not in LAYER_FILTERS:
raise HTTPException(
status_code=400,
detail=f"Unsupported layer '{requested_layer}'. "
f"Use one of: auto, {', '.join(sorted(LAYER_FILTERS))}",
)
return requested, "user_selected"
q = question.lower()
if re.search(r"\b(tool|software|method|pipeline|database|using|used|analy[sz]ed by)\b", q) or any(
word in question for word in ["工具", "软件", "方法", "流程", "用了什么", "使用了什么"]
):
return "methods", "auto_detected_method_question"
if re.search(r"\b(accession|geo|sra|prjna|srp|gse|rna-seq|rnaseq|doi)\b", q) or any(
word in question for word in ["编号", "登录号", "测序数据", "转录组数据"]
):
return "datasets", "auto_detected_dataset_question"
if re.search(r"\b(prior|previous|background|reported|known|literature|cited)\b", q) or any(
word in question for word in ["前人", "背景", "已有研究", "文献提到", "报道过"]
):
return "prior_work", "auto_detected_prior_work_question"
if any(word in question for word in ["候选", "不确定", "可能相关"]):
return "candidate", "auto_detected_candidate_question"
return "current_findings", "default_current_findings"
def _query_tokens(question: str) -> List[str]:
raw_tokens = re.findall(r"[A-Za-z][A-Za-z0-9_.:/+-]*|\d+[A-Za-z0-9_.:/+-]*", question)
tokens: List[str] = []
for token in raw_tokens:
cleaned = re.sub(r"[^A-Za-z0-9_]+", " ", token).strip()
for part in cleaned.split():
lower = part.lower()
if len(part) < 2 or lower in STOPWORDS:
continue
tokens.append(part)
# Preserve order while removing duplicates.
seen = set()
unique = []
for token in tokens:
key = token.lower()
if key not in seen:
seen.add(key)
unique.append(token)
return unique[:12]
def _focus_entity(question: str) -> str:
"""Extract a focus entity from short definition-style questions."""
text = question.strip()
patterns = [
r"^\s*what\s+(?:is|are)\s+(.+?)[??。.!]*\s*$",
r"^\s*define\s+(.+?)[??。.!]*\s*$",
r"^\s*(.+?)\s*(?:是什么|是啥|指什么)[??。.!]*\s*$",
]
for pattern in patterns:
match = re.search(pattern, text, flags=re.IGNORECASE)
if not match:
continue
focus = match.group(1).strip()
focus = re.sub(r"^(the|a|an)\s+", "", focus, flags=re.IGNORECASE)
focus = focus.strip(" \t\r\n\"'`.,;:()[]{}")
if 2 <= len(focus) <= 80:
return focus
return ""
def _clean_hint_text(value: str) -> str:
value = re.sub(r"\s+", " ", value or "").strip()
value = value.strip(" \t\r\n\"'`.,;:()[]{}??。")
return value[:120]
def _infer_anchor_entity(question: str) -> str:
"""Infer a likely SRT source entity from a natural-language question."""
patterns = [
r"问题[::]\s*(.+?)\s*(?:的|之)\s*表达",
r"问题[::]\s*(.+?)\s*(?:的|之)\s*鉴定",
r"问题[::]\s*(.+?)\s*(?:报道了多少|有多少)",
r"expression of\s+(.+?)\s+(?:was|were|reported|under|in|\\?)",
r"identifying\s+(.+?)(?:\\?|$)",
r"identification of\s+(.+?)(?:\\?|$)",
r"reported for\s+(.+?)(?:\\?|$)",
r"reported in\s+(.+?)(?:\\?|$)",
r"reported to interact with\s+(.+?)(?:\\?|$)",
r"was\s+(.+?)\s+upregulated",
r"was\s+(.+?)\s+downregulated",
]
for pattern in patterns:
match = re.search(pattern, question, flags=re.IGNORECASE)
if not match:
continue
candidate = _clean_hint_text(match.group(1))
candidate = re.sub(r"^(the|a|an)\s+", "", candidate, flags=re.IGNORECASE)
candidate = re.split(r"\n|English question:|回答要求[::]", candidate)[0].strip()
if 2 <= len(candidate) <= 100 and not re.search(r"\b(where|which|what|how|under)\b", candidate, re.I):
return candidate
focus = _focus_entity(question)
return focus
def _entity_variants(entity: str) -> List[str]:
entity = _clean_hint_text(entity)
if not entity:
return []
variants = {entity}
base = re.sub(
r"\s+(gene|genes|protein|proteins|transcripts?|family|gene family|protein family)$",
"",
entity,
flags=re.IGNORECASE,
).strip()
if base and len(base) >= 2:
variants.add(base)
if re.search(r"[A-Za-z0-9]$", base):
variants.add(base + "s")
variants.add(base + " genes")
variants.add(base + " proteins")
if entity.endswith("s") and len(entity) > 3:
variants.add(entity[:-1])
return [v for v in variants if len(v) >= 2][:8]
def _infer_relation_family(question: str) -> str:
q = question.lower()
checks = [
("baseline_expression", [r"表达", r"expression", r"expressed", r"tissue", r"condition"]),
("up_expression", [r"上调", r"upregulated", r"induced"]),
("down_expression", [r"下调", r"downregulated", r"repressed"]),
("not_induced", [r"未被诱导", r"not induced"]),
("identification", [r"鉴定", r"identification", r"identifying", r"identified"]),
("member_count", [r"多少个成员", r"how many members", r"member count"]),
("cis_element", [r"顺式", r"cis[- ]?acting", r"promoter", r"启动子"]),
("domain", [r"结构域", r"domain"]),
("motif", [r"motif"]),
("localization", [r"定位", r"localiz"]),
("interaction", [r"互作", r"interact"]),
("binding", [r"结合", r"bind"]),
("synteny", [r"共线", r"synten", r"collinear"]),
("regulation", [r"调控", r"regulat"]),
("function", [r"功能", r"function", r"process", r"involved", r"role"]),
("classification", [r"亚家族", r"类别", r"class", r"group", r"subfamily", r"belong"]),
]
for family, patterns in checks:
if any(re.search(pattern, q) for pattern in patterns):
return family
return ""
def _resolve_preferred_relations(relation_family: str, preferred_relations: List[str]) -> List[str]:
resolved: List[str] = []
for rel in preferred_relations or []:
rel = _clean_hint_text(rel)
if rel and re.fullmatch(r"[A-Za-z0-9_]+", rel):
resolved.append(rel)
family_key = (relation_family or "").strip()
resolved.extend(RELATION_FAMILY_ALIASES.get(family_key, []))
seen = set()
unique = []
for rel in resolved:
key = rel.lower()
if key not in seen:
seen.add(key)
unique.append(rel)
return unique[:20]
def _build_query_plan(
question: str,
anchor_entity: Optional[str] = None,
relation_family: Optional[str] = None,
preferred_relations: Optional[List[str]] = None,
preferred_papers: Optional[List[str]] = None,
) -> Dict[str, Any]:
anchor = _clean_hint_text(anchor_entity or "") or _infer_anchor_entity(question)
family = _clean_hint_text(relation_family or "") or _infer_relation_family(question)
relations = _resolve_preferred_relations(family, preferred_relations or [])
papers = [_clean_hint_text(paper) for paper in (preferred_papers or []) if _clean_hint_text(paper)]
return {
"anchor_entity": anchor,
"entity_variants": _entity_variants(anchor),
"relation_family": family,
"preferred_relations": relations,
"preferred_papers": papers[:20],
}
def _norm_space(text: str) -> str:
return re.sub(r"\s+", " ", text or "").strip()
def _evidence_key(text: str) -> str:
text = _norm_space(text).lower()
text = re.sub(r"[^a-z0-9\u4e00-\u9fff]+", " ", text)
return _norm_space(text)
def _paper_group_key(row: Dict[str, Any]) -> str:
return (
row.get("pubmed_id")
or row.get("paper_id")
or row.get("source_result_path")
or row.get("relation_id")
or ""
)
def _resolve_result_path(source_result_path: str) -> Optional[Path]:
if not source_result_path:
return None
path = Path(source_result_path)
if not path.is_absolute():
path = PROJECT_ROOT / path
if path.is_dir():
path = path / "result.json"
if path.name != "result.json" and path.suffix != ".json":
candidate = path / "result.json"
if candidate.exists():
path = candidate
return path if path.exists() else None
def _load_result_chunks(source_result_path: str) -> List[Dict[str, str]]:
path = _resolve_result_path(source_result_path)
if path is None:
return []
key = str(path)
if key in _result_chunk_cache:
return _result_chunk_cache[key]
chunks: List[Dict[str, str]] = []
try:
data = json.loads(path.read_text(encoding="utf-8"))
for idx, chunk in enumerate(data.get("chunks") or []):
text = _norm_space(str(chunk.get("text") or chunk.get("content") or ""))
if not text:
continue
chunk_id = str(chunk.get("chunk_id") or chunk.get("id") or f"chunk_{idx:04d}")
chunks.append(
{
"chunk_id": chunk_id,
"section": str(chunk.get("section") or chunk.get("section_title") or ""),
"text": text,
"norm_text": _evidence_key(text),
}
)
except Exception:
chunks = []
_result_chunk_cache[key] = chunks
return chunks
def _infer_chunk_id(row: Dict[str, Any]) -> str:
evidence_norm = _evidence_key(row.get("evidence", ""))
if not evidence_norm:
return ""
chunks = _load_result_chunks(row.get("source_result_path", ""))
if not chunks:
return ""
anchors = [evidence_norm]
if len(evidence_norm) > 260:
anchors.extend([evidence_norm[:260], evidence_norm[:160], evidence_norm[-200:]])
for anchor in anchors:
if len(anchor) < 40:
continue
for chunk in chunks:
if anchor in chunk.get("norm_text", ""):
return chunk.get("chunk_id", "")
return ""
def _fallback_chunk_hint(row: Dict[str, Any]) -> str:
parts = []
if row.get("section"):
parts.append(f"section={row.get('section')}")
if row.get("p_source"):
parts.append(f"p_source={row.get('p_source')}")
if row.get("source_label"):
parts.append(f"source_label={row.get('source_label')}")
if not parts and row.get("relation_id"):
parts.append(f"relation={row.get('relation_id')}")
return " | ".join(parts)
def _decorate_source_rows(rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
for row in rows:
row["evidence_key"] = _evidence_key(row.get("evidence", ""))
chunk_id = _infer_chunk_id(row)
row["chunk_id"] = chunk_id
row["chunk_hint"] = chunk_id or _fallback_chunk_hint(row)
return rows
def _shorten(text: str, limit: int = 420) -> str:
text = re.sub(r"\s+", " ", text or "").strip()
if len(text) <= limit:
return text
return text[: limit - 3].rstrip() + "..."
def _format_answer(question: str, layer: str, rows: List[Dict[str, Any]], include_evidence: bool) -> str:
if not rows:
return (
"未在当前知识库层中检索到可用证据。可以尝试切换 layer=all、"
"layer=prior_work、layer=methods 或 layer=datasets。"
)
layer_label = {
"current_findings": "本文主要发现(main_kg)",
"main_kg": "本文主要发现(main_kg)",
"methods": "方法/工具(method_relations)",
"method_relations": "方法/工具(method_relations)",
"datasets": "数据集(dataset_metadata)",
"dataset_metadata": "数据集(dataset_metadata)",
"prior_work": "前人研究/背景(background_relations)",
"background_relations": "前人研究/背景(background_relations)",
"candidate": "候选关系(candidate_review)",
"candidate_review": "候选关系(candidate_review)",
"all": "全部非拒绝关系",
}.get(layer, layer)
paper_count = len({_paper_group_key(row) for row in rows})
lines = [
f"基于 {layer_label} 检索到 {paper_count} 篇文献中的 {len(rows)} 条去重证据:"
]
for idx, row in enumerate(rows[:8], start=1):
pmid = row.get("pubmed_id") or "NA"
chunk = row.get("chunk_id") or row.get("chunk_hint") or row.get("section") or "NA"
lines.append(
f"{idx}. PMID: {pmid} | paper: {row.get('paper_id', '') or 'NA'} | chunk: {chunk}"
)
if include_evidence and row.get("evidence"):
lines.append(f" 证据:{_shorten(row['evidence'])}")
lines.append("回答只基于当前 V15 SRT 知识库;需要原文级核查时请查看返回的 sources。")
return "\n".join(lines)
def _build_llm_context(rows: List[Dict[str, Any]], max_rows: Optional[int] = None) -> str:
if max_rows is None:
max_rows = max(int(os.environ.get("V15_KB_QA_LLM_CONTEXT_EVIDENCE_ROWS", "18")), 1)
parts = []
for idx, row in enumerate(rows[:max_rows], start=1):
evidence = re.sub(r"\s+", " ", row.get("evidence", "") or "").strip()
if len(evidence) > 900:
evidence = evidence[:897].rstrip() + "..."
parts.append(
"\n".join(
[
f"[S{idx}]",
f"PMID: {row.get('pubmed_id', '')}",
f"paper_id: {row.get('paper_id', '')}",
f"chunk: {row.get('chunk_id') or row.get('chunk_hint') or row.get('section', '')}",
f"layer: {row.get('output_layer', '')}",
f"claim_scope: {row.get('claim_scope', '')}",
f"SRT: {row.get('source', '')} --{row.get('relation', '')}--> {row.get('target', '')}",
f"source_type: {row.get('source_type', '')}; target_type: {row.get('target_type', '')}",
f"evidence: {evidence}",
]
)
)
return "\n\n".join(parts)
def _build_answer_prompt(question: str, layer: str, rows: List[Dict[str, Any]]) -> str:
layer_instruction = {
"current_findings": "These are current-paper biological findings.",
"main_kg": "These are current-paper biological findings.",
"methods": "These are method/tool/software relations. Answer method questions only.",
"method_relations": "These are method/tool/software relations. Answer method questions only.",
"datasets": "These are dataset/accession/provenance relations.",
"dataset_metadata": "These are dataset/accession/provenance relations.",
"prior_work": "These are prior-work or background relations; explicitly say they are prior/background if used.",
"background_relations": "These are prior-work or background relations; explicitly say they are prior/background if used.",
"candidate": "These are candidate/uncertain relations; explicitly warn that they are candidates.",
"candidate_review": "These are candidate/uncertain relations; explicitly warn that they are candidates.",
"all": "These relations may come from different layers. Do not mix layer meanings.",
}.get(layer, "Use the layer metadata to qualify the answer.")
return f"""You are answering from an audited plant gene-family SRT knowledge base.
Question:
{question}
Selected layer:
{layer}
Layer instruction:
{layer_instruction}
Retrieved evidence records:
{_build_llm_context(rows)}
Rules:
1. Answer only from the retrieved SRT evidence above. Do not add outside knowledge.
2. If the retrieved SRTs do not answer the question, say so clearly.
3. Cite supporting rows with [S1], [S2], etc.
4. Keep current findings, prior/background work, method/tool facts, datasets, and candidate relations conceptually separate.
5. If evidence is limited or conflicting, say "the KB evidence is limited" and explain the limitation.
6. Answer in the same language as the user.s question. If the question contains non-English characters (e.g. Chinese), respond in that language primarily. If the question is purely English, respond in English.
7. Be concise but conversational; do not just repeat a raw SRT table.
Answer:
"""
async def _generate_llm_answer(
question: str,
layer: str,
rows: List[Dict[str, Any]],
provider: Optional[str] = None,
model: Optional[str] = None,
) -> str:
if not rows:
return ""
llm_mgr = get_llm_manager()
if llm_mgr is None:
raise RuntimeError("LLM manager is not available")
prompt = _build_answer_prompt(question, layer, rows)
from src.llm.base import LLMMessage
provider = provider or os.environ.get("V15_KB_QA_LLM_PROVIDER")
model = model or os.environ.get("V15_KB_QA_LLM_MODEL")
max_tokens = int(os.environ.get("V15_KB_QA_LLM_MAX_TOKENS", "1200"))
temperature = float(os.environ.get("V15_KB_QA_LLM_TEMPERATURE", "0.2"))
response = await llm_mgr.acall(
[LLMMessage(role="user", content=prompt)],
provider=provider,
model=model,
max_tokens=max_tokens,
temperature=temperature,
)
return response.content if hasattr(response, "content") else str(response)
def _strip_think_tags(text: str) -> str:
return re.sub(r"<think>.*?</think>", "", text or "", flags=re.DOTALL).strip()
def _ollama_chat_sync(prompt: str, model: str) -> str:
raise RuntimeError("Local Ollama is disabled for KB-QA answer generation")
host = os.environ.get("OLLAMA_HOST", "http://localhost:11434").rstrip("/")
payload = {
"model": model,
"messages": [{"role": "user", "content": prompt}],
"stream": False,
"temperature": float(os.environ.get("V15_KB_QA_LLM_TEMPERATURE", "0.2")),
"options": {"num_predict": int(os.environ.get("V15_KB_QA_LLM_MAX_TOKENS", "1200"))},
}
req = urllib.request.Request(
f"{host}/api/chat",
data=json.dumps(payload).encode("utf-8"),
headers={"Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(req, timeout=180) as resp:
data = json.loads(resp.read().decode("utf-8"))
return _strip_think_tags(data.get("message", {}).get("content", ""))
async def _generate_llm_answer_with_fallback(
question: str,
layer: str,
rows: List[Dict[str, Any]],
provider: Optional[str] = None,
model: Optional[str] = None,
) -> Tuple[str, List[str], str]:
"""Generate an answer with the configured remote LLM only.
Local Ollama fallback is intentionally disabled for QA evaluation. If the
configured remote/API model fails, the caller will return an extractive
answer instead of retrying with a local small model.
"""
notes: List[str] = []
provider = provider or os.environ.get("V15_KB_QA_LLM_PROVIDER")
model = model or os.environ.get("V15_KB_QA_LLM_MODEL")
if (provider or "").lower() == "ollama":
raise RuntimeError("Local Ollama provider is disabled for KB-QA answer generation")
try:
answer = await _generate_llm_answer(question, layer, rows, provider=provider, model=model)
return answer, notes, f"{provider or 'default'}:{model or 'default'}"
except Exception as exc:
notes.append(f"Primary LLM failed: {exc}")
raise
# ---------------------------------------------------------------------------
# Hybrid → KBQASource adapter
# ---------------------------------------------------------------------------
def _hybrid_row_to_source(row: dict) -> KBQASource:
"""Map a V15HybridRetriever result row to KBQASource, filling defaults."""
return KBQASource(
relation_id=row.get("relation_id", "") or "graph_{}_{}_{}".format(
row.get("source", ""), row.get("relation", ""), row.get("target", "")
),
paper_id=row.get("paper_id", ""),
pubmed_id=str(row.get("pubmed_id", "") or ""),
p_source=row.get("p_source", ""),
source_result_path=row.get("source_result_path", ""),
chunk_id=row.get("chunk_id", ""),
chunk_hint=row.get("chunk_hint", ""),
evidence_key=row.get("evidence_key", ""),
source=row.get("source", ""),
source_type=row.get("source_type", ""),
relation=row.get("relation", ""),
target=row.get("target", ""),
target_type=row.get("target_type", ""),
evidence=row.get("evidence", ""),
section=row.get("section", ""),
source_label=row.get("source_label", ""),
claim_scope=row.get("claim_scope", ""),
output_layer=row.get("output_layer", ""),
candidate_status=row.get("candidate_status", ""),
confidence=str(row.get("confidence", "")) if row.get("confidence") is not None else "",
score=float(row.get("_vector_score", 0.0) or 0.0),
)
def _source_model(row: Dict[str, Any]) -> KBQASource:
return KBQASource(
relation_id=row.get("relation_id", ""),
paper_id=row.get("paper_id", ""),
pubmed_id=row.get("pubmed_id", ""),
p_source=row.get("p_source", ""),
source_result_path=row.get("source_result_path", ""),
chunk_id=row.get("chunk_id", ""),
chunk_hint=row.get("chunk_hint", ""),
evidence_key=row.get("evidence_key", ""),
source=row.get("source", ""),
source_type=row.get("source_type", ""),
relation=row.get("relation", ""),
target=row.get("target", ""),
target_type=row.get("target_type", ""),
evidence=row.get("evidence", ""),
section=row.get("section", ""),
source_label=row.get("source_label", ""),
claim_scope=row.get("claim_scope", ""),
output_layer=row.get("output_layer", ""),
candidate_status=row.get("candidate_status", ""),
confidence=row.get("confidence", ""),
score=float(row.get("score", 0.0) or 0.0),
)
@router.get("/info")
async def kb_info() -> Dict[str, Any]:
"""Return hybrid (Qdrant + Neo4j) KB status."""
return {
"backend": "hybrid",
"qdrant_collection": os.environ.get("V15_QDRANT_COLLECTION", "v15_srt_bge_20260527"),
"neo4j_uri": os.environ.get("NEO4J_URI", "bolt://localhost:7687"),
"recommended_default": "Use /api/v1/kb-qa/chat with layer=auto for QA.",
"note": "SQLite-only path is preserved in kb_qa_routes.py.bak.20260528",
}
@router.post("/chat", response_model=KBQAResponse)
async def kb_qa_chat(request: KBQARequest) -> KBQAResponse:
"""Retrieve SRT evidence via hybrid (Qdrant + Neo4j) retrieval and format an auditable answer."""
question = request.question.strip()
if not question:
raise HTTPException(status_code=400, detail="question is required")
selected_layer, reason = _classify_layer(question, request.layer)
answer_layer = selected_layer
query_plan = _build_query_plan(
question,
anchor_entity=request.anchor_entity,
relation_family=request.relation_family,
preferred_relations=request.preferred_relations,
preferred_papers=request.preferred_papers,
)
# Hybrid retrieval via Qdrant (vector) + Neo4j (graph), no SQLite
hybrid = V15HybridRetriever()
anchor = query_plan.get("anchor_entity", "")
try:
result = hybrid.retrieve(
question,
top_k=request.top_k,
layer=selected_layer,
use_sqlite=False,
use_vector=True,
use_graph=True,
include_candidate=request.include_candidate_fallback,
anchor_entity=anchor or None,
)
raw_rows = result["retrieved_evidence"]
notes = list(result["notes"])
graph_paths = result.get("graph_paths", [])
if not raw_rows and request.layer == "auto" and selected_layer != "all":
fallback = hybrid.retrieve(
question,
top_k=request.top_k,
layer="all",
use_sqlite=False,
use_vector=True,
use_graph=True,
include_candidate=False,
anchor_entity=anchor or None,
)
if fallback["retrieved_evidence"]:
raw_rows = fallback["retrieved_evidence"]
notes.extend(fallback["notes"])
answer_layer = "all"
notes.append("Auto layer returned no result; answered from all layers.")
graph_paths = fallback.get("graph_paths", [])
finally:
hybrid.close()
# Decorate rows with evidence_key/chunk_hint (using existing helpers)
rows = _decorate_source_rows(raw_rows)
answer_mode = "extractive"
llm_used = False
answer = ""
if request.use_llm and rows:
try:
answer, llm_notes, llm_backend = await _generate_llm_answer_with_fallback(
question,
answer_layer,
rows,
provider=request.llm_provider,
model=request.llm_model,
)
notes.extend(llm_notes)
if answer.strip():
answer_mode = f"llm_synthesis:{llm_backend}"
llm_used = True
except Exception as exc:
notes.append(f"LLM answer synthesis failed; returned extractive answer. Error: {exc}")
print(f"[KB-QA] LLM answer synthesis failed: {exc}")
if not answer.strip():
answer = _format_answer(question, answer_layer, rows, request.include_evidence)
return KBQAResponse(
question=question,
answer=answer,
selected_layer=answer_layer,
layer_reason=reason,
source_count=len(rows),
paper_source_count=len({_paper_group_key(row) for row in rows}),
sources=[_hybrid_row_to_source(row) for row in rows],
kb_path="hybrid:qdrant+neo4j",
answer_mode=answer_mode,
llm_used=llm_used,
notes=notes,
)
@router.get("/entity/{entity_name}/relations")
async def entity_relations(
entity_name: str,
layer: str = Query("all"),
limit: int = Query(30, ge=1, le=200),
) -> Dict[str, Any]:
"""Deprecated: SQLite endpoint removed. Use /chat for hybrid retrieval."""
return {
"entity": entity_name,
"deprecated": True,
"message": "This endpoint was backed by SQLite and is no longer available. Use /api/v1/kb-qa/chat for hybrid (Qdrant + Neo4j) retrieval.",
"count": 0,
"relations": [],
}
@router.get("/relation/{relation_id}")
async def relation_detail(relation_id: str) -> Dict[str, Any]:
"""Deprecated: SQLite endpoint removed."""
return {
"deprecated": True,
"message": "This endpoint was backed by SQLite and is no longer available. Use /api/v1/kb-qa/chat for hybrid (Qdrant + Neo4j) retrieval.",
}
+215
View File
@@ -0,0 +1,215 @@
"""
KG-RAG融合API路由 - 方案A
"""
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from typing import List, Optional, Dict
import sys
from pathlib import Path
PROJECT_ROOT = Path(__file__).parent.parent
sys.path.insert(0, str(PROJECT_ROOT))
from src.rag.kg_rag_fusion import KGRAGFusion, KGRAGContext, KGEntity, KGRelation
router = APIRouter(prefix="/kg-rag", tags=["KG-RAG Fusion"])
# 全局实例
_fusion_system = None
def get_fusion_system():
"""获取融合系统实例"""
global _fusion_system
if _fusion_system is None:
_fusion_system = KGRAGFusion()
return _fusion_system
# ============ 请求/响应模型 ============
class KGRAGQueryRequest(BaseModel):
"""KG-RAG查询请求"""
question: str
top_k: int = 5
include_answer: bool = True
class EntityInfo(BaseModel):
"""实体信息"""
name: str
type: str
class RelationInfo(BaseModel):
"""关系信息"""
source: str
target: str
relation_type: str
evidence: Optional[str] = ""
source_paper: Optional[str] = ""
class ChunkInfo(BaseModel):
"""文本块信息"""
content: str
score: float
source: str
chunk_id: str
class KGRAGQueryResponse(BaseModel):
"""KG-RAG查询响应"""
question: str
entities: List[EntityInfo]
kg_relations: List[RelationInfo]
vector_chunks: List[ChunkInfo]
fused_context: str
sources: List[str]
answer: Optional[str] = None
class EntityExtractRequest(BaseModel):
"""实体提取请求"""
text: str
class EntityExtractResponse(BaseModel):
"""实体提取响应"""
entities: List[EntityInfo]
# ============ API端点 ============
@router.post("/chat", response_model=KGRAGQueryResponse)
async def kg_rag_chat(request: KGRAGQueryRequest):
"""
KG-RAG融合查询主接口
流程:
1. 实体识别
2. KG关系查询
3. 向量检索
4. 融合生成
"""
try:
system = get_fusion_system()
# 执行查询
result = system.query(request.question, top_k=request.top_k)
# 转换响应格式
response = KGRAGQueryResponse(
question=result.question,
entities=[EntityInfo(name=e.name, type=e.type) for e in result.entities],
kg_relations=[RelationInfo(
source=r.source,
target=r.target,
relation_type=r.relation_type,
evidence=r.evidence,
source_paper=r.source_paper
) for r in result.kg_relations],
vector_chunks=[ChunkInfo(
content=c.get("content", ""),
score=c.get("score", 0.0),
source=c.get("source", ""),
chunk_id=c.get("chunk_id", "")
) for c in result.vector_chunks],
fused_context=result.fused_context,
sources=result.sources
)
# 可选生成答案
if request.include_answer:
from src.rag.kg_rag_fusion import AnswerGenerator
generator = AnswerGenerator()
answer_result = generator.generate(request.question, result.fused_context)
response.answer = answer_result.get("answer", "")
return response
except Exception as e:
raise HTTPException(status_code=500, detail=f"KG-RAG查询失败: {str(e)}")
@router.post("/entities", response_model=EntityExtractResponse)
async def extract_entities(request: EntityExtractRequest):
"""
实体识别接口
"""
try:
from src.rag.kg_rag_fusion import EntityRecognizer
recognizer = EntityRecognizer()
entities = recognizer.recognize(request.text)
return EntityExtractResponse(
entities=[EntityInfo(name=e.name, type=e.type) for e in entities]
)
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.get("/kg-relations/{entity_name}")
async def get_entity_relations(entity_name: str, depth: int = 1):
"""
查询KG中实体的关系
"""
try:
from src.rag.kg_rag_fusion import KGRetriever
retriever = KGRetriever()
relations = retriever.query_entity_relations(entity_name, depth=depth)
retriever.close()
return {
"entity": entity_name,
"relations": [
{
"source": r.source,
"target": r.target,
"relation_type": r.relation_type,
"evidence": r.evidence,
"source_paper": r.source_paper
}
for r in relations
]
}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.get("/context/{question}")
async def get_kg_context(question: str):
"""
获取KG增强上下文(不生成答案)
"""
try:
system = get_fusion_system()
result = system.query(question, top_k=5)
return {
"question": question,
"entities": [{"name": e.name, "type": e.type} for e in result.entities],
"kg_relations_count": len(result.kg_relations),
"chunks_count": len(result.vector_chunks),
"fused_context": result.fused_context[:2000] # 截断
}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.get("/health")
async def health_check():
"""健康检查"""
return {
"status": "healthy",
"neo4j": "bolt://localhost:7687",
"components": {
"entity_recognizer": "ready",
"kg_retriever": "ready",
"vector_retriever": "ready",
"context_fusion": "ready"
}
}
print("[KG-RAG] 方案A路由加载完成")
+346
View File
@@ -0,0 +1,346 @@
#!/usr/bin/env python3
"""
知识图谱API路由 (Phase 4)
提供Neo4j图数据库查询接口
"""
from dataclasses import dataclass
from typing import Optional, List, Dict
from fastapi import APIRouter, HTTPException, Query
from pydantic import BaseModel
from src.rag.project_config import env
router = APIRouter()
@dataclass(frozen=True)
class Neo4jRouteConfig:
uri: str
user: str
password: str
def get_neo4j_config() -> Neo4jRouteConfig:
return Neo4jRouteConfig(
uri=env("NEO4J_URI", "bolt://localhost:7687"),
user=env("NEO4J_USER", "neo4j"),
password=env("NEO4J_PASSWORD", ""),
)
# 全局Neo4j连接
_driver = None
_driver_config: Neo4jRouteConfig | None = None
def get_neo4j_driver():
"""获取Neo4j驱动"""
global _driver, _driver_config
config = get_neo4j_config()
if _driver is not None and _driver_config != config:
try:
_driver.close()
finally:
_driver = None
_driver_config = None
if _driver is None:
try:
from neo4j import GraphDatabase
_driver = GraphDatabase.driver(config.uri, auth=(config.user, config.password))
_driver_config = config
except Exception as e:
raise HTTPException(status_code=500, detail=f"Neo4j连接失败: {e}")
return _driver
class KGQueryRequest(BaseModel):
"""知识图谱查询请求"""
cypher: str
parameters: Optional[Dict] = {}
class KGFamilyQuery(BaseModel):
"""家族查询请求"""
family_name: str
include_comparison: bool = True
# ============ 知识图谱统计接口 ============
@router.get("/kg/stats")
async def get_kg_stats():
"""获取知识图谱统计信息"""
try:
driver = get_neo4j_driver()
with driver.session() as session:
# 实体统计
family_count = session.run(
"MATCH (f:GeneFamily) RETURN count(f) as c"
).single()["c"]
species_count = session.run(
"MATCH (s:Species) RETURN count(s) as c"
).single()["c"]
gene_count = session.run(
"MATCH (g:Gene) RETURN count(g) as c"
).single()["c"]
group_count = session.run(
"MATCH (pg:PhylogenyGroup) RETURN count(pg) as c"
).single()["c"]
# 关系统计
relations = session.run("""
MATCH ()-[r]->()
RETURN type(r) as rel_type, count(r) as count
ORDER BY count DESC
""").data()
return {
"status": "success",
"entities": {
"gene_families": family_count,
"species": species_count,
"genes": gene_count,
"phylogeny_groups": group_count
},
"relations": relations,
"uri": get_neo4j_config().uri
}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
# ============ 基因家族查询接口 ============
@router.get("/kg/family/{family_name}")
async def get_family_info(family_name: str, include_species: bool = True):
"""获取基因家族详细信息"""
try:
driver = get_neo4j_driver()
with driver.session() as session:
# 查询家族基本信息
family_result = session.run("""
MATCH (f:GeneFamily {name: $family_name})
RETURN f.name as name,
f.full_name as full_name,
f.description as description,
f.member_count as member_count,
f.group_count as group_count
""", family_name=family_name).single()
if not family_result:
raise HTTPException(status_code=404, detail=f"家族 {family_name} 不存在")
result = dict(family_result)
# 查询物种信息
if include_species:
species_result = session.run("""
MATCH (f:GeneFamily {name: $family_name})
OPTIONAL MATCH (s:Species)-[r:HAS_FAMILY]->(f)
RETURN s.scientific_name as species,
s.common_name as common_name,
r.member_count as species_member_count,
r.is_comparison as is_comparison
ORDER BY r.member_count DESC
""", family_name=family_name).data()
result["species"] = [
{
"scientific_name": s["species"],
"common_name": s["common_name"],
"member_count": s["species_member_count"],
"is_comparison": s["is_comparison"]
}
for s in species_result if s["species"]
]
return {
"status": "success",
"family": result
}
except HTTPException:
raise
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.get("/kg/families")
async def list_families(
limit: int = Query(100, ge=1, le=1000),
offset: int = Query(0, ge=0),
order_by: str = Query("name", pattern="^(name|member_count)$")
):
"""列出所有基因家族"""
try:
driver = get_neo4j_driver()
with driver.session() as session:
cypher = f"""
MATCH (f:GeneFamily)
RETURN f.name as name,
f.full_name as full_name,
f.member_count as member_count
ORDER BY f.{order_by}
SKIP $offset
LIMIT $limit
"""
result = session.run(cypher, offset=offset, limit=limit).data()
# 获取总数
total = session.run("MATCH (f:GeneFamily) RETURN count(f) as c").single()["c"]
return {
"status": "success",
"families": result,
"pagination": {
"total": total,
"limit": limit,
"offset": offset
}
}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
# ============ 物种查询接口 ============
@router.get("/kg/species/{species_name}")
async def get_species_info(species_name: str):
"""获取物种详细信息"""
try:
driver = get_neo4j_driver()
with driver.session() as session:
# 查询物种基本信息
species_result = session.run("""
MATCH (s:Species {scientific_name: $species_name})
RETURN s.scientific_name as scientific_name,
s.common_name as common_name,
s.abbreviation as abbreviation
""", species_name=species_name).single()
if not species_result:
raise HTTPException(status_code=404, detail=f"物种 {species_name} 不存在")
result = dict(species_result)
# 查询该物种的基因家族
families = session.run("""
MATCH (s:Species {scientific_name: $species_name})
-[r:HAS_FAMILY]->(f:GeneFamily)
RETURN f.name as family_name,
f.full_name as full_name,
r.member_count as member_count
ORDER BY r.member_count DESC
""", species_name=species_name).data()
result["families"] = families
return {
"status": "success",
"species": result
}
except HTTPException:
raise
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.get("/kg/species")
async def list_species(
limit: int = Query(100, ge=1, le=1000),
offset: int = Query(0, ge=0)
):
"""列出所有物种"""
try:
driver = get_neo4j_driver()
with driver.session() as session:
result = session.run("""
MATCH (s:Species)
RETURN s.scientific_name as scientific_name,
s.common_name as common_name
ORDER BY s.scientific_name
SKIP $offset
LIMIT $limit
""", offset=offset, limit=limit).data()
total = session.run("MATCH (s:Species) RETURN count(s) as c").single()["c"]
return {
"status": "success",
"species": result,
"pagination": {
"total": total,
"limit": limit,
"offset": offset
}
}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
# ============ 高级查询接口 ============
@router.post("/kg/query")
async def execute_cypher(request: KGQueryRequest):
"""执行自定义Cypher查询 (只读)"""
try:
# 安全检查: 只允许读操作
if any(keyword in request.cypher.upper() for keyword in ['CREATE', 'DELETE', 'SET', 'REMOVE', 'MERGE']):
raise HTTPException(status_code=403, detail="只允许只读查询")
driver = get_neo4j_driver()
with driver.session() as session:
result = session.run(request.cypher, request.parameters).data()
return {
"status": "success",
"result": result
}
except HTTPException:
raise
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.get("/kg/search")
async def search_kg(
q: str = Query(..., min_length=1, description="搜索关键词"),
type: str = Query("all", pattern="^(all|family|species|gene)$")
):
"""知识图谱全文搜索"""
try:
driver = get_neo4j_driver()
with driver.session() as session:
results = []
# 搜索基因家族
if type in ["all", "family"]:
families = session.run("""
MATCH (f:GeneFamily)
WHERE f.name CONTAINS $q OR f.full_name CONTAINS $q
RETURN f.name as name, f.full_name as full_name,
'family' as type
LIMIT 10
""", q=q).data()
results.extend(families)
# 搜索物种
if type in ["all", "species"]:
species = session.run("""
MATCH (s:Species)
WHERE s.scientific_name CONTAINS $q OR s.common_name CONTAINS $q
RETURN s.scientific_name as name, s.common_name as full_name,
'species' as type
LIMIT 10
""", q=q).data()
results.extend(species)
return {
"status": "success",
"query": q,
"results": results
}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
Executable
+97
View File
@@ -0,0 +1,97 @@
from contextlib import asynccontextmanager
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from fastapi.responses import FileResponse
from api.routes import router as api_router
from api.kg_routes import router as kg_router
from api.reasoning_routes import router as reasoning_router
from api.rag_v10_routes import router as v10_router
from api.advanced_rag_routes import router as advanced_rag_router
from api.raptor_routes import router as raptor_router
from api.temporal_routes import router as temporal_router
from api.kg_rag_routes import router as kg_rag_router # 方案A: KG-RAG融合
from api.species_routes import router as species_router # 物种查询路由
from api.methodology_routes import router as methodology_router
from api.kb_qa_routes import router as kb_qa_router
from api.v15_graph_qa_routes import router as v15_graph_qa_router
from api.v15_agentic_qa_routes import router as v15_agentic_qa_router
from src.rag.project_config import PROJECT_ROOT
@asynccontextmanager
async def lifespan(app: FastAPI):
yield
app = FastAPI(
title="Gene Family RAG System",
description="纯净版RAG系统 - 为bbav20311项目提供知识检索支持",
version="1.0.0",
lifespan=lifespan,
)
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
app.include_router(api_router, prefix="/api/v1")
app.include_router(kg_router, prefix="/api/v1")
app.include_router(reasoning_router, prefix="/api/v1")
app.include_router(v10_router, prefix="/api/v1")
app.include_router(advanced_rag_router, prefix="/api/v1")
app.include_router(raptor_router, prefix="/api/v1")
app.include_router(temporal_router, prefix="/api/v1")
app.include_router(kg_rag_router, prefix="/api/v1") # 方案A: KG-RAG融合
app.include_router(species_router, prefix="/api/v1") # 物种查询路由
app.include_router(methodology_router, prefix="/api/v1")
app.include_router(kb_qa_router, prefix="/api/v1") # V15 SRT SQLite知识库QA
app.include_router(v15_graph_qa_router, prefix="/api/v1") # V15 Neo4j Graph QA
app.include_router(v15_agentic_qa_router, prefix="/api/v1") # V15 Agentic QA
# 静态文件服务
app.mount("/static", StaticFiles(directory=str(PROJECT_ROOT / "static")), name="static")
@app.get("/")
async def root():
return FileResponse(str(PROJECT_ROOT / "static" / "index.html"))
@app.get("/health")
async def health_check():
return {"status": "healthy"}
# 监控指标端点
from fastapi import Request
import time
# 简单的请求计数器
request_count = 0
request_times = []
@app.middleware("http")
async def add_metrics(request: Request, call_next):
global request_count
start_time = time.time()
request_count += 1
response = await call_next(request)
process_time = time.time() - start_time
request_times.append(process_time)
if len(request_times) > 1000:
request_times.pop(0)
return response
@app.get("/metrics")
async def metrics():
"""Prometheus监控指标"""
avg_time = sum(request_times) / len(request_times) if request_times else 0
return {
"request_count": request_count,
"avg_response_time": avg_time,
"status": "healthy"
}
+196
View File
@@ -0,0 +1,196 @@
#!/usr/bin/env python3
"""
方法学提取 API - FastAPI 版本
提供接口:
- POST /extract - 从 PDF 提取方法学参数
- GET /result/{job_id} - 获取提取结果
"""
import os
import sys
import json
import uuid
from pathlib import Path
from fastapi import FastAPI, File, UploadFile, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
from typing import Dict, List, Optional
sys.path.insert(0, str(Path(__file__).parent.parent))
from src.rag.extractors.methodology.simple_extractor import SimpleExtractor
app = FastAPI(title="Methodology Extraction API")
# CORS
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# 存储提取结果
results = {}
class ExtractionResult(BaseModel):
job_id: str
status: str
data: Optional[Dict] = None
error: Optional[str] = None
@app.post("/extract")
async def extract_methodology(pdf: UploadFile = File(...)):
"""从 PDF 提取方法学参数"""
# 生成 job_id
job_id = str(uuid.uuid4())[:8]
# 保存 PDF
upload_dir = Path(__file__).parent.parent / 'uploads'
upload_dir.mkdir(exist_ok=True)
pdf_path = upload_dir / f"{job_id}_{pdf.filename}"
with open(pdf_path, "wb") as f:
content = await pdf.read()
f.write(content)
# 提取
try:
extractor = SimpleExtractor()
result_text = extractor.extract(pdf_path)
# 解析为结构化数据
structured = parse_extraction_result(result_text)
# 保存结果
results[job_id] = {
'status': 'completed',
'pdf_path': str(pdf_path),
'raw_text': result_text,
'data': structured
}
return {
'job_id': job_id,
'status': 'completed',
'data': structured
}
except Exception as e:
results[job_id] = {
'status': 'failed',
'error': str(e)
}
raise HTTPException(status_code=500, detail=str(e))
@app.get("/result/{job_id}")
async def get_result(job_id: str):
"""获取提取结果"""
if job_id not in results:
raise HTTPException(status_code=404, detail="Job not found")
return results[job_id]
def parse_extraction_result(text: str) -> Dict:
"""解析提取结果为结构化数据"""
data = {
'species': {
'target': '',
'references': []
},
'gene_family': {
'name': '',
'pfam_id': ''
},
'stages': []
}
# 简单解析
lines = text.split('\n')
for line in lines:
line = line.strip()
# 物种
if '目标物种' in line or 'target' in line.lower():
if ':' in line:
data['species']['target'] = line.split(':', 1)[-1].strip()
elif '参考物种' in line or 'references' in line.lower():
if ':' in line:
refs = line.split(':', 1)[-1].strip()
data['species']['references'] = [r.strip() for r in refs.split('、') if r.strip()]
elif '基因家族' in line:
if ':' in line:
parts = line.split(':', 1)[-1].strip()
if '(' in parts and 'Pfam:' in parts:
name = parts.split('(')[0].strip()
pfam = parts.split('Pfam:')[-1].split(')')[0].strip()
data['gene_family']['name'] = name
data['gene_family']['pfam_id'] = pfam
# 默认阶段
data['stages'] = [
{
'id': 'identify',
'name': '1. 基因家族鉴定',
'params': [
{'name': 'tool', 'label': '工具', 'value': 'BLASTp + HMMER'},
{'name': 'pfam_id', 'label': 'Pfam ID', 'value': 'PF03110'},
{'name': 'e_value', 'label': 'E-value', 'value': ''},
{'name': 'identity', 'label': 'Identity', 'value': ''}
]
},
{
'id': 'phylogeny',
'name': '2. 系统发育分析',
'params': [
{'name': 'software', 'label': '软件', 'value': 'MEGA 6.0'},
{'name': 'method', 'label': '方法', 'value': 'Neighbor-Joining'},
{'name': 'bootstrap', 'label': 'Bootstrap', 'value': '1000'}
]
},
{
'id': 'motif',
'name': '3. 保守基序分析',
'params': [
{'name': 'tool', 'label': '工具', 'value': 'MEME'},
{'name': 'max_motifs', 'label': 'Max motifs', 'value': '10'}
]
},
{
'id': 'structure',
'name': '4. 基因结构分析',
'params': [
{'name': 'tool', 'label': '工具', 'value': 'GSDS'}
]
},
{
'id': 'synteny',
'name': '5. 共线性分析',
'params': [
{'name': 'tool', 'label': '工具', 'value': 'MCScanX'}
]
},
{
'id': 'expression',
'name': '6. 表达分析',
'params': [
{'name': 'method', 'label': '方法', 'value': 'qRT-PCR'}
]
}
]
return data
if __name__ == '__main__':
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=5001)
+302
View File
@@ -0,0 +1,302 @@
#!/usr/bin/env python3
"""
方法学提取 API v2 - 支持9个阶段(完整版)
"""
import os
import sys
import json
import uuid
from pathlib import Path
from fastapi import FastAPI, File, UploadFile, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from typing import Dict, List
sys.path.insert(0, str(Path(__file__).parent.parent))
from src.rag.extractors.methodology.simple_extractor import SimpleExtractor
app = FastAPI(title="Methodology Extraction API v2 - Full Data")
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
results = {}
@app.post("/extract")
async def extract_methodology(pdf: UploadFile = File(...)):
"""从 PDF 提取方法学参数 - 返回完整数据"""
job_id = str(uuid.uuid4())[:8]
upload_dir = Path(__file__).parent.parent / 'uploads'
upload_dir.mkdir(exist_ok=True)
pdf_path = upload_dir / f"{job_id}_{pdf.filename}"
with open(pdf_path, "wb") as f:
content = await pdf.read()
f.write(content)
try:
extractor = SimpleExtractor()
result_text = extractor.extract(pdf_path)
structured = parse_extraction_result(result_text)
# TODO: 修复 parse_extraction_result 真正解析 LLM 结果
# 临时方案:返回原始文本 + 尝试解析
structured = parse_extraction_result(result_text)
results[job_id] = {
'status': 'completed',
'pdf_path': str(pdf_path),
'raw_text': result_text,
'data': structured
}
return {
'job_id': job_id,
'status': 'completed',
'raw_text': result_text, # 添加原始文本供调试
'data': structured
}
except Exception as e:
results[job_id] = {'status': 'failed', 'error': str(e)}
raise HTTPException(status_code=500, detail=str(e))
@app.get("/result/{job_id}")
async def get_result(job_id: str):
"""获取提取结果"""
if job_id not in results:
raise HTTPException(status_code=404, detail="Job not found")
return results[job_id]
def parse_extraction_result(text: str) -> Dict:
"""
解析提取结果(9个阶段)- 从 LLM JSON 输出解析
"""
import re
# 尝试从文本中提取 JSON
try:
# 查找 ```json ... ``` 代码块
json_match = re.search(r'```json\s*(.*?)\s*```', text, re.DOTALL)
if json_match:
json_str = json_match.group(1)
else:
# 尝试直接解析整个文本为 JSON
json_str = text.strip()
parsed = json.loads(json_str)
# 验证必要字段
if 'species' in parsed and 'stages' in parsed:
return parsed
except Exception as e:
print(f"JSON 解析失败: {e}")
# 如果 JSON 解析失败,返回空结构
return {
'species': {'target': '', 'references': []},
'gene_family': {'name': '', 'pfam_id': ''},
'stages': []
}
def _old_parse_extraction_result(text: str) -> Dict:
"""
[已废弃] 解析提取结果(9个阶段)- 完整版
从完整JSON模板加载,确保所有字段完整
"""
# 加载完整JSON模板
template_path = Path(__file__).parent.parent / 'output' / 'spl_methods_extraction_complete.json'
if template_path.exists():
with open(template_path, 'r', encoding='utf-8') as f:
full_data = json.load(f)
else:
full_data = {}
# 构建前端需要的格式
data = {
'species': {
'target': full_data.get('species', {}).get('target', {}).get('scientific_name', ''),
'references': [s.get('name', '') for s in full_data.get('species', {}).get('references', [])]
},
'gene_family': {
'name': full_data.get('gene_family', {}).get('name', ''),
'full_name': full_data.get('gene_family', {}).get('full_name', ''),
'pfam_id': full_data.get('gene_family', {}).get('pfam_id', ''),
'domain': full_data.get('gene_family', {}).get('domain', ''),
'gene_count': full_data.get('gene_family', {}).get('gene_count', 0)
},
'stages': []
}
# 从完整数据提取各阶段
protein_analysis = full_data.get('protein_sequence_analysis', {})
chromosomal = full_data.get('chromosomal_analysis', {})
expression = full_data.get('expression_analysis', {})
statistics = full_data.get('statistical_analysis', {})
# 1. 基因家族鉴定
identification = protein_analysis.get('identification', {})
data['stages'].append({
'id': 'identify',
'name': '1. 基因家族鉴定',
'params': [
{'name': 'tool', 'label': '工具', 'value': ', '.join(identification.get('tools', []))},
{'name': 'genome_source', 'label': '基因组来源', 'value': full_data.get('species', {}).get('target', {}).get('genome_database', '')},
{'name': 'genome_url', 'label': '基因组URL', 'value': full_data.get('species', {}).get('target', {}).get('genome_url', '')},
{'name': 'pfam_id', 'label': 'Pfam ID', 'value': identification.get('domain', '').split('(')[-1].replace(')', '') if '(' in identification.get('domain', '') else ''},
{'name': 'pfam_url', 'label': 'Pfam URL', 'value': 'http://pfam.xfam.org/'},
{'name': 'reference_sequences', 'label': '参考序列', 'value': identification.get('reference_sequences', '')},
{'name': 'reference_db', 'label': '参考数据库', 'value': full_data.get('species', {}).get('references', [{}])[0].get('database', '')},
{'name': 'ref_seq_length', 'label': '参考序列长度', 'value': '131aa-1035aa'},
{'name': 'e_value', 'label': 'E-value', 'value': '原文未明确'},
{'name': 'identity', 'label': 'Identity', 'value': '原文未明确'},
{'name': 'coverage', 'label': 'Coverage', 'value': '原文未明确'},
{'name': 'filter_criteria', 'label': '筛选标准', 'value': identification.get('filtering', '')},
{'name': 'final_count', 'label': '最终基因数', 'value': str(identification.get('final_count', ''))},
]
})
# 2. 蛋白理化性质
pep = protein_analysis.get('physicochemical_properties', {})
pep_results = pep.get('results', {})
data['stages'].append({
'id': 'pep',
'name': '2. 蛋白理化性质',
'params': [
{'name': 'tool', 'label': '工具', 'value': pep.get('tool', '')},
{'name': 'url', 'label': 'URL', 'value': pep.get('url', '')},
{'name': 'properties_analyzed', 'label': '分析内容', 'value': ', '.join(pep.get('properties_analyzed', []))},
{'name': 'protein_length', 'label': '蛋白长度', 'value': pep_results.get('protein_length', '')},
{'name': 'molecular_weight', 'label': '分子量', 'value': pep_results.get('molecular_weight', '')},
{'name': 'isoelectric_point', 'label': '等电点 PI', 'value': pep_results.get('isoelectric_point', '')},
{'name': 'subcellular_localization', 'label': '亚细胞定位', 'value': pep_results.get('subcellular_localization', '')},
]
})
# 3. 系统发育分析
phylo = protein_analysis.get('phylogenetic_analysis', {})
data['stages'].append({
'id': 'phylogeny',
'name': '3. 系统发育分析',
'params': [
{'name': 'software', 'label': '软件', 'value': f"{phylo.get('software', '')} {phylo.get('version', '')}".strip()},
{'name': 'alignment_tool', 'label': '比对工具', 'value': f"{protein_analysis.get('sequence_alignment', {}).get('tool', '')} {protein_analysis.get('sequence_alignment', {}).get('version', '')}".strip()},
{'name': 'method', 'label': '建树方法', 'value': phylo.get('method', '').split('(')[0].strip() if '(' in phylo.get('method', '') else phylo.get('method', '')},
{'name': 'bootstrap', 'label': 'Bootstrap', 'value': str(phylo.get('bootstrap', ''))},
{'name': 'model', 'label': '替代模型', 'value': phylo.get('substitution_model', '')},
{'name': 'comparison', 'label': '比较对象', 'value': phylo.get('comparison', '')},
{'name': 'grouping', 'label': '分组结果', 'value': phylo.get('grouping', '')},
{'name': 'reference_species', 'label': '参考物种', 'value': ', '.join([s.get('name', '') for s in full_data.get('species', {}).get('references', [])])},
{'name': 'database', 'label': '序列数据库', 'value': full_data.get('species', {}).get('references', [{}])[0].get('database', '')},
]
})
# 4. 基因结构分析
gene_struct = protein_analysis.get('gene_structure', {})
data['stages'].append({
'id': 'structure',
'name': '4. 基因结构分析',
'params': [
{'name': 'tool', 'label': '工具', 'value': gene_struct.get('tool', '')},
{'name': 'url', 'label': 'URL', 'value': gene_struct.get('url', '')},
{'name': 'method', 'label': '方法', 'value': gene_struct.get('method', '')},
{'name': 'analysis', 'label': '分析内容', 'value': gene_struct.get('analysis', '')},
]
})
# 5. 保守基序分析
motifs = protein_analysis.get('conserved_domains', {}).get('conserved_motifs', {})
data['stages'].append({
'id': 'motif',
'name': '5. 保守基序分析',
'params': [
{'name': 'tool', 'label': '工具', 'value': motifs.get('tool', '')},
{'name': 'url', 'label': 'URL', 'value': motifs.get('url', '')},
{'name': 'mode', 'label': '模式', 'value': motifs.get('parameters', {}).get('mode', '')},
{'name': 'max_motifs', 'label': 'Max motifs', 'value': str(motifs.get('parameters', {}).get('max_motifs', ''))},
{'name': 'width', 'label': 'Width', 'value': motifs.get('parameters', {}).get('width', '')},
{'name': 'detected_motifs', 'label': '检测到的motifs', 'value': motifs.get('detected_motifs', '')},
{'name': 'conserved_sequences', 'label': '保守序列', 'value': ', '.join(motifs.get('conserved_sequences', []))},
]
})
# 6. 染色体定位
data['stages'].append({
'id': 'karyotype',
'name': '6. 染色体定位',
'params': [
{'name': 'mapping_method', 'label': '定位方法', 'value': chromosomal.get('mapping_method', '')},
{'name': 'location', 'label': '定位结果', 'value': chromosomal.get('location', '')},
]
})
# 7. 共线性分析
duplication = chromosomal.get('duplication_analysis', {})
data['stages'].append({
'id': 'synteny',
'name': '7. 共线性分析',
'params': [
{'name': 'tool', 'label': '工具', 'value': duplication.get('tool', '')},
{'name': 'full_name', 'label': '完整名称', 'value': duplication.get('full_name', '')},
{'name': 'synteny_tool', 'label': '共线性绘图工具', 'value': duplication.get('synteny_tool', '')},
{'name': 'synteny_url', 'label': '共线性工具URL', 'value': duplication.get('synteny_url', '')},
{'name': 'segmental_duplications', 'label': '片段复制数', 'value': str(duplication.get('results', {}).get('segmental_duplications', ''))},
{'name': 'tandem_duplications', 'label': '串联复制数', 'value': str(duplication.get('results', {}).get('tandem_duplications', ''))},
]
})
# 8. 表达分析
primer_design = expression.get('primer_design', {})
primer_params = primer_design.get('parameters', {})
data['stages'].append({
'id': 'expression',
'name': '8. 表达分析',
'params': [
{'name': 'method', 'label': '方法', 'value': expression.get('method', '')},
{'name': 'full_name', 'label': '方法全称', 'value': expression.get('full_name', '')},
{'name': 'kit', 'label': '试剂盒', 'value': expression.get('kit', '')},
{'name': 'reference_gene', 'label': '内参基因', 'value': expression.get('reference_gene', '')},
{'name': 'genes_analyzed', 'label': '分析基因数', 'value': str(expression.get('genes_analyzed', ''))},
{'name': 'primer_tool', 'label': '引物设计工具', 'value': primer_design.get('tool', '')},
{'name': 'primer_url', 'label': '引物工具URL', 'value': primer_design.get('url', '')},
{'name': 'product_length', 'label': '产物长度', 'value': primer_params.get('product_length', '')},
{'name': 'gc_content', 'label': 'GC含量', 'value': primer_params.get('gc_content', '')},
{'name': 'melting_temp', 'label': '退火温度', 'value': primer_params.get('melting_temperature', '')},
{'name': 'primer_source', 'label': '引物序列来源', 'value': primer_design.get('primer_sequences_file', '')},
{'name': 'data_analysis', 'label': '数据分析方法', 'value': expression.get('data_analysis', '')},
{'name': 'tissues', 'label': '分析组织', 'value': ', '.join(expression.get('tissues_analyzed', []))},
{'name': 'stages', 'label': '发育阶段', 'value': ', '.join(expression.get('developmental_stages', []))},
]
})
# 9. 统计分析
data['stages'].append({
'id': 'statistics',
'name': '9. 统计分析',
'params': [
{'name': 'software', 'label': '软件', 'value': statistics.get('software', '')},
{'name': 'company', 'label': '公司', 'value': statistics.get('company', '')},
{'name': 'method', 'label': '方法', 'value': ', '.join(statistics.get('methods', []))},
{'name': 'significance_levels', 'label': '显著性水平', 'value': ', '.join(statistics.get('significance_levels', []))},
]
})
return data
if __name__ == '__main__':
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=5002)
+998
View File
@@ -0,0 +1,998 @@
"""Methodology extraction batch result routes."""
from __future__ import annotations
import csv
import json
import os
import re
import shlex
import subprocess
import uuid
from datetime import datetime
from pathlib import Path
from typing import Any
from fastapi import APIRouter, BackgroundTasks, File, Form, HTTPException, UploadFile
from pydantic import BaseModel, Field
from src.rag.unified.bbav5_methodology_config import build_bbav5_methodology_config, build_methodology_execution_plan, ensure_methodology_execution_contract, resolve_batch_dir
from src.rag.unified.figure_table_planner import build_figure_table_outputs
from src.rag.unified.methodology_qa_sandbox import DEFAULT_SANDBOX_ROOT, answer_from_sandbox, build_methodology_qa_sandbox, sanitize_token
from src.rag.project_config import PROJECT_ROOT, conda_env_path
# Use LLM extraction results by default (can be overridden via env var)
BATCH_DIR_NAME = os.environ.get("METHODOLOGY_BATCH_DIR", "methodology_batch_llm")
BATCH_DIR = PROJECT_ROOT / "data" / "evaluation" / BATCH_DIR_NAME
PAPERS_CSV = PROJECT_ROOT / "data" / "evaluation" / "methodology_papers.csv"
GOLD_CSV = PROJECT_ROOT / "data" / "evaluation" / "methodology_gold.csv"
REVIEW_RESULTS_JSON = BATCH_DIR / "review_results.json"
REVIEW_RESULTS_CSV = BATCH_DIR / "review_results.csv"
UPLOAD_JOBS_DIR = PROJECT_ROOT / "data" / "evaluation" / "methodology_upload_jobs"
UPLOAD_OUTPUT_ROOT = PROJECT_ROOT / "data" / "evaluation" / "0505"
DEFAULT_MINERU_CMD = str(conda_env_path("minerupy311") / "bin" / "mineru")
DEFAULT_MINERU_API_URL = "http://127.0.0.1:50215"
DEFAULT_MINERU_BACKEND = "hybrid-auto-engine"
DEFAULT_MINERU_METHOD = "auto"
DEFAULT_MINERU_LANG = "en"
DEFAULT_MINERU_MIN_FREE_GPU_MB = 6144
DEFAULT_MINERU_LOW_MEM_MIN_FREE_GPU_MB = 4096
DEFAULT_MINERU_RUNTIME_MODE = "single"
router = APIRouter(prefix="/methodology", tags=["methodology"])
class ReviewResult(BaseModel):
observation_id: str = Field(..., min_length=1)
paper_id: str = ""
review_status: str = ""
stage_review: str = ""
tools_review: str = ""
versions_review: str = ""
parameters_review: str = ""
inputs_outputs_review: str = ""
evidence_review: str = ""
bbav5_review: str = ""
bbav5_stage_review: str = ""
bbav5_tools_review: str = ""
support_status_review: str = ""
corrected_stage_value: str = ""
corrected_tools_value: str = ""
corrected_versions_value: str = ""
corrected_parameters_value: str = ""
corrected_evidence_value: str = ""
corrected_inputs_value: str = ""
corrected_outputs_value: str = ""
corrected_bbav5_stage_value: str = ""
corrected_bbav5_tools_value: str = ""
corrected_support_status_value: str = ""
manual_note: str = ""
class ReviewResultsPayload(BaseModel):
results: list[ReviewResult]
class MethodologySandboxBuildPayload(BaseModel):
methodology_dir: str = Field(..., min_length=1)
sandbox_root: str = str(DEFAULT_SANDBOX_ROOT)
run_id: str = ""
include_figures: bool = True
content_list_path: str = ""
class MethodologySandboxAnswerPayload(BaseModel):
question: str = Field(..., min_length=1)
registry_path: str = ""
sandbox_dir: str = ""
REVIEW_RESULT_FIELDS = [
"observation_id",
"paper_id",
"review_status",
"stage_review",
"tools_review",
"versions_review",
"parameters_review",
"inputs_outputs_review",
"evidence_review",
"bbav5_review",
"bbav5_stage_review",
"bbav5_tools_review",
"support_status_review",
"corrected_stage_value",
"corrected_tools_value",
"corrected_versions_value",
"corrected_parameters_value",
"corrected_evidence_value",
"corrected_inputs_value",
"corrected_outputs_value",
"corrected_bbav5_stage_value",
"corrected_bbav5_tools_value",
"corrected_support_status_value",
"manual_note",
]
def read_csv(path: Path) -> list[dict[str, str]]:
if not path.exists():
return []
with path.open("r", encoding="utf-8-sig", newline="") as handle:
return list(csv.DictReader(handle))
def read_json(path: Path) -> Any:
if not path.exists():
raise HTTPException(status_code=404, detail=f"Missing file: {path}")
return json.loads(path.read_text(encoding="utf-8"))
def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=REVIEW_RESULT_FIELDS)
writer.writeheader()
for row in rows:
writer.writerow({field: row.get(field, "") for field in REVIEW_RESULT_FIELDS})
def read_review_results() -> list[dict[str, Any]]:
if not REVIEW_RESULTS_JSON.exists():
return []
data = json.loads(REVIEW_RESULTS_JSON.read_text(encoding="utf-8"))
if isinstance(data, dict):
return list(data.get("results", []))
return list(data)
def write_job_status(job_dir: Path, payload: dict[str, Any]) -> None:
payload = {"updated_at": datetime.now().isoformat(timespec="seconds"), **payload}
job_dir.mkdir(parents=True, exist_ok=True)
(job_dir / "status.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
def parse_int_env(name: str, default: int) -> int:
value = os.environ.get(name, "").strip()
if not value:
return default
try:
parsed = int(value)
except ValueError:
return default
return max(0, parsed)
def parse_optional_int_env(name: str) -> int | None:
value = os.environ.get(name, "").strip()
if not value:
return None
try:
return max(0, int(value))
except ValueError:
return None
def env_bool(name: str) -> bool:
return os.environ.get(name, "").strip().lower() in {"1", "true", "yes", "on"}
def env_bool_text(name: str) -> str:
value = os.environ.get(name, "").strip().lower()
if value in {"1", "true", "yes", "on"}:
return "true"
if value in {"0", "false", "no", "off"}:
return "false"
return ""
def normalize_mineru_runtime_mode(value: str) -> str:
mode = str(value or "").strip().lower()
return mode if mode in {"single", "batch"} else DEFAULT_MINERU_RUNTIME_MODE
def mineru_default_min_free_gpu_mb(runtime: dict[str, str]) -> int:
mode = normalize_mineru_runtime_mode(runtime.get("mode", ""))
backend = str(runtime.get("backend") or "").strip().lower()
method = str(runtime.get("method") or "").strip().lower()
formula = str(runtime.get("formula") or "").strip().lower()
table = str(runtime.get("table") or "").strip().lower()
low_mem_pipeline = (
mode == "single"
and backend == "pipeline"
and method == "txt"
and formula == "false"
and table == "false"
)
if low_mem_pipeline:
return DEFAULT_MINERU_LOW_MEM_MIN_FREE_GPU_MB
return DEFAULT_MINERU_MIN_FREE_GPU_MB
def parse_gpu_memory_csv(text: str) -> list[dict[str, int]]:
rows: list[dict[str, int]] = []
for line in text.splitlines():
parts = [part.strip() for part in line.split(",")]
if len(parts) < 4:
continue
try:
rows.append(
{
"index": int(parts[0]),
"total_mb": int(parts[1]),
"used_mb": int(parts[2]),
"free_mb": int(parts[3]),
}
)
except ValueError:
continue
return rows
def parse_gpu_process_csv(text: str) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
for line in text.splitlines():
parts = [part.strip() for part in line.split(",", 2)]
if len(parts) < 3:
continue
try:
rows.append({"pid": int(parts[0]), "name": parts[1], "used_mb": int(parts[2])})
except ValueError:
continue
return rows
def cuda_visible_gpu_indices(cuda_visible_devices: str) -> list[int]:
indices: list[int] = []
for token in cuda_visible_devices.split(","):
token = token.strip()
if token.isdigit():
indices.append(int(token))
return indices
def format_gpu_processes(processes: list[dict[str, Any]], limit: int = 8) -> str:
if not processes:
return "未检测到 GPU 计算进程"
ordered = sorted(processes, key=lambda row: int(row.get("used_mb") or 0), reverse=True)
formatted = []
for row in ordered[:limit]:
name = Path(str(row.get("name") or "")).name or str(row.get("name") or "")
formatted.append(f"PID {row.get('pid')} {name} {row.get('used_mb')}MiB")
if len(ordered) > limit:
formatted.append(f"... 另有 {len(ordered) - limit} 个进程")
return "; ".join(formatted)
def build_mineru_gpu_preflight_error(
gpu_rows: list[dict[str, int]],
process_rows: list[dict[str, Any]],
cuda_visible_devices: str,
min_free_mb: int,
runtime_mode: str = DEFAULT_MINERU_RUNTIME_MODE,
) -> str:
if min_free_mb <= 0:
return ""
visible_indices = cuda_visible_gpu_indices(cuda_visible_devices)
candidate_rows = [row for row in gpu_rows if row["index"] in visible_indices] if visible_indices else gpu_rows
if not candidate_rows:
return (
"MinerU GPU 预检失败:无法从 nvidia-smi 读取可用 GPU。"
f"CUDA_VISIBLE_DEVICES={cuda_visible_devices or '<empty>'}"
)
best = max(candidate_rows, key=lambda row: row["free_mb"])
if best["free_mb"] >= min_free_mb:
return ""
processes = format_gpu_processes(process_rows)
mode = normalize_mineru_runtime_mode(runtime_mode)
if mode == "single":
return (
"MinerU 按需启动需要 GPU 显存,但当前 GPU 显存不足,不能启动新的 PDF 解析任务。"
f"GPU{best['index']} free={best['free_mb']}MiB / total={best['total_mb']}MiB,"
f"需要至少 {min_free_mb}MiB 空闲显存。"
f"当前 GPU 进程:{processes}。"
"请先释放占用显存的 QA/VLLM 进程,或改用低显存 MinerU 配置后重试。"
)
return (
"MinerU 服务已热加载,但当前 GPU 显存余量不足,不能安全提交新的 PDF 解析任务。"
f"GPU{best['index']} free={best['free_mb']}MiB / total={best['total_mb']}MiB,"
f"需要至少 {min_free_mb}MiB 空闲显存。"
f"当前 GPU 进程:{processes}。"
"请先释放占用显存的 QA/VLLM/MinerU 进程,或调低 METHODOLOGY_MINERU_MIN_FREE_GPU_MB 后重试。"
)
def mineru_gpu_preflight_error(
cuda_visible_devices: str,
runtime_mode: str = DEFAULT_MINERU_RUNTIME_MODE,
runtime: dict[str, str] | None = None,
) -> str:
if os.environ.get("METHODOLOGY_MINERU_GPU_PREFLIGHT", "1").strip().lower() in {"0", "false", "no", "off"}:
return ""
runtime_payload = runtime or {"mode": runtime_mode}
min_free_mb = parse_optional_int_env("METHODOLOGY_MINERU_MIN_FREE_GPU_MB")
if min_free_mb is None:
min_free_mb = mineru_default_min_free_gpu_mb(runtime_payload)
if min_free_mb <= 0:
return ""
try:
memory = subprocess.run(
["nvidia-smi", "--query-gpu=index,memory.total,memory.used,memory.free", "--format=csv,noheader,nounits"],
cwd=PROJECT_ROOT,
text=True,
capture_output=True,
timeout=5,
)
except Exception as exc:
return f"MinerU GPU 预检失败:无法执行 nvidia-smi:{exc}"
if memory.returncode != 0:
return f"MinerU GPU 预检失败:nvidia-smi 返回 {memory.returncode}:{memory.stderr.strip() or memory.stdout.strip()}"
processes_text = ""
try:
processes = subprocess.run(
["nvidia-smi", "--query-compute-apps=pid,process_name,used_memory", "--format=csv,noheader,nounits"],
cwd=PROJECT_ROOT,
text=True,
capture_output=True,
timeout=5,
)
if processes.returncode == 0:
processes_text = processes.stdout
except Exception:
processes_text = ""
return build_mineru_gpu_preflight_error(
parse_gpu_memory_csv(memory.stdout),
parse_gpu_process_csv(processes_text),
cuda_visible_devices,
min_free_mb,
runtime_payload.get("mode", runtime_mode),
)
def summarize_mineru_failure(log_path: Path, returncode: int) -> str:
message = f"MinerU failed with exit code {returncode}; see {log_path}"
if not log_path.exists():
return message
try:
text = log_path.read_text(encoding="utf-8", errors="replace")
except Exception:
return message
if re.search(r"CUDA out of memory|OutOfMemoryError|cuda memory", text, re.IGNORECASE):
return (
"MinerU 解析失败:CUDA 显存不足。"
"这通常表示 GPU 已被 QA/VLLM/MinerU 进程占用,运行 PDF 检测/解析时没有足够额外显存。"
f"日志:{log_path}"
)
return message
def build_mineru_runtime() -> tuple[list[str], dict[str, str], dict[str, str]]:
env = os.environ.copy()
env.setdefault("MINERU_MODEL_SOURCE", "modelscope")
env.setdefault("CUDA_VISIBLE_DEVICES", "0")
env.setdefault("TORCH_CUDNN_V8_API_DISABLED", "1")
mineru_cmd = shlex.split(os.environ.get("MINERU_CMD", DEFAULT_MINERU_CMD))
runtime_mode = normalize_mineru_runtime_mode(
os.environ.get("METHODOLOGY_MINERU_RUNTIME_MODE", DEFAULT_MINERU_RUNTIME_MODE)
)
runtime = {
"mode": runtime_mode,
"model_source": env.get("MINERU_MODEL_SOURCE", ""),
"cuda_visible_devices": env.get("CUDA_VISIBLE_DEVICES", ""),
"api_url": os.environ.get("V15_MINERU_API_URL", DEFAULT_MINERU_API_URL).strip(),
"use_api": "1" if runtime_mode == "batch" and env_bool("V15_MINERU_USE_API") else "0",
"backend": os.environ.get("V15_MINERU_BACKEND", DEFAULT_MINERU_BACKEND).strip() or DEFAULT_MINERU_BACKEND,
"method": os.environ.get("V15_MINERU_METHOD", DEFAULT_MINERU_METHOD).strip() or DEFAULT_MINERU_METHOD,
"lang": os.environ.get("V15_MINERU_LANG", DEFAULT_MINERU_LANG).strip() or DEFAULT_MINERU_LANG,
"formula": env_bool_text("V15_MINERU_FORMULA"),
"table": env_bool_text("V15_MINERU_TABLE"),
}
return mineru_cmd, env, runtime
def build_mineru_command(mineru_cmd: list[str], source_path: Path, mineru_out: Path, runtime: dict[str, str]) -> list[str]:
cmd = [
*mineru_cmd,
"-p",
str(source_path),
"-o",
str(mineru_out),
"-m",
runtime["method"],
"-b",
runtime["backend"],
"-l",
runtime["lang"],
]
if runtime.get("formula"):
cmd.extend(["-f", runtime["formula"]])
if runtime.get("table"):
cmd.extend(["-t", runtime["table"]])
if runtime.get("mode") == "batch" and runtime.get("use_api") == "1" and runtime.get("api_url"):
cmd.extend(["--api-url", runtime["api_url"]])
return cmd
def read_job_status(job_id: str) -> dict[str, Any]:
status_path = UPLOAD_JOBS_DIR / job_id / "status.json"
if not status_path.exists():
raise HTTPException(status_code=404, detail=f"Unknown methodology extraction job: {job_id}")
return json.loads(status_path.read_text(encoding="utf-8"))
def list_upload_job_statuses(limit: int = 50) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
if not UPLOAD_JOBS_DIR.exists():
return rows
for status_path in UPLOAD_JOBS_DIR.glob("*/status.json"):
try:
payload = json.loads(status_path.read_text(encoding="utf-8"))
except Exception:
continue
output_dir = str(payload.get("output_dir") or "")
source_path = str(payload.get("source_path") or "")
row = {
"job_id": payload.get("job_id") or status_path.parent.name,
"status": payload.get("status", ""),
"stage": payload.get("stage", ""),
"substage": payload.get("substage", ""),
"message": payload.get("message", ""),
"updated_at": payload.get("updated_at", ""),
"paper_id": payload.get("paper_id", ""),
"family": payload.get("family", ""),
"source_filename": Path(source_path).name if source_path else "",
"source_path": source_path,
"output_dir": output_dir,
"batch_dir": output_dir,
"stage_count": payload.get("stage_count", 0),
"species_count": payload.get("species_count", 0),
"figure_table_caption_count": payload.get("figure_table_caption_count", 0),
"figure_table_stage_count": payload.get("figure_table_stage_count", 0),
"config_url": payload.get("config_url", f"/api/v1/methodology/extract-jobs/{status_path.parent.name}/bbav5-config"),
"methodology_plan_url": payload.get("methodology_plan_url", f"/api/v1/methodology/extract-jobs/{status_path.parent.name}/plan"),
"methodology_sandbox_requested": payload.get("methodology_sandbox_requested", False),
"methodology_sandbox_status": payload.get("methodology_sandbox_status", ""),
"methodology_sandbox_dir": payload.get("methodology_sandbox_dir", ""),
"methodology_sandbox_registry": payload.get("methodology_sandbox_registry", ""),
"methodology_sandbox_collection": payload.get("methodology_sandbox_collection", ""),
}
rows.append(row)
rows.sort(key=lambda row: str(row.get("updated_at") or ""), reverse=True)
return rows[: max(1, min(int(limit or 50), 200))]
def run_uploaded_methodology_job(
job_id: str,
source_path: Path,
paper_id: str,
family: str,
output_dir: Path,
llm_model: str,
llm_request_timeout: int,
sandbox: bool = False,
sandbox_run_id: str = "",
) -> None:
job_dir = UPLOAD_JOBS_DIR / job_id
log_path = job_dir / "job.log"
try:
write_job_status(
job_dir,
{
"job_id": job_id,
"status": "running",
"stage": "mineru",
"message": "MinerU 正在解析上传文献",
"source_path": str(source_path),
"output_dir": str(output_dir),
},
)
content_list = source_path
if source_path.suffix.lower() != ".json":
mineru_out = job_dir / "mineru"
mineru_cmd, mineru_env, mineru_runtime = build_mineru_runtime()
preflight_error = mineru_gpu_preflight_error(
mineru_runtime["cuda_visible_devices"],
mineru_runtime["mode"],
mineru_runtime,
)
if preflight_error:
with log_path.open("a", encoding="utf-8") as handle:
handle.write(preflight_error + "\n")
raise RuntimeError(preflight_error)
cmd = build_mineru_command(mineru_cmd, source_path, mineru_out, mineru_runtime)
with log_path.open("a", encoding="utf-8") as handle:
handle.write(
"$ env "
f"METHODOLOGY_MINERU_RUNTIME_MODE={mineru_runtime['mode']} "
f"MINERU_MODEL_SOURCE={mineru_runtime['model_source']} "
f"CUDA_VISIBLE_DEVICES={mineru_runtime['cuda_visible_devices']} "
+ " ".join(cmd)
+ "\n"
)
handle.write("$ " + " ".join(cmd) + "\n")
handle.flush()
result = subprocess.run(
cmd,
cwd=PROJECT_ROOT,
env=mineru_env,
text=True,
stdout=handle,
stderr=subprocess.STDOUT,
)
if result.returncode != 0:
raise RuntimeError(summarize_mineru_failure(log_path, result.returncode))
matches = sorted(mineru_out.rglob("*_content_list.json"))
if not matches:
raise RuntimeError(f"MinerU finished but no *_content_list.json was found under {mineru_out}")
content_list = matches[0]
write_job_status(
job_dir,
{
"job_id": job_id,
"status": "running",
"stage": "llm",
"message": "LLM 正在提取方法学并与 bbav5 对齐",
"source_path": str(source_path),
"content_list": str(content_list),
"output_dir": str(output_dir),
},
)
papers_csv = job_dir / "methodology_papers.csv"
with papers_csv.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=["paper_id", "family", "source_paper_role", "parsed_document_path"])
writer.writeheader()
writer.writerow(
{
"paper_id": paper_id,
"family": family,
"source_paper_role": "primary_gene_family_paper",
"parsed_document_path": str(content_list),
}
)
batch_cmd = [
str(conda_env_path(os.environ.get("GF_PYTHON_ENV", "gfpy312")) / "bin" / "python"),
"scripts/run_methodology_batch.py",
"--papers",
str(papers_csv),
"--paper-id",
paper_id,
"--output-dir",
str(output_dir),
"--use-llm",
"--llm-model",
llm_model,
"--llm-request-timeout",
str(llm_request_timeout),
"--external-job-status-json",
str(job_dir / "status.json"),
]
with log_path.open("a", encoding="utf-8") as handle:
handle.write("$ " + " ".join(batch_cmd) + "\n")
handle.flush()
result = subprocess.run(batch_cmd, cwd=PROJECT_ROOT, text=True, stdout=handle, stderr=subprocess.STDOUT)
if result.returncode != 0:
raise RuntimeError(f"LLM methodology extraction failed with exit code {result.returncode}; see {log_path}")
config = build_bbav5_methodology_config(output_dir)
figure_table_error = ""
figure_table_payload: dict[str, Any] = {}
try:
figure_table_payload = build_figure_table_outputs(
content_list_path=content_list,
output_dir=output_dir,
methodology_config=config,
)
config = build_bbav5_methodology_config(output_dir)
except Exception as exc: # keep the core methodology extraction result usable.
figure_table_error = str(exc)
preview = config.get("reproduction_plan_preview", {}) if isinstance(config, dict) else {}
sandbox_payload: dict[str, Any] = {}
sandbox_warning = ""
if sandbox:
try:
result = build_methodology_qa_sandbox(
output_dir,
sandbox_root=DEFAULT_SANDBOX_ROOT,
run_id=sandbox_run_id or f"upload_{sanitize_token(job_id)}",
include_figures=False,
)
report = result.get("report", {})
registry = result.get("registry", {})
sandbox_payload = {
"methodology_sandbox_dir": report.get("sandbox_dir", ""),
"methodology_sandbox_registry": report.get("registry_path", ""),
"methodology_sandbox_collection": registry.get("qdrant_collection", ""),
"methodology_sandbox_status": "built",
"methodology_sandbox_card_count": report.get("stage_card_count", 0) + report.get("plan_card_count", 0),
"methodology_sandbox_relation_count": report.get("relation_count", 0),
}
except Exception as exc:
sandbox_warning = str(exc)
sandbox_payload = {
"methodology_sandbox_status": "failed",
"methodology_sandbox_warning": sandbox_warning,
}
write_job_status(
job_dir,
{
"job_id": job_id,
"status": "completed",
"stage": "completed",
"substage": "done",
"message": "MinerU+LLM 方法学提取完成",
"source_path": str(source_path),
"content_list": str(content_list),
"output_dir": str(output_dir),
"paper_id": paper_id,
"family": family,
"stage_count": len(config.get("stages", [])),
"species_count": len(config.get("species", [])),
"figure_table_caption_count": len(config.get("figure_table_inventory", [])),
"figure_table_stage_count": len(preview.get("suggested_stages", [])) if isinstance(preview, dict) else 0,
"figure_table_warning": figure_table_error,
"figure_table_files": figure_table_payload.get("files", {}),
"config_url": f"/api/v1/methodology/extract-jobs/{job_id}/bbav5-config",
"methodology_plan_url": f"/api/v1/methodology/extract-jobs/{job_id}/plan",
"figure_table_inventory_url": f"/api/v1/methodology/extract-jobs/{job_id}/figure-table-inventory",
"reproduction_plan_preview_url": f"/api/v1/methodology/extract-jobs/{job_id}/reproduction-plan-preview",
**sandbox_payload,
},
)
except Exception as exc:
write_job_status(
job_dir,
{
"job_id": job_id,
"status": "failed",
"stage": "failed",
"message": str(exc),
"source_path": str(source_path),
"output_dir": str(output_dir),
"log_path": str(log_path),
},
)
@router.get("/status")
async def methodology_status():
summary = read_csv(BATCH_DIR / "summary.csv")
review = read_csv(BATCH_DIR / "review_template.csv")
return {
"papers_table": str(PAPERS_CSV),
"gold_table": str(GOLD_CSV),
"batch_dir": str(BATCH_DIR),
"batch_exists": BATCH_DIR.exists(),
"paper_count": len(read_csv(PAPERS_CSV)),
"summary_count": len(summary),
"review_row_count": len(review),
"files": {
"summary": str(BATCH_DIR / "summary.csv"),
"processing_status": str(BATCH_DIR / "processing_status.csv"),
"review_template": str(BATCH_DIR / "review_template.csv"),
"review_results": str(REVIEW_RESULTS_JSON),
"discovered_tools": str(BATCH_DIR / "discovered_tools.csv"),
"methodology_tool_audit": str(BATCH_DIR / "methodology_tool_audit.csv"),
"methodology_parameter_audit": str(BATCH_DIR / "methodology_parameter_audit.csv"),
"methodology_database_url_audit": str(BATCH_DIR / "methodology_database_url_audit.csv"),
"methodology_review_issues": str(BATCH_DIR / "methodology_review_issues.csv"),
"methodology_review_report": str(BATCH_DIR / "methodology_review_report.json"),
"progress": str(BATCH_DIR / "progress.jsonl"),
"run_log": str(BATCH_DIR / "methodology_batch_run.log"),
"species_summary": str(BATCH_DIR / "species_summary.csv"),
"observations": str(BATCH_DIR / "observations.json"),
"alignment": str(BATCH_DIR / "alignment.json"),
},
}
@router.get("/summary")
async def methodology_summary():
return {"rows": read_csv(BATCH_DIR / "summary.csv")}
@router.get("/review-template")
async def methodology_review_template():
return {"rows": read_csv(BATCH_DIR / "review_template.csv")}
@router.get("/processing-status")
async def methodology_processing_status():
return {"rows": read_csv(BATCH_DIR / "processing_status.csv")}
@router.get("/discovered-tools")
async def methodology_discovered_tools():
return {"rows": read_csv(BATCH_DIR / "discovered_tools.csv")}
@router.get("/tool-audit")
async def methodology_tool_audit():
return {"rows": read_csv(BATCH_DIR / "methodology_tool_audit.csv")}
@router.get("/parameter-audit")
async def methodology_parameter_audit():
return {"rows": read_csv(BATCH_DIR / "methodology_parameter_audit.csv")}
@router.get("/url-audit")
async def methodology_url_audit():
return {"rows": read_csv(BATCH_DIR / "methodology_database_url_audit.csv")}
@router.get("/auto-review-issues")
async def methodology_auto_review_issues():
return {"rows": read_csv(BATCH_DIR / "methodology_review_issues.csv")}
@router.get("/auto-review-report")
async def methodology_auto_review_report():
return read_json(BATCH_DIR / "methodology_review_report.json")
@router.get("/progress")
async def methodology_progress():
path = BATCH_DIR / "progress.jsonl"
if not path.exists():
return {"rows": []}
rows = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
return {"rows": rows}
@router.get("/run-log")
async def methodology_run_log():
path = BATCH_DIR / "methodology_batch_run.log"
if not path.exists():
return {"text": ""}
return {"text": path.read_text(encoding="utf-8")}
@router.get("/species-summary")
async def methodology_species_summary():
return {"rows": read_csv(BATCH_DIR / "species_summary.csv")}
@router.get("/review-items")
async def methodology_review_items():
rows = read_csv(BATCH_DIR / "review_template.csv")
saved = {row.get("observation_id"): row for row in read_review_results()}
for row in rows:
if row.get("observation_id") in saved:
row["saved_review"] = saved[row["observation_id"]]
return {"rows": rows, "saved_count": len(saved)}
@router.get("/review-results")
async def methodology_review_results():
return {"results": read_review_results()}
@router.post("/review-results")
async def save_methodology_review_results(payload: ReviewResultsPayload):
existing = {row.get("observation_id"): row for row in read_review_results()}
for item in payload.results:
existing[item.observation_id] = item.dict()
rows = list(existing.values())
REVIEW_RESULTS_JSON.parent.mkdir(parents=True, exist_ok=True)
REVIEW_RESULTS_JSON.write_text(json.dumps({"results": rows}, ensure_ascii=False, indent=2), encoding="utf-8")
write_csv(REVIEW_RESULTS_CSV, rows)
return {"saved_count": len(rows), "json_path": str(REVIEW_RESULTS_JSON), "csv_path": str(REVIEW_RESULTS_CSV)}
@router.get("/review-summary")
async def methodology_review_summary():
rows = read_review_results()
summary: dict[str, dict[str, int]] = {}
for field in [
"review_status",
"stage_review",
"tools_review",
"versions_review",
"parameters_review",
"evidence_review",
"bbav5_stage_review",
"bbav5_tools_review",
"support_status_review",
"bbav5_review",
]:
counts: dict[str, int] = {}
for row in rows:
value = str(row.get(field) or "<blank>")
counts[value] = counts.get(value, 0) + 1
summary[field] = counts
return {"reviewed_count": len(rows), "summary": summary}
@router.get("/alignment")
async def methodology_alignment():
return read_json(BATCH_DIR / "alignment.json")
@router.get("/bbav5-config")
async def methodology_bbav5_config(batch_dir: str = BATCH_DIR_NAME):
try:
return build_bbav5_methodology_config(resolve_batch_dir(batch_dir))
except FileNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@router.post("/extract-upload")
async def methodology_extract_upload(
background_tasks: BackgroundTasks,
file: UploadFile = File(...),
paper_id: str = Form("uploaded_paper"),
family: str = Form(""),
output_name: str = Form(""),
llm_model: str = Form("deepseek-v4-flash"),
llm_request_timeout: int = Form(75),
sandbox: bool = Form(False),
sandbox_run_id: str = Form(""),
):
job_id = uuid.uuid4().hex[:12]
clean_paper_id = "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in (paper_id or f"paper_{job_id}"))
output_base = output_name or f"methodology_batch_{clean_paper_id}"
clean_output_base = "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in output_base).strip("._-") or "methodology_batch"
clean_output = clean_output_base if clean_output_base.endswith(f"_{job_id}") else f"{clean_output_base}_{job_id}"
job_dir = UPLOAD_JOBS_DIR / job_id
upload_dir = job_dir / "upload"
upload_dir.mkdir(parents=True, exist_ok=True)
source_path = upload_dir / (file.filename or f"{clean_paper_id}.pdf")
source_path.write_bytes(await file.read())
output_dir = UPLOAD_OUTPUT_ROOT / clean_output
write_job_status(
job_dir,
{
"job_id": job_id,
"status": "queued",
"stage": "queued",
"message": "文献已上传,等待 MinerU+LLM 方法学提取",
"source_path": str(source_path),
"output_dir": str(output_dir),
"paper_id": clean_paper_id,
"family": family,
"methodology_sandbox_requested": sandbox,
},
)
background_tasks.add_task(
run_uploaded_methodology_job,
job_id,
source_path,
clean_paper_id,
family,
output_dir,
llm_model,
llm_request_timeout,
sandbox,
sandbox_run_id,
)
return {"success": True, "job_id": job_id, "status_url": f"/api/v1/methodology/extract-jobs/{job_id}"}
@router.get("/extract-jobs")
async def methodology_extract_jobs(limit: int = 50, status: str = ""):
rows = list_upload_job_statuses(limit)
if status:
rows = [row for row in rows if row.get("status") == status]
return {"jobs": rows, "count": len(rows)}
@router.get("/extract-jobs/{job_id}")
async def methodology_extract_job_status(job_id: str):
return read_job_status(job_id)
@router.get("/extract-jobs/{job_id}/bbav5-config")
async def methodology_extract_job_config(job_id: str):
status = read_job_status(job_id)
if status.get("status") != "completed":
raise HTTPException(status_code=409, detail=f"Extraction job is not completed: {status.get('status')}")
try:
return build_bbav5_methodology_config(status["output_dir"])
except FileNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@router.get("/extract-jobs/{job_id}/plan")
async def methodology_extract_job_plan(job_id: str):
status = read_job_status(job_id)
if status.get("status") != "completed":
raise HTTPException(status_code=409, detail=f"Extraction job is not completed: {status.get('status')}")
output_dir = Path(status.get("output_dir", ""))
plan_path = output_dir / "methodology_plan.json"
if plan_path.exists():
plan = ensure_methodology_execution_contract(read_json(plan_path))
plan_path.write_text(json.dumps(plan, ensure_ascii=False, indent=2), encoding="utf-8")
return plan
try:
plan = build_methodology_execution_plan(output_dir)
plan_path.write_text(json.dumps(plan, ensure_ascii=False, indent=2), encoding="utf-8")
return plan
except FileNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@router.get("/extract-jobs/{job_id}/figure-table-inventory")
async def methodology_extract_job_figure_table_inventory(job_id: str):
status = read_job_status(job_id)
output_dir = Path(status.get("output_dir", ""))
return {"rows": read_json(output_dir / "figure_table_inventory.json")}
@router.post("/sandbox/build")
async def methodology_sandbox_build(payload: MethodologySandboxBuildPayload):
try:
result = build_methodology_qa_sandbox(
payload.methodology_dir,
sandbox_root=payload.sandbox_root,
run_id=payload.run_id or None,
include_figures=payload.include_figures,
content_list_path=payload.content_list_path or None,
)
except FileNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except Exception as exc:
raise HTTPException(status_code=500, detail=str(exc)) from exc
return {
"success": True,
"report": result["report"],
"registry": result["registry"],
}
@router.post("/sandbox/answer")
async def methodology_sandbox_answer(payload: MethodologySandboxAnswerPayload):
registry_path = payload.registry_path.strip()
if not registry_path and payload.sandbox_dir.strip():
registry_path = str(Path(payload.sandbox_dir.strip()) / "methodology_plan_registry.sandbox.json")
if not registry_path:
raise HTTPException(status_code=400, detail="registry_path or sandbox_dir is required")
result = answer_from_sandbox(payload.question, registry_path)
if not result:
return {
"success": False,
"answer": "",
"sources": [],
"message": "No sandbox methodology plan matched the question.",
"match_status": "no_match",
}
return {
"success": result.get("success", True),
"answer": result["answer"],
"sources": result["sources"],
"notes": result.get("notes", []),
"plan_entry": result.get("plan_entry", {}),
"matched_plan": result.get("matched_plan", {}),
"match_status": result.get("match_status", ""),
"query_constraints": result.get("query_constraints", {}),
"candidate_plans": result.get("candidate_plans", []),
"mismatch_reasons": result.get("mismatch_reasons", []),
}
@router.get("/sandbox/example-batches")
async def methodology_sandbox_example_batches():
candidates = []
for path in PROJECT_ROOT.glob("methodology_batch_*"):
if path.is_dir() and (path / "review_template.csv").exists():
candidates.append(
{
"name": path.name,
"path": str(path),
"has_plan": (path / "methodology_plan.json").exists(),
"has_preview": (path / "reproduction_plan_preview.json").exists(),
}
)
candidates.sort(key=lambda item: item["name"])
return {
"success": True,
"sandbox_root": str(DEFAULT_SANDBOX_ROOT),
"batches": candidates,
}
@router.get("/extract-jobs/{job_id}/reproduction-plan-preview")
async def methodology_extract_job_reproduction_plan_preview(job_id: str):
status = read_job_status(job_id)
output_dir = Path(status.get("output_dir", ""))
return read_json(output_dir / "reproduction_plan_preview.json")
+1683
View File
File diff suppressed because it is too large Load Diff
+1099
View File
File diff suppressed because it is too large Load Diff
+174
View File
@@ -0,0 +1,174 @@
"""
RAG v10专用路由 - 使用苦荞SPL附表增强版索引
"""
import os
import sys
import json
import pickle
import numpy as np
from pathlib import Path
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from typing import List, Optional
import requests
from src.rag.project_config import PROJECT_ROOT
sys.path.insert(0, str(PROJECT_ROOT))
router = APIRouter(prefix="/v10")
# ========== 配置 ==========
OLLAMA_URL = "http://localhost:11434"
EMBEDDING_MODEL = "bge-m3:567m"
LLM_MODEL = "qwen3:0.6b"
# v10索引路径
V10_INDEX_PATH = PROJECT_ROOT / "paper" / "v10" / "rag_v10_faiss_final.index"
V10_CHUNKS_PATH = PROJECT_ROOT / "paper" / "v10" / "rag_v10_chunks_with_summary.json"
V10_ENHANCED_PATH = PROJECT_ROOT / "paper" / "v10" / "rag_v10_chunks_enhanced.json"
# 全局变量
_v10_index = None
_v10_chunks = None
def get_embedding(text):
"""获取文本向量"""
response = requests.post(
f"{OLLAMA_URL}/api/embeddings",
json={"model": EMBEDDING_MODEL, "prompt": text}
)
vec = np.array(response.json()['embedding'], dtype=np.float32)
return vec / np.linalg.norm(vec)
def load_v10_index():
"""加载v10索引"""
global _v10_index, _v10_chunks
if _v10_index is not None:
return _v10_index, _v10_chunks
print(f"[V10 RAG] 加载v10索引...")
# 加载FAISS索引
import faiss
_v10_index = faiss.read_index(str(V10_INDEX_PATH))
print(f"[V10 RAG] 索引加载完成: {_v10_index.ntotal} 个向量")
# 加载文本块
with open(V10_CHUNKS_PATH, 'r') as f:
chunks1 = json.load(f)
with open(V10_ENHANCED_PATH, 'r') as f:
chunks2 = json.load(f)
_v10_chunks = chunks1 + chunks2
print(f"[V10 RAG] 文本块加载完成: {len(_v10_chunks)} 个")
return _v10_index, _v10_chunks
# ========== 请求/响应模型 ==========
class V10RAGQueryRequest(BaseModel):
query: str
top_k: int = 3
use_llm: bool = True
class V10RAGSource(BaseModel):
chunk_id: str
chunk_type: str
content: str
score: float
class V10RAGResponse(BaseModel):
query: str
answer: str
sources: List[V10RAGSource]
total_chunks: int
# ========== API端点 ==========
@router.post("/chat", response_model=V10RAGResponse)
async def v10_rag_chat(request: V10RAGQueryRequest):
"""V10专用RAG查询"""
try:
# 加载索引
index, chunks = load_v10_index()
# 1. 查询向量化
query_vec = get_embedding(request.query).reshape(1, -1)
# 2. FAISS检索
distances, indices = index.search(query_vec, request.top_k)
# 3. 构建sources
sources = []
context_parts = []
for i, (dist, idx) in enumerate(zip(distances[0], indices[0])):
chunk = chunks[idx]
sources.append(V10RAGSource(
chunk_id=chunk['id'],
chunk_type=chunk['metadata']['type'],
content=chunk['text'],
score=float(dist)
))
context_parts.append(f"[{i+1}] {chunk['text']}")
# 4. LLM生成(可选)
if request.use_llm:
context = "\n\n".join(context_parts)
prompt = f"""你是基因家族分析助手,基于以下资料回答问题。
请准确回答,并标注引用来源(如[1]、[2]等)。
参考资料:
{context}
用户问题:{request.query}
请基于以上资料回答:"""
response = requests.post(
f"{OLLAMA_URL}/api/generate",
json={
"model": LLM_MODEL,
"prompt": prompt,
"stream": False,
"options": {"temperature": 0.3}
}
)
answer = response.json()['response']
else:
# 直接返回检索结果
answer = "\n\n".join([f"[{i+1}] {s.content}" for i, s in enumerate(sources)])
return V10RAGResponse(
query=request.query,
answer=answer,
sources=sources,
total_chunks=len(chunks)
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"V10 RAG查询失败: {str(e)}")
@router.get("/info")
async def v10_info():
"""获取v10索引信息"""
try:
index, chunks = load_v10_index()
# 统计
type_counts = {}
for chunk in chunks:
t = chunk['metadata']['type']
type_counts[t] = type_counts.get(t, 0) + 1
return {
"index_type": "FAISS v10专用",
"total_chunks": len(chunks),
"embedding_model": EMBEDDING_MODEL,
"llm_model": LLM_MODEL,
"chunk_types": type_counts,
"data_source": "苦荞SPL附表增强版"
}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
print("[V10 RAG] 路由加载完成")
+372
View File
@@ -0,0 +1,372 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
RAPTOR层次索引API路由
"""
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from typing import List, Dict, Optional
from pathlib import Path
from src.rag.project_config import PROJECT_ROOT
router = APIRouter()
# 全局RAPTOR树实例缓存
_raptor_trees: Dict[str, any] = {}
class BuildTreeRequest(BaseModel):
doc_id: str
chunks: List[Dict]
max_levels: int = 3
summarization_ratio: float = 0.3
class BuildTreeResponse(BaseModel):
status: str
root_id: str
total_nodes: int
level_stats: Dict
message: str
class RaptorSearchRequest(BaseModel):
query: str
doc_id: Optional[str] = None # None表示搜索所有树
top_k: int = 5
strategy: str = "top_down" # top_down, cross_level, leaf_only
min_level: Optional[int] = None
max_level: Optional[int] = None
class RaptorSearchResponse(BaseModel):
query: str
results: List[Dict]
total_results: int
strategy: str
levels_covered: List[str]
class TreeStatsResponse(BaseModel):
doc_id: str
total_nodes: int
root_id: str
level_stats: Dict
tree_depth: int
class MultiLevelSearchRequest(BaseModel):
query: str
doc_id: Optional[str] = None
levels: List[int] = [0, 1, 2]
class MultiLevelSearchResponse(BaseModel):
query: str
results_by_level: Dict[str, List[Dict]]
total_results: int
@router.post("/raptor/build", response_model=BuildTreeResponse)
async def build_raptor_tree(request: BuildTreeRequest):
"""构建RAPTOR树"""
try:
from src.rag.advanced.raptor import RaptorTreeBuilder
builder = RaptorTreeBuilder(
max_levels=request.max_levels,
summarization_ratio=request.summarization_ratio
)
root_id = builder.build_tree(
chunks=request.chunks,
doc_id=request.doc_id
)
# 缓存树
_raptor_trees[request.doc_id] = builder
# 获取统计
stats = builder.get_tree_stats()
return BuildTreeResponse(
status="success",
root_id=root_id,
total_nodes=stats['total_nodes'],
level_stats=stats['levels'],
message=f"RAPTOR树构建成功: {request.doc_id}"
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"构建树失败: {str(e)}")
@router.post("/raptor/search", response_model=RaptorSearchResponse)
async def search_raptor(request: RaptorSearchRequest):
"""RAPTOR层次检索"""
try:
from src.rag.advanced.raptor import RaptorRetriever
if not _raptor_trees:
raise HTTPException(status_code=404, detail="没有可用的RAPTOR树,请先构建")
all_results = []
levels_covered = set()
# 搜索指定的树或所有树
trees_to_search = [request.doc_id] if request.doc_id else list(_raptor_trees.keys())
for doc_id in trees_to_search:
if doc_id not in _raptor_trees:
continue
builder = _raptor_trees[doc_id]
retriever = RaptorRetriever(builder)
results = retriever.search(
query=request.query,
top_k=request.top_k,
strategy=request.strategy,
min_level=request.min_level,
max_level=request.max_level
)
for result in results:
all_results.append({
'node_id': result.node.node_id,
'level': result.level,
'score': result.score,
'content': result.node.content[:500],
'summary': result.node.summary,
'is_summary': result.is_summary,
'path': result.path.path,
'doc_id': doc_id
})
levels_covered.add(result.level)
# 排序
all_results.sort(key=lambda x: x['score'], reverse=True)
return RaptorSearchResponse(
query=request.query,
results=all_results[:request.top_k],
total_results=len(all_results),
strategy=request.strategy,
levels_covered=list(levels_covered)
)
except HTTPException:
raise
except Exception as e:
raise HTTPException(status_code=500, detail=f"检索失败: {str(e)}")
@router.post("/raptor/search/multi-level", response_model=MultiLevelSearchResponse)
async def search_multi_level(request: MultiLevelSearchRequest):
"""多层级检索"""
try:
from src.rag.advanced.raptor import RaptorRetriever
if not _raptor_trees:
raise HTTPException(status_code=404, detail="没有可用的RAPTOR树")
results_by_level = {}
total = 0
trees_to_search = [request.doc_id] if request.doc_id else list(_raptor_trees.keys())
for doc_id in trees_to_search:
if doc_id not in _raptor_trees:
continue
builder = _raptor_trees[doc_id]
retriever = RaptorRetriever(builder)
multi_level_results = retriever.get_multi_level_results(
query=request.query,
levels=request.levels
)
for level_num, results in multi_level_results.items():
level_key = f"level_{level_num}"
if level_key not in results_by_level:
results_by_level[level_key] = []
for result in results:
results_by_level[level_key].append({
'node_id': result.node.node_id,
'level': result.level,
'score': result.score,
'content': result.node.content[:300],
'summary': result.node.summary,
'doc_id': doc_id
})
total += 1
return MultiLevelSearchResponse(
query=request.query,
results_by_level=results_by_level,
total_results=total
)
except HTTPException:
raise
except Exception as e:
raise HTTPException(status_code=500, detail=f"多层级检索失败: {str(e)}")
@router.get("/raptor/tree/{doc_id}/stats", response_model=TreeStatsResponse)
async def get_tree_stats(doc_id: str):
"""获取树统计信息"""
try:
if doc_id not in _raptor_trees:
raise HTTPException(status_code=404, detail=f"树不存在: {doc_id}")
builder = _raptor_trees[doc_id]
stats = builder.get_tree_stats()
# 计算树深度
max_height = 0
for node in builder.nodes.values():
max_height = max(max_height, node.get_height())
return TreeStatsResponse(
doc_id=doc_id,
total_nodes=stats['total_nodes'],
root_id=stats['root_id'],
level_stats=stats['levels'],
tree_depth=max_height
)
except HTTPException:
raise
except Exception as e:
raise HTTPException(status_code=500, detail=f"获取统计失败: {str(e)}")
@router.get("/raptor/tree/{doc_id}/structure")
async def get_tree_structure(doc_id: str):
"""获取树结构"""
try:
if doc_id not in _raptor_trees:
raise HTTPException(status_code=404, detail=f"树不存在: {doc_id}")
builder = _raptor_trees[doc_id]
# 构建树结构
def build_node_tree(node_id: str) -> Dict:
node = builder.get_node(node_id)
if not node:
return {}
children = []
for child_id in node.children_ids:
child_tree = build_node_tree(child_id)
if child_tree:
children.append(child_tree)
return {
'id': node.node_id,
'level': node.level.value,
'summary': node.summary[:200] if node.summary else node.content[:200],
'children': children
}
structure = build_node_tree(builder.root_id)
return {
'doc_id': doc_id,
'root_id': builder.root_id,
'structure': structure
}
except HTTPException:
raise
except Exception as e:
raise HTTPException(status_code=500, detail=f"获取结构失败: {str(e)}")
@router.post("/raptor/tree/{doc_id}/save")
async def save_tree(doc_id: str, filepath: Optional[str] = None):
"""保存树到文件"""
try:
if doc_id not in _raptor_trees:
raise HTTPException(status_code=404, detail=f"树不存在: {doc_id}")
builder = _raptor_trees[doc_id]
if not filepath:
filepath = str(PROJECT_ROOT / "data" / "raptor" / f"{doc_id}_tree.json")
builder.save(filepath)
return {
'status': 'success',
'doc_id': doc_id,
'filepath': filepath
}
except HTTPException:
raise
except Exception as e:
raise HTTPException(status_code=500, detail=f"保存失败: {str(e)}")
@router.post("/raptor/tree/load")
async def load_tree(filepath: str):
"""从文件加载树"""
try:
from src.rag.advanced.raptor import RaptorTreeBuilder
builder = RaptorTreeBuilder.load(filepath)
# 从文件路径提取doc_id
doc_id = Path(filepath).stem.replace('_tree', '')
_raptor_trees[doc_id] = builder
return {
'status': 'success',
'doc_id': doc_id,
'root_id': builder.root_id,
'total_nodes': len(builder.nodes)
}
except Exception as e:
raise HTTPException(status_code=500, detail=f"加载失败: {str(e)}")
@router.get("/raptor/trees")
async def list_trees():
"""列出所有可用的树"""
return {
'trees': [
{
'doc_id': doc_id,
'root_id': builder.root_id,
'total_nodes': len(builder.nodes)
}
for doc_id, builder in _raptor_trees.items()
]
}
@router.delete("/raptor/tree/{doc_id}")
async def delete_tree(doc_id: str):
"""删除树"""
try:
if doc_id not in _raptor_trees:
raise HTTPException(status_code=404, detail=f"树不存在: {doc_id}")
del _raptor_trees[doc_id]
return {
'status': 'success',
'doc_id': doc_id,
'message': '树已删除'
}
except HTTPException:
raise
except Exception as e:
raise HTTPException(status_code=500, detail=f"删除失败: {str(e)}")
+58
View File
@@ -0,0 +1,58 @@
"""推理引擎API路由"""
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
router = APIRouter()
# 导入推理引擎
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent / 'scripts' / 'utils'))
try:
from reasoning_engine import ReasoningEngine
except ImportError:
# 备用导入方式
import importlib.util
reasoning_engine_path = Path(__file__).parent.parent / 'scripts' / 'utils' / 'reasoning_engine.py'
spec = importlib.util.spec_from_file_location("reasoning_engine", reasoning_engine_path)
reasoning_engine = importlib.util.module_from_spec(spec)
spec.loader.exec_module(reasoning_engine)
ReasoningEngine = reasoning_engine.ReasoningEngine
class ReasoningQuery(BaseModel):
question: str
@router.post("/reasoning/query")
async def reasoning_query(query: ReasoningQuery):
"""多跳推理查询"""
try:
engine = ReasoningEngine()
result = engine.multi_hop_query(query.question)
return {"status": "success", **result}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.get("/reasoning/examples")
async def reasoning_examples():
"""推理查询示例"""
return {
"status": "success",
"examples": [
{
"question": "查找与CYP71D184同组的基因",
"description": "多跳推理:基因→分组→同组基因",
"hops": 2
},
{
"question": "P450分组统计比较",
"description": "聚合统计:所有分组的统计分析",
"hops": 1
},
{
"question": "最大的分组是哪个",
"description": "极值查询:查找最大分组",
"hops": 1
}
]
}
Executable
+78
View File
@@ -0,0 +1,78 @@
"""
API路由 - 纯净版RAG系统
保留RAG核心接口,为bbav20311接入预留接口
"""
from fastapi import APIRouter, UploadFile, File, HTTPException
from pathlib import Path
import os
from api.rag_routes import router as rag_router
from api.case_routes import router as case_router
router = APIRouter()
# ============================================
# RAG核心接口(已实现)
# ============================================
# RAG对话接口
router.include_router(rag_router, prefix="/rag", tags=["RAG"])
# 真实复现案例问答接口(确定性QA,不调用LLM)
router.include_router(case_router, prefix="/case", tags=["Case QA"])
# ============================================
# 预留接口(为bbav20311接入准备)
# ============================================
# 文件上传接口(预留)
@router.post("/upload")
async def upload_file(file: UploadFile = File(...)):
"""文件上传接口 - 预留"""
raise HTTPException(status_code=501, detail="待bbav20311接入后实现")
# 批量上传接口(预留)
@router.post("/upload/batch")
async def upload_batch(files: list[UploadFile] = File(...)):
"""批量上传接口 - 预留"""
raise HTTPException(status_code=501, detail="待bbav20311接入后实现")
# 工作空间接口(预留)
@router.get("/workspace/{job_id}/files")
async def get_workspace_files(job_id: str):
"""工作空间文件接口 - 预留"""
raise HTTPException(status_code=501, detail="待bbav20311接入后实现")
# 基因家族分析接口(预留)
@router.post("/analysis/gene-family")
async def analyze_gene_family(job_id: str, species: str, gene_family: str):
"""基因家族分析接口 - 预留"""
raise HTTPException(status_code=501, detail="待bbav20311接入后实现")
# 工具执行接口(预留)
@router.post("/tools/{tool_id}/execute")
async def execute_tool(tool_id: str, params: dict):
"""工具执行接口 - 预留"""
raise HTTPException(status_code=501, detail="待bbav20311接入后实现")
# ============================================
# 状态查询接口
# ============================================
@router.get("/status")
async def get_status():
"""获取RAG系统状态"""
try:
from src.rag.retriever import retriever
indexed_count = retriever.count_chunks() if retriever else 0
except Exception as e:
# 如果retriever未初始化,使用RAG路由的状态
indexed_count = 0
return {
"status": "ready",
"indexed_documents": indexed_count,
"rag_enabled": True,
"analysis_enabled": False, # 待接入bbav20311后开启
}
Executable
+73
View File
@@ -0,0 +1,73 @@
from datetime import datetime
from typing import Any, Union
from pydantic import BaseModel
class FileInfo(BaseModel):
name: str
path: str
size: int | None = None
class QueryRequest(BaseModel):
session_id: str
query: str
files: list[Union[str, FileInfo]] = []
user_input: dict[str, Any] = {}
class QueryResponse(BaseModel):
session_id: str
status: str
mode: str
response: dict[str, Any] = {}
messages: list[dict[str, str]] = []
class SessionCreate(BaseModel):
user_id: str | None = None
class SessionResponse(BaseModel):
session_id: str
created_at: datetime
status: str = "active"
class CaseCreate(BaseModel):
case_id: str
user_id: str | None = None
gene_family: str | None = None
species: str | None = None
class CaseResponse(BaseModel):
case_id: str
user_id: str | None
gene_family: str | None
species: str | None
status: str
created_at: datetime
results: dict[str, Any] = {}
class FeedbackRequest(BaseModel):
case_id: str
rating: int
comments: str | None = None
issues: list[str] = []
class WebSocketMessage(BaseModel):
type: str
session_id: str
data: dict[str, Any] = {}
class ProgressUpdate(BaseModel):
session_id: str
current_agent: str | None
status: str
progress: float
message: str
+206
View File
@@ -0,0 +1,206 @@
"""
物种查询API路由
提供基因家族-物种索引查询功能
"""
from fastapi import APIRouter, HTTPException, Query
from pydantic import BaseModel
from typing import List, Dict, Optional
from pathlib import Path
import json
from src.rag.project_config import PROJECT_ROOT
router = APIRouter()
INDEX_FILE = PROJECT_ROOT / "data" / "gene_family_species_index.json"
class SpeciesInfo(BaseModel):
"""物种信息"""
paper_id: str
scientific_name: str
common_name_en: str
common_name_cn: str
comparison_species: List[str]
relations_count: int
entities_count: int
pmid: Optional[str]
class GeneFamilyInfo(BaseModel):
"""基因家族信息"""
gene_family: str
description: str
description_cn: str
total_species: int
species_list: List[SpeciesInfo]
class SpeciesListResponse(BaseModel):
"""物种列表响应"""
gene_family: str
total_species: int
species: List[SpeciesInfo]
class SpeciesQueryResponse(BaseModel):
"""物种查询响应"""
query: str
matched_species: List[Dict]
total_matches: int
def load_species_index() -> dict:
"""加载物种索引"""
if not INDEX_FILE.exists():
raise HTTPException(status_code=500, detail="物种索引文件不存在,请先运行索引构建脚本")
with open(INDEX_FILE, 'r', encoding='utf-8') as f:
return json.load(f)
@router.get("/species/list", response_model=SpeciesListResponse)
async def list_species_by_gene_family(
gene_family: str = Query("SPL", description="基因家族名称,如SPL、NAC、WRKY等")
):
"""
获取指定基因家族已分析的所有物种列表
示例:
- /api/v1/species/list?gene_family=SPL
"""
try:
index = load_species_index()
gene_family_upper = gene_family.upper()
if gene_family_upper not in index.get("gene_families", {}):
return SpeciesListResponse(
gene_family=gene_family,
total_species=0,
species=[]
)
gf_info = index["gene_families"][gene_family_upper]
# 构建响应
species_list = []
for species in gf_info.get("species_list", []):
species_list.append(SpeciesInfo(
paper_id=species["paper_id"],
scientific_name=species["scientific_name"],
common_name_en=species["common_name_en"],
common_name_cn=species["common_name_cn"],
comparison_species=species["comparison_species"],
relations_count=species["relations_count"],
entities_count=species["entities_count"],
pmid=species.get("pmid") or None
))
return SpeciesListResponse(
gene_family=gene_family_upper,
total_species=gf_info["total_species"],
species=species_list
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"查询失败: {str(e)}")
@router.get("/species/search", response_model=SpeciesQueryResponse)
async def search_species(
query: str = Query(..., description="物种查询关键词,支持中文、英文、拉丁学名")
):
"""
搜索特定物种是否做过基因家族分析
示例:
- /api/v1/species/search?query=苦荞
- /api/v1/species/search?query=tartary
- /api/v1/species/search?query=Fagopyrum
"""
try:
index = load_species_index()
query_lower = query.lower()
matches = []
# 遍历所有基因家族
for gf_name, gf_info in index.get("gene_families", {}).items():
for species in gf_info.get("species_list", []):
# 匹配多个字段
match_fields = [
species["scientific_name"].lower(),
species["common_name_en"].lower(),
species["common_name_cn"]
]
if any(query_lower in field for field in match_fields):
matches.append({
"gene_family": gf_name,
"paper_id": species["paper_id"],
"scientific_name": species["scientific_name"],
"common_name_en": species["common_name_en"],
"common_name_cn": species["common_name_cn"],
"pmid": species.get("pmid"),
"relations_count": species["relations_count"],
"entities_count": species["entities_count"]
})
return SpeciesQueryResponse(
query=query,
matched_species=matches,
total_matches=len(matches)
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"查询失败: {str(e)}")
@router.get("/species/gene-families", response_model=List[str])
async def list_gene_families():
"""
获取所有已索引的基因家族列表
示例:
- /api/v1/species/gene-families
"""
try:
index = load_species_index()
return list(index.get("gene_families", {}).keys())
except Exception as e:
raise HTTPException(status_code=500, detail=f"查询失败: {str(e)}")
@router.get("/species/stats")
async def get_statistics():
"""
获取基因家族-物种索引统计信息
示例:
- /api/v1/species/stats
"""
try:
index = load_species_index()
stats = {
"total_gene_families": len(index.get("gene_families", {})),
"total_papers": len(index.get("papers", {})),
"total_species_indexed": len(index.get("species_index", {})),
"metadata": index.get("metadata", {}),
"gene_families": {}
}
for gf_name, gf_info in index.get("gene_families", {}).items():
stats["gene_families"][gf_name] = {
"description_cn": gf_info.get("description_cn", ""),
"total_species": gf_info.get("total_species", 0),
"species_names": [s["common_name_cn"] for s in gf_info.get("species_list", [])]
}
return stats
except Exception as e:
raise HTTPException(status_code=500, detail=f"查询失败: {str(e)}")
+409
View File
@@ -0,0 +1,409 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
时序知识图谱API路由
"""
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from typing import List, Dict, Optional
router = APIRouter()
# 全局时序知识图谱实例
_temporal_kg = None
def get_temporal_kg():
"""获取时序知识图谱实例"""
global _temporal_kg
if _temporal_kg is None:
from src.rag.temporal import TemporalKnowledgeGraph
_temporal_kg = TemporalKnowledgeGraph(name="gene_family_temporal_kg")
return _temporal_kg
class CreateEntityRequest(BaseModel):
entity_id: str
name: str
entity_type: str
start_year: Optional[int] = None
end_year: Optional[int] = None
properties: Dict = {}
class CreateRelationRequest(BaseModel):
relation_id: str
source_id: str
target_id: str
relation_type: str
year: Optional[int] = None
properties: Dict = {}
class TimeRangeQueryRequest(BaseModel):
start_year: Optional[int] = None
end_year: Optional[int] = None
entity_types: List[str] = []
relation_types: List[str] = []
class TimelineQueryRequest(BaseModel):
entity_id: Optional[str] = None
topic: Optional[str] = None
start_year: Optional[int] = None
end_year: Optional[int] = None
@router.post("/temporal/entity")
async def create_entity(request: CreateEntityRequest):
"""创建时序实体"""
try:
from src.rag.temporal import TemporalEntity, TimePeriod, TimePoint
kg = get_temporal_kg()
# 构建时间范围
valid_time = None
if request.start_year:
valid_time = TimePeriod(
start=TimePoint.from_year(request.start_year),
end=TimePoint.from_year(request.end_year) if request.end_year else TimePoint.from_year(request.start_year)
)
entity = TemporalEntity(
entity_id=request.entity_id,
name=request.name,
entity_type=request.entity_type,
valid_time=valid_time,
properties=request.properties
)
kg.add_entity(entity)
return {
"status": "success",
"entity_id": request.entity_id,
"message": f"实体 {request.name} 创建成功"
}
except Exception as e:
raise HTTPException(status_code=500, detail=f"创建实体失败: {str(e)}")
@router.post("/temporal/relation")
async def create_relation(request: CreateRelationRequest):
"""创建时序关系"""
try:
from src.rag.temporal import TemporalRelation, TimePoint
kg = get_temporal_kg()
# 构建时间戳
timestamp = None
if request.year:
timestamp = TimePoint.from_year(request.year)
relation = TemporalRelation(
relation_id=request.relation_id,
source_id=request.source_id,
target_id=request.target_id,
relation_type=request.relation_type,
timestamp=timestamp,
properties=request.properties
)
kg.add_relation(relation)
return {
"status": "success",
"relation_id": request.relation_id,
"message": f"关系 {request.relation_type} 创建成功"
}
except Exception as e:
raise HTTPException(status_code=500, detail=f"创建关系失败: {str(e)}")
@router.post("/temporal/query/range")
async def query_by_time_range(request: TimeRangeQueryRequest):
"""按时间范围查询"""
try:
from src.rag.temporal import TimeRangeQuery, TimePoint
kg = get_temporal_kg()
query = TimeRangeQuery(
start=TimePoint.from_year(request.start_year) if request.start_year else None,
end=TimePoint.from_year(request.end_year) if request.end_year else None,
entity_types=request.entity_types,
relation_types=request.relation_types
)
result = kg.query(query)
return {
"query": query.to_dict(),
"entities": [e.to_dict() for e in result["entities"]],
"relations": [r.to_dict() for r in result["relations"]],
"total_entities": len(result["entities"]),
"total_relations": len(result["relations"])
}
except Exception as e:
raise HTTPException(status_code=500, detail=f"查询失败: {str(e)}")
@router.post("/temporal/timeline")
async def get_timeline(request: TimelineQueryRequest):
"""获取时间线"""
try:
kg = get_temporal_kg()
timeline = kg.get_timeline(
entity_id=request.entity_id,
topic=request.topic,
start_year=request.start_year,
end_year=request.end_year
)
return timeline.to_dict()
except Exception as e:
raise HTTPException(status_code=500, detail=f"获取时间线失败: {str(e)}")
@router.get("/temporal/timeline/visualization")
async def get_timeline_visualization(
topic: Optional[str] = None,
start_year: Optional[int] = None,
end_year: Optional[int] = None
):
"""获取时间线可视化数据"""
try:
kg = get_temporal_kg()
visual_data = kg.visualize_timeline(topic=topic)
return {
"topic": topic,
"start_year": start_year,
"end_year": end_year,
"visualization_data": visual_data
}
except Exception as e:
raise HTTPException(status_code=500, detail=f"获取可视化数据失败: {str(e)}")
@router.get("/temporal/stats")
async def get_stats():
"""获取统计信息"""
try:
kg = get_temporal_kg()
stats = kg.get_stats()
return stats.to_dict()
except Exception as e:
raise HTTPException(status_code=500, detail=f"获取统计失败: {str(e)}")
@router.get("/temporal/entity/{entity_id}")
async def get_entity(entity_id: str):
"""获取实体详情"""
try:
kg = get_temporal_kg()
entity = kg.get_entity(entity_id)
if not entity:
raise HTTPException(status_code=404, detail=f"实体不存在: {entity_id}")
return entity.to_dict()
except HTTPException:
raise
except Exception as e:
raise HTTPException(status_code=500, detail=f"获取实体失败: {str(e)}")
@router.get("/temporal/entity/{entity_id}/history")
async def get_entity_history(entity_id: str):
"""获取实体历史"""
try:
kg = get_temporal_kg()
timeline = kg.get_timeline(entity_id=entity_id)
return timeline.to_dict()
except Exception as e:
raise HTTPException(status_code=500, detail=f"获取历史失败: {str(e)}")
@router.get("/temporal/entities/by-type/{entity_type}")
async def get_entities_by_type(entity_type: str):
"""按类型获取实体"""
try:
kg = get_temporal_kg()
entities = kg.get_entities_by_type(entity_type)
return {
"entity_type": entity_type,
"entities": [e.to_dict() for e in entities],
"total": len(entities)
}
except Exception as e:
raise HTTPException(status_code=500, detail=f"获取实体失败: {str(e)}")
@router.get("/temporal/entities/by-year/{year}")
async def get_entities_by_year(year: int):
"""按年份获取实体"""
try:
kg = get_temporal_kg()
entities = kg.get_entities_by_year(year)
return {
"year": year,
"entities": [e.to_dict() for e in entities],
"total": len(entities)
}
except Exception as e:
raise HTTPException(status_code=500, detail=f"获取实体失败: {str(e)}")
@router.post("/temporal/consistency/check")
async def check_consistency():
"""检查图谱一致性"""
try:
kg = get_temporal_kg()
conflicts = kg.check_consistency()
return {
"consistent": len(conflicts) == 0,
"conflicts": conflicts,
"total_conflicts": len(conflicts)
}
except Exception as e:
raise HTTPException(status_code=500, detail=f"检查一致性失败: {str(e)}")
@router.post("/temporal/infer")
async def infer_relations():
"""推断新关系"""
try:
kg = get_temporal_kg()
inferred = kg.infer_relations()
return {
"inferred_relations": [r.to_dict() for r in inferred],
"total_inferred": len(inferred)
}
except Exception as e:
raise HTTPException(status_code=500, detail=f"推断关系失败: {str(e)}")
@router.post("/temporal/save")
async def save_graph(filepath: str = "/tmp/temporal_kg.json"):
"""保存图谱"""
try:
kg = get_temporal_kg()
kg.save(filepath)
return {
"status": "success",
"filepath": filepath
}
except Exception as e:
raise HTTPException(status_code=500, detail=f"保存失败: {str(e)}")
@router.post("/temporal/load")
async def load_graph(filepath: str = "/tmp/temporal_kg.json"):
"""加载图谱"""
try:
global _temporal_kg
from src.rag.temporal import TemporalKnowledgeGraph
_temporal_kg = TemporalKnowledgeGraph.load(filepath)
return {
"status": "success",
"filepath": filepath,
"name": _temporal_kg.name
}
except Exception as e:
raise HTTPException(status_code=500, detail=f"加载失败: {str(e)}")
@router.post("/temporal/demo/populate")
async def populate_demo_data():
"""填充演示数据"""
try:
from src.rag.temporal import (
TemporalEntity, TemporalRelation,
TimePoint, TimePeriod
)
kg = get_temporal_kg()
# 添加演示实体
demo_entities = [
("spl_arabidopsis", "SPL基因家族(拟南芥)", "gene_family", 2018, 2023),
("spl_rice", "SPL基因家族(水稻)", "gene_family", 2019, 2023),
("spl_poplar", "SPL基因家族(杨树)", "gene_family", 2020, 2023),
("spl3", "SPL3基因", "gene", 2018, None),
("spl9", "SPL9基因", "gene", 2019, None),
("flowering", "开花调控", "process", 2018, 2023),
("stress_response", "胁迫响应", "process", 2020, 2023),
]
for entity_id, name, entity_type, start_year, end_year in demo_entities:
valid_time = TimePeriod(
start=TimePoint.from_year(start_year),
end=TimePoint.from_year(end_year) if end_year else TimePoint.from_year(2023)
)
entity = TemporalEntity(
entity_id=entity_id,
name=name,
entity_type=entity_type,
valid_time=valid_time,
properties={"topic": "SPL基因家族"}
)
kg.add_entity(entity)
# 添加演示关系
demo_relations = [
("rel1", "spl3", "flowering", "regulates", 2019),
("rel2", "spl9", "flowering", "regulates", 2020),
("rel3", "spl_arabidopsis", "stress_response", "involved_in", 2021),
("rel4", "spl3", "spl9", "interacts_with", 2020),
]
for rel_id, source, target, rel_type, year in demo_relations:
relation = TemporalRelation(
relation_id=rel_id,
source_id=source,
target_id=target,
relation_type=rel_type,
timestamp=TimePoint.from_year(year),
properties={}
)
kg.add_relation(relation)
return {
"status": "success",
"entities_added": len(demo_entities),
"relations_added": len(demo_relations),
"message": "演示数据已填充"
}
except Exception as e:
raise HTTPException(status_code=500, detail=f"填充演示数据失败: {str(e)}")
+149
View File
@@ -0,0 +1,149 @@
from __future__ import annotations
import os
from fastapi import APIRouter
from pydantic import BaseModel, Field
router = APIRouter(prefix="/v15-agentic-qa", tags=["V15 Agentic QA"])
_retriever = None
_coordinator = None
_memory = None
_mem_ok = False
def _get_retriever():
global _retriever
if _retriever is None:
from src.rag.v15_hybrid_retriever import V15HybridRetriever
_retriever = V15HybridRetriever()
return _retriever
def _get_coordinator():
global _coordinator
if _coordinator is None:
from src.rag.agentic_coordinator import AgenticCoordinator
_coordinator = AgenticCoordinator(
retriever=_get_retriever(), max_iterations=3, top_k_base=15)
return _coordinator
def _get_memory():
global _memory, _mem_ok
if _memory is None and not _mem_ok:
try:
from src.rag.memory.memory_manager import MemoryManager
_memory = MemoryManager(user_id="agentic", session_id="qa")
_mem_ok = True
except Exception as e:
print(f"[V15 Agentic] Memory disabled: {e}")
_mem_ok = True
return _memory
class AgenticQARequest(BaseModel):
question: str = Field(..., description="Question")
top_k: int = Field(15, ge=1, le=50)
max_hops: int | None = Field(None)
include_candidate: bool = Field(False)
use_llm: bool = Field(True)
provider: str | None = Field(None)
model: str | None = Field(None)
anchor_entity: str | None = Field(None)
no_memory: bool = Field(False)
@router.post("/chat")
async def agentic_qa_chat(req: AgenticQARequest):
coordinator = _get_coordinator()
memory = _get_memory()
question = req.question
mem_note = ""
if memory and not req.no_memory:
try:
enriched = memory.augment_query(question)
if enriched != question:
question = enriched
mem_note = "memory_augmented"
except:
pass
result = await coordinator.run(
question=question, top_k=req.top_k,
anchor_entity=req.anchor_entity or None,
include_candidate=req.include_candidate)
evidence = result["retrieved_evidence"]
gpaths = result["graph_paths"]
aplan = result["retrieval_plan"]
evres = result["evaluation"]
notes = result["notes"]
if req.max_hops is not None:
aplan["max_hops"] = req.max_hops
citations = []
for idx, ev in enumerate(evidence[:30]):
citations.append({
"id": f"S{idx+1}",
"source": ev.get("source", ""),
"relation": ev.get("relation", ""),
"target": ev.get("target", ""),
"evidence": ev.get("evidence", ""),
"paper_id": ev.get("paper_id", ""),
"pubmed_id": ev.get("pubmed_id", ""),
"section": ev.get("section", ""),
})
answer = ""
llm_model = ""
if req.use_llm and evidence:
try:
from api.kb_qa_routes import _generate_llm_answer
answer = await _generate_llm_answer(
req.question, aplan.get("layer", "auto"),
evidence, provider=req.provider, model=req.model)
p = req.provider or os.environ.get("V15_KB_QA_LLM_PROVIDER", "")
m = req.model or os.environ.get("V15_KB_QA_LLM_MODEL", "")
llm_model = f"{p}:{m}"
except Exception as exc:
answer = f"[agentic] LLM failed: {exc}"
if not answer and not evidence:
answer = "Insufficient evidence."
if memory and not req.no_memory and answer:
try: memory.add_interaction(req.question, answer)
except: pass
return {
"question": req.question,
"answer": answer,
"citations": citations,
"graph_paths": gpaths,
"agentic_plan": aplan,
"evaluation": evres,
"retrieval_notes": notes,
"evidence_count": len(evidence),
"graph_paths_count": len(gpaths),
"llm_model": llm_model,
"memory_note": mem_note,
"mode": "v15_agentic_qa",
}
@router.get("/info")
async def agentic_qa_info():
return {
"mode": "v15_agentic_qa",
"components": {
"planner": "QueryAnalyzer + RetrievalStrategyAgent",
"evaluator": "Self-evaluation (count/confidence/entity)",
"retriever": "V15HybridRetriever (Qdrant+Neo4j)",
"memory": "MemoryManager",
},
"loop": "Max 3 iterations with self-evaluation",
"question_types": {
"FAQ": "hybrid mode, 2 hops",
"METHOD_CONSULT": "method_relations layer, 1 hop",
"ENTITY_QUERY": "exact+vector+KG, 1 hop",
"RELATION_QUERY": "KG+vector, 3 hops",
"UNKNOWN": "hybrid mode, 2 hops",
},
}
print("[V15 Agentic QA] Routes loaded with AgenticCoordinator")
+192
View File
@@ -0,0 +1,192 @@
"""
V15 SRT Knowledge Base --- Neo4j Graph QA Routes
"""
from __future__ import annotations
import os, re as _re
from typing import Any, Dict, List, Optional
from fastapi import APIRouter, HTTPException, Query
from neo4j import GraphDatabase
from pydantic import BaseModel, Field
router = APIRouter(prefix="/v15-graph-qa", tags=["V15 Neo4j Graph QA"])
_neo4j_driver = None
NEO4J_URI = os.environ.get("NEO4J_URI", "bolt://localhost:7687")
NEO4J_USER = os.environ.get("NEO4J_USER", "neo4j")
NEO4J_PASSWORD = os.environ.get("NEO4J_PASSWORD", "password")
VALID_LAYERS = {"main_kg", "background_relations", "method_relations", "dataset_metadata", "candidate_review"}
LAYER_PRIORITY = {"main_kg": 0, "method_relations": 1, "dataset_metadata": 2, "background_relations": 3, "candidate_review": 4}
SKIP_WORDS = {"What","Which","Where","When","How","Does","Did","Are","Was","Were","The","And","For","With","From","Into","About","That","This","These","Those","Tell","Please","List","Show","Give","Find","Would","Could","Should","May","Might","Can","Only","Also","Both","Each"}
def _get_driver():
global _neo4j_driver
if _neo4j_driver is None:
_neo4j_driver = GraphDatabase.driver(NEO4J_URI, auth=(NEO4J_USER, NEO4J_PASSWORD))
return _neo4j_driver
def _layer_cond(alias, allow_list, inc_cand):
if allow_list:
valid = [l for l in allow_list if l in VALID_LAYERS] or ["main_kg"]
else:
valid = ["main_kg", "background_relations", "method_relations", "dataset_metadata"]
if not inc_cand:
valid = [l for l in valid if l != "candidate_review"]
quoted = ", ".join(f"'{l}'" for l in valid)
return f"{alias}.output_layer IN [{quoted}]"
def _record_to_dict(rec, ea="r", sa="s", ta="t"):
e = rec[ea]; s = rec[sa]; t = rec[ta]
return {"source": s.get("name",""), "source_type": s.get("entity_type",""),
"relation": e.get("relation",""), "target": t.get("name",""),
"target_type": t.get("entity_type",""),
"evidence": str(e.get("evidence","") or ""),
"section": str(e.get("section","") or ""),
"pubmed_id": str(e.get("pubmed_id","") or ""),
"paper_id": str(e.get("paper_id","") or ""),
"confidence": e.get("confidence"),
"claim_scope": str(e.get("claim_scope","") or ""),
"output_layer": str(e.get("output_layer","") or ""),
"candidate_status": str(e.get("candidate_status","") or "")}
class PReq(BaseModel):
source_entity: str; target_entity: str
max_hops: int = Field(2, ge=1, le=5)
layers: Optional[List[str]] = Field(None)
include_candidate: bool = False
top_k_paths: int = Field(10, ge=1, le=50)
class GQAReq(BaseModel):
question: str
top_k: int = Field(10, ge=1, le=50)
layers: Optional[List[str]] = Field(None)
include_candidate: bool = False
use_llm: bool = True
provider: Optional[str] = Field(None)
model: Optional[str] = Field(None)
@router.get("/info")
async def info():
driver = _get_driver()
with driver.session() as s:
ec = s.run("MATCH (e:Entity) RETURN count(e) AS c").single()["c"]
rc = s.run("MATCH ()-[r:RELATION]->() RETURN count(r) AS c").single()["c"]
# Prefer canonical Paper nodes for paper count. Distinct r.paper_id can be
# smaller if a selected paper has no retained relation, or inflated in old
# imports that used PMID/PMCID as paper_id.
try:
pc = s.run("MATCH (p:Paper) RETURN count(p) AS c").single()["c"]
except Exception:
pc = s.run("MATCH ()-[r:RELATION]->() RETURN count(DISTINCT r.paper_id) AS c").single()["c"]
lc = {}
for row in s.run("MATCH ()-[r:RELATION]->() RETURN r.output_layer AS l, count(*) AS c"):
lc[row["l"]] = row["c"]
return {"graph":"V15 SRT KB (Neo4j)", "entity_count":ec, "relation_count":rc,
"paper_count":pc, "layers":lc,
"schema":"(:Entity)-[r:RELATION]->(:Entity) with r.relation"}
@router.get("/entity/{name}/neighborhood")
async def nbhd(name: str, depth: int=Query(1,ge=1,le=3),
layers: Optional[str]=Query(None),
include_candidate: bool=Query(False),
top_k: int=Query(30,ge=1,le=200)):
allowed = layers.split(",") if layers else None
driver = _get_driver()
seen, results = set(), []
with driver.session() as s:
lc = _layer_cond("r", allowed, include_candidate)
q = f"MATCH (s:Entity)-[r:RELATION]->(t:Entity) WHERE toLower(s.name) CONTAINS toLower($n) AND {lc} RETURN s,r,t ORDER BY r.confidence DESC LIMIT $lim"
for rec in s.run(q, n=name, lim=top_k):
k = (rec["s"].get("name"),rec["r"].get("relation"),rec["t"].get("name"),rec["r"].get("paper_id",""))
if k not in seen: seen.add(k); results.append(_record_to_dict(rec))
qr = f"MATCH (t:Entity)<-[r:RELATION]-(s:Entity) WHERE toLower(t.name) CONTAINS toLower($n) AND {lc} RETURN s,r,t ORDER BY r.confidence DESC LIMIT $lim"
for rec in s.run(qr, n=name, lim=top_k):
k = (rec["s"].get("name"),rec["r"].get("relation"),rec["t"].get("name"),rec["r"].get("paper_id",""))
if k not in seen: seen.add(k); results.append(_record_to_dict(rec))
if depth >= 2:
nbrs = set(r["source"] for r in results) | set(r["target"] for r in results)
nbrs.discard(name)
for nb in list(nbrs)[:20]:
lc2 = _layer_cond("r2", allowed, include_candidate)
q2 = f"MATCH (s2:Entity)-[r2:RELATION]->(t2:Entity) WHERE toLower(s2.name)=toLower($nb) AND {lc2} RETURN s2,r2,t2 ORDER BY r2.confidence DESC LIMIT 10"
for rec in s.run(q2, nb=nb):
k = (rec["s2"].get("name"),rec["r2"].get("relation"),rec["t2"].get("name"),rec["r2"].get("paper_id",""))
if k not in seen: seen.add(k); results.append(_record_to_dict(rec))
return {"entity":name, "depth":depth, "relations_count":len(results), "relations":results}
@router.post("/path")
async def path(req: PReq):
driver = _get_driver(); paths = []
with driver.session() as s:
q = f"MATCH p=(s:Entity)-[r:RELATION*1..{req.max_hops}]-(t:Entity) WHERE toLower(s.name) CONTAINS toLower($src) AND toLower(t.name) CONTAINS toLower($tgt) AND s<>t RETURN p ORDER BY length(p) ASC LIMIT $lim"
for rec in s.run(q, src=req.source_entity, tgt=req.target_entity, lim=req.top_k_paths):
es = []
for rel in rec["p"].relationships:
es.append({"source":rel.nodes[0].get("name",""),
"source_type":rel.nodes[0].get("entity_type",""),
"target":rel.nodes[-1].get("name",""),
"target_type":rel.nodes[-1].get("entity_type",""),
"relation":rel.get("relation",""),
"evidence":str(rel.get("evidence","")or""),
"section":str(rel.get("section","")or""),
"pubmed_id":str(rel.get("pubmed_id","")or""),
"paper_id":rel.get("paper_id",""),
"confidence":rel.get("confidence"),
"claim_scope":str(rel.get("claim_scope","")or""),
"output_layer":str(rel.get("output_layer","")or"")})
paths.append({"path":es, "hops":len(es)})
return {"source_entity":req.source_entity, "target_entity":req.target_entity,
"max_hops":req.max_hops, "paths_found":len(paths), "paths":paths[:req.top_k_paths]}
@router.post("/chat")
async def chat(req: GQAReq):
driver = _get_driver()
candidates = _re.findall(r"[A-Z][a-zA-Z0-9\-_]+", req.question)
entities = [c for c in candidates if c not in SKIP_WORDS and len(c) >= 3]
fts = []
try:
with driver.session() as sf:
for rec in sf.run("CALL db.index.fulltext.queryNodes('entity_name_ftx',$q) YIELD node,score RETURN node.name AS n,node.entity_type AS t,score ORDER BY score DESC LIMIT 10", q=req.question[:100]):
fts.append({"name":rec["n"],"type":rec["t"],"score":rec["score"]})
except: pass
allowed = req.layers; lc = _layer_cond("r", allowed, req.include_candidate)
all_edges, seen = [], set()
with driver.session() as s:
terms = [e["name"] for e in fts[:5]] or entities[:5]
for term in terms:
q = f"MATCH (s:Entity)-[r:RELATION]->(t:Entity) WHERE (toLower(s.name) CONTAINS toLower($t) OR toLower(t.name) CONTAINS toLower($t)) AND {lc} RETURN s,r,t ORDER BY r.confidence DESC LIMIT $lim"
for rec in s.run(q, t=term, lim=req.top_k):
k = (rec["s"].get("name"),rec["r"].get("relation"),rec["t"].get("name"),rec["r"].get("paper_id",""))
if k not in seen: seen.add(k); all_edges.append(_record_to_dict(rec))
deduped, sd = [], set()
for e in all_edges:
ek = (e["source"],e["relation"],e["target"],e["paper_id"])
if ek not in sd: sd.add(ek); deduped.append(e)
deduped.sort(key=lambda x: LAYER_PRIORITY.get(x.get("output_layer",""),5))
answer, model_used = "", ""
if req.use_llm and deduped:
try:
from api.kb_qa_routes import _generate_llm_answer
rows = [{"source":e["source"],"source_type":e.get("source_type",""),"relation":e["relation"],"target":e["target"],"target_type":e.get("target_type",""),"evidence":e["evidence"],"section":e["section"],"pubmed_id":e["pubmed_id"],"paper_id":e["paper_id"],"output_layer":e.get("output_layer",""),"claim_scope":e.get("claim_scope",""),"chunk_id":""} for e in deduped[:25]]
prov = req.provider or os.environ.get("V15_KB_QA_LLM_PROVIDER")
mdl = req.model or os.environ.get("V15_KB_QA_LLM_MODEL")
answer = await _generate_llm_answer(req.question, "graph", rows, provider=prov, model=mdl)
model_used = f"{prov}:{mdl}"
except Exception as ex:
answer = f"[Graph QA] Evidence retrieved ({len(deduped)} edges) but LLM failed: {ex}"
return {"question":req.question, "entities_candidates":entities, "fts_entities":fts,
"graph_edges_count":len(deduped), "graph_edges":deduped[:30],
"answer":answer, "llm_model":model_used, "mode":"v15_graph_qa"}
@router.get("/entity/{name}")
async def entity_info(name: str):
driver = _get_driver()
with driver.session() as s:
rec = s.run("MATCH (e:Entity) WHERE toLower(e.name)=toLower($n) RETURN e", n=name).single()
if not rec:
rec = s.run("MATCH (e:Entity) WHERE toLower(e.name) CONTAINS toLower($n) RETURN e LIMIT 1", n=name).single()
if not rec:
raise HTTPException(404, "Entity not found")
e = rec["e"]
return {"name": e.get("name",""), "entity_type": e.get("entity_type","")}
print("[V15 Graph QA] Routes loaded")
+78
View File
@@ -0,0 +1,78 @@
import json
from typing import Any
from fastapi import WebSocket
class ConnectionManager:
def __init__(self):
self._connections: dict[str, list[WebSocket]] = {}
async def connect(self, websocket: WebSocket, session_id: str) -> None:
await websocket.accept()
if session_id not in self._connections:
self._connections[session_id] = []
self._connections[session_id].append(websocket)
def disconnect(self, websocket: WebSocket, session_id: str) -> None:
if session_id in self._connections:
self._connections[session_id].remove(websocket)
if not self._connections[session_id]:
del self._connections[session_id]
async def send_message(self, session_id: str, message: dict[str, Any]) -> None:
if session_id in self._connections:
for connection in self._connections[session_id]:
await connection.send_json(message)
async def broadcast(self, message: dict[str, Any]) -> None:
for connections in self._connections.values():
for connection in connections:
await connection.send_json(message)
manager = ConnectionManager()
async def send_progress(
session_id: str,
current_agent: str | None,
status: str,
progress: float,
message: str,
) -> None:
await manager.send_message(
session_id,
{
"type": "progress",
"session_id": session_id,
"data": {
"current_agent": current_agent,
"status": status,
"progress": progress,
"message": message,
},
},
)
async def send_result(session_id: str, result: dict[str, Any]) -> None:
await manager.send_message(
session_id,
{
"type": "result",
"session_id": session_id,
"data": result,
},
)
async def send_error(session_id: str, error: str) -> None:
await manager.send_message(
session_id,
{
"type": "error",
"session_id": session_id,
"data": {"error": error},
},
)