初始提交
This commit is contained in:
Executable
+30
@@ -0,0 +1,30 @@
|
||||
from api.main import app
|
||||
from api.routes import router
|
||||
from api.schemas import (
|
||||
CaseCreate,
|
||||
CaseResponse,
|
||||
FeedbackRequest,
|
||||
ProgressUpdate,
|
||||
QueryRequest,
|
||||
QueryResponse,
|
||||
SessionCreate,
|
||||
SessionResponse,
|
||||
WebSocketMessage,
|
||||
)
|
||||
from api.websocket import ConnectionManager, manager
|
||||
|
||||
__all__ = [
|
||||
"app",
|
||||
"router",
|
||||
"CaseCreate",
|
||||
"CaseResponse",
|
||||
"FeedbackRequest",
|
||||
"ProgressUpdate",
|
||||
"QueryRequest",
|
||||
"QueryResponse",
|
||||
"SessionCreate",
|
||||
"SessionResponse",
|
||||
"WebSocketMessage",
|
||||
"ConnectionManager",
|
||||
"manager",
|
||||
]
|
||||
Executable
+437
@@ -0,0 +1,437 @@
|
||||
"""
|
||||
高级RAG API路由
|
||||
集成查询扩展、实体链接、引用溯源、动态知识图谱、跨论文融合、Neo4j图算法
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from typing import List, Dict, Optional
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# ========== 查询扩展API ==========
|
||||
|
||||
class QueryExpansionRequest(BaseModel):
|
||||
query: str
|
||||
num_expansions: int = 5
|
||||
strategy: str = "expansion" # expansion, intent_focused, keyword_combination
|
||||
|
||||
|
||||
class QueryExpansionResponse(BaseModel):
|
||||
original: str
|
||||
rewritten: str
|
||||
alternatives: List[str]
|
||||
intent: str
|
||||
confidence: float
|
||||
keywords: List[str]
|
||||
synonyms: Dict[str, List[str]]
|
||||
|
||||
|
||||
@router.post("/advanced/query/expand")
|
||||
async def expand_query_endpoint(request: QueryExpansionRequest):
|
||||
"""查询扩展端点"""
|
||||
try:
|
||||
from src.rag.advanced import QueryExpander, QueryRewriter
|
||||
|
||||
expander = QueryExpander()
|
||||
expansion = expander.expand_query(request.query, request.num_expansions)
|
||||
|
||||
return QueryExpansionResponse(
|
||||
original=request.query,
|
||||
rewritten=expansion.expanded_queries[0] if expansion.expanded_queries else request.query,
|
||||
alternatives=expansion.expanded_queries[1:] if len(expansion.expanded_queries) > 1 else [],
|
||||
intent=expansion.intent.value,
|
||||
confidence=expansion.intent_confidence,
|
||||
keywords=expansion.keywords,
|
||||
synonyms=expansion.synonyms
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"查询扩展失败: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/advanced/query/rewrite")
|
||||
async def rewrite_query_endpoint(request: QueryExpansionRequest):
|
||||
"""查询重写端点"""
|
||||
try:
|
||||
from src.rag.advanced import QueryRewriter
|
||||
|
||||
rewriter = QueryRewriter()
|
||||
result = rewriter.rewrite(request.query, request.strategy)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"查询重写失败: {str(e)}")
|
||||
|
||||
|
||||
# ========== 实体链接API ==========
|
||||
|
||||
class EntityLinkingRequest(BaseModel):
|
||||
text: str
|
||||
|
||||
|
||||
class EntityLinkingResponse(BaseModel):
|
||||
entities: List[Dict]
|
||||
linked_count: int
|
||||
total_count: int
|
||||
|
||||
|
||||
@router.post("/advanced/entity/link")
|
||||
async def entity_linking_endpoint(request: EntityLinkingRequest):
|
||||
"""实体链接端点"""
|
||||
try:
|
||||
from src.rag.advanced import EntityLinker
|
||||
|
||||
linker = EntityLinker()
|
||||
linked_entities = linker.link_entities(request.text)
|
||||
|
||||
entities = []
|
||||
for le in linked_entities:
|
||||
entity_info = {
|
||||
"mention": le.mention.text,
|
||||
"start_pos": le.mention.start_pos,
|
||||
"end_pos": le.mention.end_pos,
|
||||
"context": le.mention.context[:100],
|
||||
"linked": le.kb_entity is not None,
|
||||
"link_confidence": le.link_confidence,
|
||||
"disambiguation_score": le.disambiguation_score,
|
||||
}
|
||||
|
||||
if le.kb_entity:
|
||||
entity_info.update({
|
||||
"kb_id": le.kb_entity.entity_id,
|
||||
"kb_name": le.kb_entity.name,
|
||||
"kb_type": le.kb_entity.entity_type.value,
|
||||
"kb_definition": le.kb_entity.definition,
|
||||
"kb_aliases": le.kb_entity.aliases[:5],
|
||||
})
|
||||
|
||||
entities.append(entity_info)
|
||||
|
||||
linked_count = sum(1 for e in entities if e["linked"])
|
||||
|
||||
return EntityLinkingResponse(
|
||||
entities=entities,
|
||||
linked_count=linked_count,
|
||||
total_count=len(entities)
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"实体链接失败: {str(e)}")
|
||||
|
||||
|
||||
# ========== 引用溯源API ==========
|
||||
|
||||
class CitationRequest(BaseModel):
|
||||
answer: str
|
||||
sources: List[Dict]
|
||||
min_citations: int = 3
|
||||
|
||||
|
||||
class CitationResponse(BaseModel):
|
||||
answer: str
|
||||
citations: List[Dict]
|
||||
confidence: float
|
||||
coverage_score: float
|
||||
total_citations: int
|
||||
|
||||
|
||||
@router.post("/advanced/citation/add")
|
||||
async def add_citations_endpoint(request: CitationRequest):
|
||||
"""添加引用溯源端点"""
|
||||
try:
|
||||
from src.rag.advanced import CitationIntegrator
|
||||
|
||||
integrator = CitationIntegrator()
|
||||
result = integrator.integrate_citations(
|
||||
request.answer,
|
||||
request.sources,
|
||||
request.min_citations
|
||||
)
|
||||
|
||||
return CitationResponse(
|
||||
answer=result["answer"],
|
||||
citations=result["citations"],
|
||||
confidence=result["confidence"],
|
||||
coverage_score=result["coverage_score"],
|
||||
total_citations=result["total_citations"]
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"添加引用失败: {str(e)}")
|
||||
|
||||
|
||||
# ========== 动态知识图谱API ==========
|
||||
|
||||
class DynamicKGRequest(BaseModel):
|
||||
sources: List[Dict]
|
||||
|
||||
|
||||
class DynamicKGResponse(BaseModel):
|
||||
entities: List[Dict]
|
||||
relations: List[Dict]
|
||||
total_entities: int
|
||||
total_relations: int
|
||||
conflicts: int
|
||||
resolved: int
|
||||
needs_review: int
|
||||
graph_data: Dict
|
||||
|
||||
|
||||
@router.post("/advanced/kg/build")
|
||||
async def build_dynamic_kg_endpoint(request: DynamicKGRequest):
|
||||
"""动态构建知识图谱端点"""
|
||||
try:
|
||||
from src.rag.advanced import DynamicKGBuilder
|
||||
|
||||
builder = DynamicKGBuilder()
|
||||
result = builder.build_from_search_results(request.sources)
|
||||
|
||||
# 获取图谱数据
|
||||
graph_data = builder.get_graph_data()
|
||||
|
||||
return DynamicKGResponse(
|
||||
entities=result["entities"],
|
||||
relations=result["relations"],
|
||||
total_entities=result["total_entities"],
|
||||
total_relations=result["total_relations"],
|
||||
conflicts=result["conflicts"],
|
||||
resolved=result["resolved"],
|
||||
needs_review=result["needs_review"],
|
||||
graph_data=graph_data
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"构建知识图谱失败: {str(e)}")
|
||||
|
||||
|
||||
# ========== 跨论文融合API ==========
|
||||
|
||||
class CrossPaperFusionRequest(BaseModel):
|
||||
papers: List[Dict]
|
||||
|
||||
|
||||
class CrossPaperFusionResponse(BaseModel):
|
||||
entities: List[Dict]
|
||||
relations: List[Dict]
|
||||
stats: Dict
|
||||
|
||||
|
||||
@router.post("/advanced/fusion/cross-paper")
|
||||
async def cross_paper_fusion_endpoint(request: CrossPaperFusionRequest):
|
||||
"""跨论文知识融合端点"""
|
||||
try:
|
||||
from src.rag.advanced import CrossPaperFusion
|
||||
|
||||
fusion = CrossPaperFusion()
|
||||
result = fusion.fuse_knowledge(request.papers)
|
||||
|
||||
return CrossPaperFusionResponse(
|
||||
entities=result["entities"],
|
||||
relations=result["relations"],
|
||||
stats=result["stats"]
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"跨论文融合失败: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/advanced/fusion/provenance/{entity_id}")
|
||||
async def get_provenance_endpoint(entity_id: str, request: CrossPaperFusionRequest):
|
||||
"""获取实体溯源信息端点"""
|
||||
try:
|
||||
from src.rag.advanced import CrossPaperFusion
|
||||
|
||||
fusion = CrossPaperFusion()
|
||||
fusion.fuse_knowledge(request.papers)
|
||||
provenance = fusion.get_entity_provenance(entity_id)
|
||||
|
||||
return provenance
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"获取溯源信息失败: {str(e)}")
|
||||
|
||||
|
||||
# ========== Neo4j图算法API ==========
|
||||
|
||||
class ShortestPathRequest(BaseModel):
|
||||
start_node: str
|
||||
end_node: str
|
||||
relationship_type: Optional[str] = None
|
||||
|
||||
|
||||
class ShortestPathResponse(BaseModel):
|
||||
status: str
|
||||
path: Optional[Dict]
|
||||
message: Optional[str]
|
||||
|
||||
|
||||
@router.post("/advanced/graph/shortest-path")
|
||||
async def shortest_path_endpoint(request: ShortestPathRequest):
|
||||
"""最短路径分析端点"""
|
||||
try:
|
||||
from src.rag.advanced import GraphAlgorithmAPI
|
||||
|
||||
api = GraphAlgorithmAPI()
|
||||
result = api.shortest_path_endpoint(
|
||||
request.start_node,
|
||||
request.end_node,
|
||||
request.relationship_type
|
||||
)
|
||||
|
||||
return ShortestPathResponse(**result)
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"最短路径分析失败: {str(e)}")
|
||||
|
||||
|
||||
@router.get("/advanced/graph/communities")
|
||||
async def community_detection_endpoint():
|
||||
"""社区发现端点"""
|
||||
try:
|
||||
from src.rag.advanced import GraphAlgorithmAPI
|
||||
|
||||
api = GraphAlgorithmAPI()
|
||||
result = api.community_detection_endpoint()
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"社区发现失败: {str(e)}")
|
||||
|
||||
|
||||
class CentralityRequest(BaseModel):
|
||||
algorithm: str = "degree" # degree, betweenness, pagerank
|
||||
top_k: int = 10
|
||||
|
||||
|
||||
@router.post("/advanced/graph/centrality")
|
||||
async def centrality_analysis_endpoint(request: CentralityRequest):
|
||||
"""中心性分析端点"""
|
||||
try:
|
||||
from src.rag.advanced import GraphAlgorithmAPI
|
||||
|
||||
api = GraphAlgorithmAPI()
|
||||
result = api.centrality_analysis_endpoint(request.algorithm, request.top_k)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"中心性分析失败: {str(e)}")
|
||||
|
||||
|
||||
@router.get("/advanced/graph/stats")
|
||||
async def graph_stats_endpoint():
|
||||
"""图统计端点"""
|
||||
try:
|
||||
from src.rag.advanced import GraphAlgorithmAPI
|
||||
|
||||
api = GraphAlgorithmAPI()
|
||||
result = api.graph_stats_endpoint()
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"获取图统计失败: {str(e)}")
|
||||
|
||||
|
||||
# ========== 综合RAG API ==========
|
||||
|
||||
class AdvancedRAGRequest(BaseModel):
|
||||
query: str
|
||||
enable_expansion: bool = True
|
||||
enable_entity_linking: bool = True
|
||||
enable_citation: bool = True
|
||||
enable_dynamic_kg: bool = True
|
||||
top_k: int = 5
|
||||
|
||||
|
||||
class AdvancedRAGResponse(BaseModel):
|
||||
query: str
|
||||
expanded_query: Optional[str]
|
||||
answer: str
|
||||
entities: List[Dict]
|
||||
citations: List[Dict]
|
||||
knowledge_graph: Optional[Dict]
|
||||
confidence: float
|
||||
|
||||
|
||||
@router.post("/advanced/rag")
|
||||
async def advanced_rag_endpoint(request: AdvancedRAGRequest):
|
||||
"""高级RAG综合端点"""
|
||||
try:
|
||||
from src.rag.advanced import (
|
||||
QueryExpander,
|
||||
EntityLinker,
|
||||
CitationIntegrator,
|
||||
DynamicKGBuilder
|
||||
)
|
||||
|
||||
# 1. 查询扩展
|
||||
expanded_query = request.query
|
||||
if request.enable_expansion:
|
||||
expander = QueryExpander()
|
||||
expansion = expander.expand_query(request.query, num_expansions=3)
|
||||
expanded_query = expansion.expanded_queries[0] if expansion.expanded_queries else request.query
|
||||
|
||||
# 2. 实体链接
|
||||
entities = []
|
||||
if request.enable_entity_linking:
|
||||
linker = EntityLinker()
|
||||
linked = linker.link_entities(request.query)
|
||||
entities = [
|
||||
{
|
||||
"mention": le.mention.text,
|
||||
"linked": le.kb_entity is not None,
|
||||
"kb_name": le.kb_entity.name if le.kb_entity else None,
|
||||
"kb_type": le.kb_entity.entity_type.value if le.kb_entity else None,
|
||||
"confidence": le.link_confidence
|
||||
}
|
||||
for le in linked
|
||||
]
|
||||
|
||||
# 3. 模拟检索结果(实际应该调用检索器)
|
||||
mock_sources = [
|
||||
{
|
||||
"doc_id": f"PMID{10000000 + i}",
|
||||
"content": f"Sample content for {request.query}",
|
||||
"score": 0.9 - i * 0.1,
|
||||
"chunk_id": f"chunk_{i}",
|
||||
"metadata": {"title": f"Paper {i}", "year": "2023"}
|
||||
}
|
||||
for i in range(request.top_k)
|
||||
]
|
||||
|
||||
# 4. 生成答案
|
||||
answer = f"基于检索到的{len(mock_sources)}条文献,关于'{request.query}'的分析如下..."
|
||||
|
||||
# 5. 添加引用
|
||||
citations = []
|
||||
if request.enable_citation:
|
||||
integrator = CitationIntegrator()
|
||||
citation_result = integrator.integrate_citations(answer, mock_sources, min_citations=3)
|
||||
answer = citation_result["answer"]
|
||||
citations = citation_result["citations"]
|
||||
|
||||
# 6. 构建动态知识图谱
|
||||
kg_data = None
|
||||
if request.enable_dynamic_kg:
|
||||
builder = DynamicKGBuilder()
|
||||
kg_result = builder.build_from_search_results(mock_sources)
|
||||
kg_data = builder.get_graph_data()
|
||||
|
||||
return AdvancedRAGResponse(
|
||||
query=request.query,
|
||||
expanded_query=expanded_query if request.enable_expansion else None,
|
||||
answer=answer,
|
||||
entities=entities,
|
||||
citations=citations,
|
||||
knowledge_graph=kg_data,
|
||||
confidence=0.85
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
raise HTTPException(status_code=500, detail=f"高级RAG处理失败: {str(e)}")
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
"""Maintained FastAPI entrypoint for the PlantAI-rag cleanup baseline."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
from api.kg_routes import router as kg_router
|
||||
from api.methodology_routes import router as methodology_router
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
app = FastAPI(
|
||||
title="PlantAI-rag API",
|
||||
description="Maintained API surface for the PlantAI-rag open-source cleanup baseline.",
|
||||
version="0.1.0",
|
||||
)
|
||||
app.include_router(kg_router, prefix="/api/v1")
|
||||
app.include_router(methodology_router, prefix="/api/v1")
|
||||
|
||||
@app.get("/")
|
||||
async def root() -> dict[str, str]:
|
||||
return {
|
||||
"name": "PlantAI-rag",
|
||||
"status": "ready",
|
||||
"runtime_surface": "maintained_baseline",
|
||||
}
|
||||
|
||||
@app.get("/health")
|
||||
async def health() -> dict[str, str]:
|
||||
return {"status": "healthy"}
|
||||
|
||||
return app
|
||||
|
||||
|
||||
app = create_app()
|
||||
Executable
+71
@@ -0,0 +1,71 @@
|
||||
"""Case-specific deterministic QA routes.
|
||||
|
||||
These endpoints expose validated reproduction-case packages before they are
|
||||
merged into the general RAG chat route. They do not call an LLM.
|
||||
"""
|
||||
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from src.rag.unified.case_qa import CasePackageQAEngine
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
FT_SPL_CASE_PACKAGE = PROJECT_ROOT / "data" / "ft_spl_reproduction_case_package.json"
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class CaseQARequest(BaseModel):
|
||||
query: str = Field(..., min_length=1)
|
||||
|
||||
|
||||
class CaseQAResponse(BaseModel):
|
||||
query: str
|
||||
intent: str
|
||||
answer: str
|
||||
citations: list[str]
|
||||
facts: dict[str, Any]
|
||||
|
||||
|
||||
class CaseStatusResponse(BaseModel):
|
||||
case_id: str
|
||||
package_path: str
|
||||
package_exists: bool
|
||||
supported: bool
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_ft_spl_engine() -> CasePackageQAEngine:
|
||||
if not FT_SPL_CASE_PACKAGE.exists():
|
||||
raise FileNotFoundError(str(FT_SPL_CASE_PACKAGE))
|
||||
return CasePackageQAEngine(FT_SPL_CASE_PACKAGE)
|
||||
|
||||
|
||||
@router.get("/ft-spl/status", response_model=CaseStatusResponse)
|
||||
async def ft_spl_case_status():
|
||||
return CaseStatusResponse(
|
||||
case_id="ft_spl_reproduction_job_1777615271527",
|
||||
package_path=str(FT_SPL_CASE_PACKAGE),
|
||||
package_exists=FT_SPL_CASE_PACKAGE.exists(),
|
||||
supported=FT_SPL_CASE_PACKAGE.exists(),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/ft-spl/qa", response_model=CaseQAResponse)
|
||||
async def ft_spl_case_qa(request: CaseQARequest):
|
||||
try:
|
||||
response = get_ft_spl_engine().answer(request.query)
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(status_code=503, detail=f"Case package missing: {exc}") from exc
|
||||
return CaseQAResponse(
|
||||
query=response.query,
|
||||
intent=response.intent,
|
||||
answer=response.answer,
|
||||
citations=response.citations,
|
||||
facts=response.facts,
|
||||
)
|
||||
Executable
+935
@@ -0,0 +1,935 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Hybrid (Qdrant + Neo4j) QA routes for the V15 SRT knowledge base.
|
||||
|
||||
This route uses V15HybridRetriever for vector (Qdrant) and graph (Neo4j)
|
||||
retrieval. The old SQLite-only path is preserved in the .bak file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import asyncio
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
from pydantic import BaseModel, Field
|
||||
from src.rag.v15_hybrid_retriever import V15HybridRetriever
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
DEFAULT_KB_DIR = PROJECT_ROOT / "data" / "knowledge_base" / "v15_srt_kb_20260525"
|
||||
|
||||
router = APIRouter(prefix="/kb-qa", tags=["V15 SRT KB QA"])
|
||||
_llm_manager = None
|
||||
_result_chunk_cache: Dict[str, List[Dict[str, str]]] = {}
|
||||
|
||||
|
||||
STOPWORDS = {
|
||||
"what",
|
||||
"which",
|
||||
"where",
|
||||
"when",
|
||||
"who",
|
||||
"how",
|
||||
"does",
|
||||
"did",
|
||||
"are",
|
||||
"was",
|
||||
"were",
|
||||
"the",
|
||||
"and",
|
||||
"for",
|
||||
"with",
|
||||
"from",
|
||||
"into",
|
||||
"about",
|
||||
"that",
|
||||
"this",
|
||||
"these",
|
||||
"those",
|
||||
"gene",
|
||||
"genes",
|
||||
"protein",
|
||||
"proteins",
|
||||
"family",
|
||||
"tell",
|
||||
"me",
|
||||
"is",
|
||||
"of",
|
||||
"in",
|
||||
"to",
|
||||
"a",
|
||||
"an",
|
||||
}
|
||||
|
||||
LAYER_FILTERS = {
|
||||
"current_findings": (
|
||||
"r.output_layer = 'main_kg' "
|
||||
"AND COALESCE(r.claim_scope, '') NOT IN ('background_context', 'cited_prior_work') "
|
||||
"AND COALESCE(r.candidate_status, '') != 'candidate'"
|
||||
),
|
||||
"main_kg": (
|
||||
"r.output_layer = 'main_kg' "
|
||||
"AND COALESCE(r.claim_scope, '') NOT IN ('background_context', 'cited_prior_work') "
|
||||
"AND COALESCE(r.candidate_status, '') != 'candidate'"
|
||||
),
|
||||
"methods": "r.output_layer = 'method_relations' AND COALESCE(r.candidate_status, '') != 'candidate'",
|
||||
"method_relations": "r.output_layer = 'method_relations' AND COALESCE(r.candidate_status, '') != 'candidate'",
|
||||
"datasets": "r.output_layer = 'dataset_metadata' AND COALESCE(r.candidate_status, '') != 'candidate'",
|
||||
"dataset_metadata": "r.output_layer = 'dataset_metadata' AND COALESCE(r.candidate_status, '') != 'candidate'",
|
||||
"prior_work": "r.output_layer = 'background_relations'",
|
||||
"background_relations": "r.output_layer = 'background_relations'",
|
||||
"candidate": "(r.output_layer = 'candidate_review' OR r.candidate_status = 'candidate')",
|
||||
"candidate_review": "(r.output_layer = 'candidate_review' OR r.candidate_status = 'candidate')",
|
||||
"all": "1 = 1",
|
||||
}
|
||||
|
||||
RELATION_FAMILY_ALIASES = {
|
||||
"baseline_expression": [
|
||||
"expressed_in",
|
||||
"highly_expressed_during",
|
||||
"only_expressed_during",
|
||||
"differentially_expressed_under",
|
||||
"upregulated_in",
|
||||
"downregulated_in",
|
||||
"downregulated_during",
|
||||
"peak_at",
|
||||
"not_induced_under",
|
||||
],
|
||||
"up_expression": ["upregulated_in", "induced_in", "increased_in"],
|
||||
"down_expression": ["downregulated_in", "downregulated_during", "repressed_in", "decreased_in"],
|
||||
"differential_expression": ["differentially_expressed_under"],
|
||||
"expression_peak": ["peak_at", "reached_maximum_at"],
|
||||
"not_induced": ["not_induced_under"],
|
||||
"identification": ["identified_in", "identified_from"],
|
||||
"member_count": ["has_member_count", "contains_gene_family_members"],
|
||||
"classification": ["member_of", "classified_into", "divided_into", "separated_into", "categorized_into"],
|
||||
"domain": ["contains_domain", "has_domain"],
|
||||
"cis_element": ["contains_cis_element", "has_cis_element", "has_cis_elements"],
|
||||
"motif": ["contains_motif"],
|
||||
"localization": ["localizes_to"],
|
||||
"interaction": ["interacts_with"],
|
||||
"binding": ["binds_to"],
|
||||
"targeting": ["targets"],
|
||||
"homology": ["clustered_with", "homologous_to"],
|
||||
"synteny": ["syntenic_with", "has_collinear_pairs_with", "collinear_with"],
|
||||
"regulation": ["regulates", "positively_regulates", "negatively_regulates"],
|
||||
"effect": ["promotes", "enhances", "overexpression_enhances", "overexpression_increases", "overexpression_improves"],
|
||||
"function": ["plays_role_in", "play_roles_in", "involved_in", "participates_in"],
|
||||
"identity": ["is_a"],
|
||||
}
|
||||
|
||||
|
||||
class KBQARequest(BaseModel):
|
||||
question: str = Field(..., description="Question to answer from the V15 SRT KB")
|
||||
top_k: int = Field(8, ge=1, le=50)
|
||||
layer: str = Field(
|
||||
"auto",
|
||||
description=(
|
||||
"auto/current_findings/methods/datasets/prior_work/"
|
||||
"candidate/all"
|
||||
),
|
||||
)
|
||||
include_evidence: bool = True
|
||||
include_candidate_fallback: bool = False
|
||||
use_llm: bool = True
|
||||
llm_provider: Optional[str] = None
|
||||
llm_model: Optional[str] = None
|
||||
anchor_entity: Optional[str] = Field(
|
||||
None,
|
||||
description="Optional structured retrieval hint, e.g. GmP450 genes",
|
||||
)
|
||||
relation_family: Optional[str] = Field(
|
||||
None,
|
||||
description="Optional question/relation family hint, e.g. baseline_expression",
|
||||
)
|
||||
preferred_relations: List[str] = Field(
|
||||
default_factory=list,
|
||||
description="Optional exact relation hints for structured retrieval",
|
||||
)
|
||||
preferred_papers: List[str] = Field(
|
||||
default_factory=list,
|
||||
description="Optional paper_id hints for benchmark/debug QA; used as boost, not as a hard filter",
|
||||
)
|
||||
|
||||
|
||||
class KBQASource(BaseModel):
|
||||
relation_id: str
|
||||
paper_id: str = ""
|
||||
pubmed_id: str = ""
|
||||
p_source: str = ""
|
||||
source_result_path: str = ""
|
||||
chunk_id: str = ""
|
||||
chunk_hint: str = ""
|
||||
evidence_key: str = ""
|
||||
source: str
|
||||
source_type: str = ""
|
||||
relation: str
|
||||
target: str
|
||||
target_type: str = ""
|
||||
evidence: str = ""
|
||||
section: str = ""
|
||||
source_label: str = ""
|
||||
claim_scope: str = ""
|
||||
output_layer: str = ""
|
||||
candidate_status: str = ""
|
||||
confidence: str = ""
|
||||
score: float = 0.0
|
||||
|
||||
|
||||
class KBQAResponse(BaseModel):
|
||||
question: str
|
||||
answer: str
|
||||
selected_layer: str
|
||||
layer_reason: str
|
||||
source_count: int
|
||||
paper_source_count: int = 0
|
||||
sources: List[KBQASource]
|
||||
kb_path: str
|
||||
answer_mode: str = "extractive"
|
||||
llm_used: bool = False
|
||||
notes: List[str] = []
|
||||
|
||||
|
||||
def _kb_dir() -> Path:
|
||||
configured = os.environ.get("V15_SRT_KB_DIR")
|
||||
return Path(configured).expanduser() if configured else DEFAULT_KB_DIR
|
||||
|
||||
|
||||
def get_llm_manager():
|
||||
"""Lazy-load the existing project LLM manager for answer synthesis."""
|
||||
global _llm_manager
|
||||
if _llm_manager is not None:
|
||||
return _llm_manager
|
||||
try:
|
||||
import yaml
|
||||
from src.llm.manager import LLMManager
|
||||
|
||||
config_path = PROJECT_ROOT / "config" / "llm.yaml"
|
||||
if config_path.exists():
|
||||
config = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
|
||||
else:
|
||||
config = {"providers": {}, "default": {}}
|
||||
|
||||
# KB-QA must be able to use DeepSeek even when the shared llm.yaml was
|
||||
# created before the DeepSeek provider was added to the project.
|
||||
config.setdefault("providers", {}).setdefault(
|
||||
"deepseek",
|
||||
{
|
||||
"name": "DeepSeek",
|
||||
"api_base": os.environ.get("DEEPSEEK_BASE_URL", "https://api.deepseek.com"),
|
||||
"models": [
|
||||
{"name": "deepseek-v4-flash", "context_length": 128000},
|
||||
{"name": "deepseek-v4-pro", "context_length": 128000},
|
||||
{"name": "deepseek-chat", "context_length": 64000},
|
||||
],
|
||||
"env_key": "DEEPSEEK_API_KEY",
|
||||
},
|
||||
)
|
||||
|
||||
provider = os.environ.get("V15_KB_QA_LLM_PROVIDER")
|
||||
model = os.environ.get("V15_KB_QA_LLM_MODEL")
|
||||
if provider:
|
||||
config.setdefault("default", {})["provider"] = provider
|
||||
if model:
|
||||
config.setdefault("default", {})["model"] = model
|
||||
if os.environ.get("V15_KB_QA_LLM_MAX_TOKENS"):
|
||||
config.setdefault("default", {})["max_tokens"] = int(os.environ["V15_KB_QA_LLM_MAX_TOKENS"])
|
||||
if os.environ.get("V15_KB_QA_LLM_TEMPERATURE"):
|
||||
config.setdefault("default", {})["temperature"] = float(os.environ["V15_KB_QA_LLM_TEMPERATURE"])
|
||||
|
||||
_llm_manager = LLMManager(config)
|
||||
return _llm_manager
|
||||
except Exception as exc:
|
||||
print(f"[KB-QA] LLM manager init failed: {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def _classify_layer(question: str, requested_layer: str) -> Tuple[str, str]:
|
||||
requested = (requested_layer or "auto").strip().lower()
|
||||
if requested != "auto":
|
||||
if requested not in LAYER_FILTERS:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Unsupported layer '{requested_layer}'. "
|
||||
f"Use one of: auto, {', '.join(sorted(LAYER_FILTERS))}",
|
||||
)
|
||||
return requested, "user_selected"
|
||||
|
||||
q = question.lower()
|
||||
if re.search(r"\b(tool|software|method|pipeline|database|using|used|analy[sz]ed by)\b", q) or any(
|
||||
word in question for word in ["工具", "软件", "方法", "流程", "用了什么", "使用了什么"]
|
||||
):
|
||||
return "methods", "auto_detected_method_question"
|
||||
if re.search(r"\b(accession|geo|sra|prjna|srp|gse|rna-seq|rnaseq|doi)\b", q) or any(
|
||||
word in question for word in ["编号", "登录号", "测序数据", "转录组数据"]
|
||||
):
|
||||
return "datasets", "auto_detected_dataset_question"
|
||||
if re.search(r"\b(prior|previous|background|reported|known|literature|cited)\b", q) or any(
|
||||
word in question for word in ["前人", "背景", "已有研究", "文献提到", "报道过"]
|
||||
):
|
||||
return "prior_work", "auto_detected_prior_work_question"
|
||||
if any(word in question for word in ["候选", "不确定", "可能相关"]):
|
||||
return "candidate", "auto_detected_candidate_question"
|
||||
return "current_findings", "default_current_findings"
|
||||
|
||||
|
||||
def _query_tokens(question: str) -> List[str]:
|
||||
raw_tokens = re.findall(r"[A-Za-z][A-Za-z0-9_.:/+-]*|\d+[A-Za-z0-9_.:/+-]*", question)
|
||||
tokens: List[str] = []
|
||||
for token in raw_tokens:
|
||||
cleaned = re.sub(r"[^A-Za-z0-9_]+", " ", token).strip()
|
||||
for part in cleaned.split():
|
||||
lower = part.lower()
|
||||
if len(part) < 2 or lower in STOPWORDS:
|
||||
continue
|
||||
tokens.append(part)
|
||||
# Preserve order while removing duplicates.
|
||||
seen = set()
|
||||
unique = []
|
||||
for token in tokens:
|
||||
key = token.lower()
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
unique.append(token)
|
||||
return unique[:12]
|
||||
|
||||
|
||||
def _focus_entity(question: str) -> str:
|
||||
"""Extract a focus entity from short definition-style questions."""
|
||||
text = question.strip()
|
||||
patterns = [
|
||||
r"^\s*what\s+(?:is|are)\s+(.+?)[??。.!]*\s*$",
|
||||
r"^\s*define\s+(.+?)[??。.!]*\s*$",
|
||||
r"^\s*(.+?)\s*(?:是什么|是啥|指什么)[??。.!]*\s*$",
|
||||
]
|
||||
for pattern in patterns:
|
||||
match = re.search(pattern, text, flags=re.IGNORECASE)
|
||||
if not match:
|
||||
continue
|
||||
focus = match.group(1).strip()
|
||||
focus = re.sub(r"^(the|a|an)\s+", "", focus, flags=re.IGNORECASE)
|
||||
focus = focus.strip(" \t\r\n\"'`.,;:()[]{}")
|
||||
if 2 <= len(focus) <= 80:
|
||||
return focus
|
||||
return ""
|
||||
|
||||
|
||||
def _clean_hint_text(value: str) -> str:
|
||||
value = re.sub(r"\s+", " ", value or "").strip()
|
||||
value = value.strip(" \t\r\n\"'`.,;:()[]{}??。")
|
||||
return value[:120]
|
||||
|
||||
|
||||
def _infer_anchor_entity(question: str) -> str:
|
||||
"""Infer a likely SRT source entity from a natural-language question."""
|
||||
patterns = [
|
||||
r"问题[::]\s*(.+?)\s*(?:的|之)\s*表达",
|
||||
r"问题[::]\s*(.+?)\s*(?:的|之)\s*鉴定",
|
||||
r"问题[::]\s*(.+?)\s*(?:报道了多少|有多少)",
|
||||
r"expression of\s+(.+?)\s+(?:was|were|reported|under|in|\\?)",
|
||||
r"identifying\s+(.+?)(?:\\?|$)",
|
||||
r"identification of\s+(.+?)(?:\\?|$)",
|
||||
r"reported for\s+(.+?)(?:\\?|$)",
|
||||
r"reported in\s+(.+?)(?:\\?|$)",
|
||||
r"reported to interact with\s+(.+?)(?:\\?|$)",
|
||||
r"was\s+(.+?)\s+upregulated",
|
||||
r"was\s+(.+?)\s+downregulated",
|
||||
]
|
||||
for pattern in patterns:
|
||||
match = re.search(pattern, question, flags=re.IGNORECASE)
|
||||
if not match:
|
||||
continue
|
||||
candidate = _clean_hint_text(match.group(1))
|
||||
candidate = re.sub(r"^(the|a|an)\s+", "", candidate, flags=re.IGNORECASE)
|
||||
candidate = re.split(r"\n|English question:|回答要求[::]", candidate)[0].strip()
|
||||
if 2 <= len(candidate) <= 100 and not re.search(r"\b(where|which|what|how|under)\b", candidate, re.I):
|
||||
return candidate
|
||||
focus = _focus_entity(question)
|
||||
return focus
|
||||
|
||||
|
||||
def _entity_variants(entity: str) -> List[str]:
|
||||
entity = _clean_hint_text(entity)
|
||||
if not entity:
|
||||
return []
|
||||
variants = {entity}
|
||||
base = re.sub(
|
||||
r"\s+(gene|genes|protein|proteins|transcripts?|family|gene family|protein family)$",
|
||||
"",
|
||||
entity,
|
||||
flags=re.IGNORECASE,
|
||||
).strip()
|
||||
if base and len(base) >= 2:
|
||||
variants.add(base)
|
||||
if re.search(r"[A-Za-z0-9]$", base):
|
||||
variants.add(base + "s")
|
||||
variants.add(base + " genes")
|
||||
variants.add(base + " proteins")
|
||||
if entity.endswith("s") and len(entity) > 3:
|
||||
variants.add(entity[:-1])
|
||||
return [v for v in variants if len(v) >= 2][:8]
|
||||
|
||||
|
||||
def _infer_relation_family(question: str) -> str:
|
||||
q = question.lower()
|
||||
checks = [
|
||||
("baseline_expression", [r"表达", r"expression", r"expressed", r"tissue", r"condition"]),
|
||||
("up_expression", [r"上调", r"upregulated", r"induced"]),
|
||||
("down_expression", [r"下调", r"downregulated", r"repressed"]),
|
||||
("not_induced", [r"未被诱导", r"not induced"]),
|
||||
("identification", [r"鉴定", r"identification", r"identifying", r"identified"]),
|
||||
("member_count", [r"多少个成员", r"how many members", r"member count"]),
|
||||
("cis_element", [r"顺式", r"cis[- ]?acting", r"promoter", r"启动子"]),
|
||||
("domain", [r"结构域", r"domain"]),
|
||||
("motif", [r"motif"]),
|
||||
("localization", [r"定位", r"localiz"]),
|
||||
("interaction", [r"互作", r"interact"]),
|
||||
("binding", [r"结合", r"bind"]),
|
||||
("synteny", [r"共线", r"synten", r"collinear"]),
|
||||
("regulation", [r"调控", r"regulat"]),
|
||||
("function", [r"功能", r"function", r"process", r"involved", r"role"]),
|
||||
("classification", [r"亚家族", r"类别", r"class", r"group", r"subfamily", r"belong"]),
|
||||
]
|
||||
for family, patterns in checks:
|
||||
if any(re.search(pattern, q) for pattern in patterns):
|
||||
return family
|
||||
return ""
|
||||
|
||||
|
||||
def _resolve_preferred_relations(relation_family: str, preferred_relations: List[str]) -> List[str]:
|
||||
resolved: List[str] = []
|
||||
for rel in preferred_relations or []:
|
||||
rel = _clean_hint_text(rel)
|
||||
if rel and re.fullmatch(r"[A-Za-z0-9_]+", rel):
|
||||
resolved.append(rel)
|
||||
family_key = (relation_family or "").strip()
|
||||
resolved.extend(RELATION_FAMILY_ALIASES.get(family_key, []))
|
||||
seen = set()
|
||||
unique = []
|
||||
for rel in resolved:
|
||||
key = rel.lower()
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
unique.append(rel)
|
||||
return unique[:20]
|
||||
|
||||
|
||||
def _build_query_plan(
|
||||
question: str,
|
||||
anchor_entity: Optional[str] = None,
|
||||
relation_family: Optional[str] = None,
|
||||
preferred_relations: Optional[List[str]] = None,
|
||||
preferred_papers: Optional[List[str]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
anchor = _clean_hint_text(anchor_entity or "") or _infer_anchor_entity(question)
|
||||
family = _clean_hint_text(relation_family or "") or _infer_relation_family(question)
|
||||
relations = _resolve_preferred_relations(family, preferred_relations or [])
|
||||
papers = [_clean_hint_text(paper) for paper in (preferred_papers or []) if _clean_hint_text(paper)]
|
||||
return {
|
||||
"anchor_entity": anchor,
|
||||
"entity_variants": _entity_variants(anchor),
|
||||
"relation_family": family,
|
||||
"preferred_relations": relations,
|
||||
"preferred_papers": papers[:20],
|
||||
}
|
||||
|
||||
|
||||
def _norm_space(text: str) -> str:
|
||||
return re.sub(r"\s+", " ", text or "").strip()
|
||||
|
||||
|
||||
def _evidence_key(text: str) -> str:
|
||||
text = _norm_space(text).lower()
|
||||
text = re.sub(r"[^a-z0-9\u4e00-\u9fff]+", " ", text)
|
||||
return _norm_space(text)
|
||||
|
||||
|
||||
def _paper_group_key(row: Dict[str, Any]) -> str:
|
||||
return (
|
||||
row.get("pubmed_id")
|
||||
or row.get("paper_id")
|
||||
or row.get("source_result_path")
|
||||
or row.get("relation_id")
|
||||
or ""
|
||||
)
|
||||
|
||||
|
||||
def _resolve_result_path(source_result_path: str) -> Optional[Path]:
|
||||
if not source_result_path:
|
||||
return None
|
||||
path = Path(source_result_path)
|
||||
if not path.is_absolute():
|
||||
path = PROJECT_ROOT / path
|
||||
if path.is_dir():
|
||||
path = path / "result.json"
|
||||
if path.name != "result.json" and path.suffix != ".json":
|
||||
candidate = path / "result.json"
|
||||
if candidate.exists():
|
||||
path = candidate
|
||||
return path if path.exists() else None
|
||||
|
||||
|
||||
def _load_result_chunks(source_result_path: str) -> List[Dict[str, str]]:
|
||||
path = _resolve_result_path(source_result_path)
|
||||
if path is None:
|
||||
return []
|
||||
key = str(path)
|
||||
if key in _result_chunk_cache:
|
||||
return _result_chunk_cache[key]
|
||||
chunks: List[Dict[str, str]] = []
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
for idx, chunk in enumerate(data.get("chunks") or []):
|
||||
text = _norm_space(str(chunk.get("text") or chunk.get("content") or ""))
|
||||
if not text:
|
||||
continue
|
||||
chunk_id = str(chunk.get("chunk_id") or chunk.get("id") or f"chunk_{idx:04d}")
|
||||
chunks.append(
|
||||
{
|
||||
"chunk_id": chunk_id,
|
||||
"section": str(chunk.get("section") or chunk.get("section_title") or ""),
|
||||
"text": text,
|
||||
"norm_text": _evidence_key(text),
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
chunks = []
|
||||
_result_chunk_cache[key] = chunks
|
||||
return chunks
|
||||
|
||||
|
||||
def _infer_chunk_id(row: Dict[str, Any]) -> str:
|
||||
evidence_norm = _evidence_key(row.get("evidence", ""))
|
||||
if not evidence_norm:
|
||||
return ""
|
||||
chunks = _load_result_chunks(row.get("source_result_path", ""))
|
||||
if not chunks:
|
||||
return ""
|
||||
|
||||
anchors = [evidence_norm]
|
||||
if len(evidence_norm) > 260:
|
||||
anchors.extend([evidence_norm[:260], evidence_norm[:160], evidence_norm[-200:]])
|
||||
for anchor in anchors:
|
||||
if len(anchor) < 40:
|
||||
continue
|
||||
for chunk in chunks:
|
||||
if anchor in chunk.get("norm_text", ""):
|
||||
return chunk.get("chunk_id", "")
|
||||
return ""
|
||||
|
||||
|
||||
def _fallback_chunk_hint(row: Dict[str, Any]) -> str:
|
||||
parts = []
|
||||
if row.get("section"):
|
||||
parts.append(f"section={row.get('section')}")
|
||||
if row.get("p_source"):
|
||||
parts.append(f"p_source={row.get('p_source')}")
|
||||
if row.get("source_label"):
|
||||
parts.append(f"source_label={row.get('source_label')}")
|
||||
if not parts and row.get("relation_id"):
|
||||
parts.append(f"relation={row.get('relation_id')}")
|
||||
return " | ".join(parts)
|
||||
|
||||
|
||||
def _decorate_source_rows(rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
for row in rows:
|
||||
row["evidence_key"] = _evidence_key(row.get("evidence", ""))
|
||||
chunk_id = _infer_chunk_id(row)
|
||||
row["chunk_id"] = chunk_id
|
||||
row["chunk_hint"] = chunk_id or _fallback_chunk_hint(row)
|
||||
return rows
|
||||
|
||||
|
||||
def _shorten(text: str, limit: int = 420) -> str:
|
||||
text = re.sub(r"\s+", " ", text or "").strip()
|
||||
if len(text) <= limit:
|
||||
return text
|
||||
return text[: limit - 3].rstrip() + "..."
|
||||
|
||||
|
||||
def _format_answer(question: str, layer: str, rows: List[Dict[str, Any]], include_evidence: bool) -> str:
|
||||
if not rows:
|
||||
return (
|
||||
"未在当前知识库层中检索到可用证据。可以尝试切换 layer=all、"
|
||||
"layer=prior_work、layer=methods 或 layer=datasets。"
|
||||
)
|
||||
|
||||
layer_label = {
|
||||
"current_findings": "本文主要发现(main_kg)",
|
||||
"main_kg": "本文主要发现(main_kg)",
|
||||
"methods": "方法/工具(method_relations)",
|
||||
"method_relations": "方法/工具(method_relations)",
|
||||
"datasets": "数据集(dataset_metadata)",
|
||||
"dataset_metadata": "数据集(dataset_metadata)",
|
||||
"prior_work": "前人研究/背景(background_relations)",
|
||||
"background_relations": "前人研究/背景(background_relations)",
|
||||
"candidate": "候选关系(candidate_review)",
|
||||
"candidate_review": "候选关系(candidate_review)",
|
||||
"all": "全部非拒绝关系",
|
||||
}.get(layer, layer)
|
||||
|
||||
paper_count = len({_paper_group_key(row) for row in rows})
|
||||
lines = [
|
||||
f"基于 {layer_label} 检索到 {paper_count} 篇文献中的 {len(rows)} 条去重证据:"
|
||||
]
|
||||
for idx, row in enumerate(rows[:8], start=1):
|
||||
pmid = row.get("pubmed_id") or "NA"
|
||||
chunk = row.get("chunk_id") or row.get("chunk_hint") or row.get("section") or "NA"
|
||||
lines.append(
|
||||
f"{idx}. PMID: {pmid} | paper: {row.get('paper_id', '') or 'NA'} | chunk: {chunk}"
|
||||
)
|
||||
if include_evidence and row.get("evidence"):
|
||||
lines.append(f" 证据:{_shorten(row['evidence'])}")
|
||||
lines.append("回答只基于当前 V15 SRT 知识库;需要原文级核查时请查看返回的 sources。")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _build_llm_context(rows: List[Dict[str, Any]], max_rows: Optional[int] = None) -> str:
|
||||
if max_rows is None:
|
||||
max_rows = max(int(os.environ.get("V15_KB_QA_LLM_CONTEXT_EVIDENCE_ROWS", "18")), 1)
|
||||
parts = []
|
||||
for idx, row in enumerate(rows[:max_rows], start=1):
|
||||
evidence = re.sub(r"\s+", " ", row.get("evidence", "") or "").strip()
|
||||
if len(evidence) > 900:
|
||||
evidence = evidence[:897].rstrip() + "..."
|
||||
parts.append(
|
||||
"\n".join(
|
||||
[
|
||||
f"[S{idx}]",
|
||||
f"PMID: {row.get('pubmed_id', '')}",
|
||||
f"paper_id: {row.get('paper_id', '')}",
|
||||
f"chunk: {row.get('chunk_id') or row.get('chunk_hint') or row.get('section', '')}",
|
||||
f"layer: {row.get('output_layer', '')}",
|
||||
f"claim_scope: {row.get('claim_scope', '')}",
|
||||
f"SRT: {row.get('source', '')} --{row.get('relation', '')}--> {row.get('target', '')}",
|
||||
f"source_type: {row.get('source_type', '')}; target_type: {row.get('target_type', '')}",
|
||||
f"evidence: {evidence}",
|
||||
]
|
||||
)
|
||||
)
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
def _build_answer_prompt(question: str, layer: str, rows: List[Dict[str, Any]]) -> str:
|
||||
layer_instruction = {
|
||||
"current_findings": "These are current-paper biological findings.",
|
||||
"main_kg": "These are current-paper biological findings.",
|
||||
"methods": "These are method/tool/software relations. Answer method questions only.",
|
||||
"method_relations": "These are method/tool/software relations. Answer method questions only.",
|
||||
"datasets": "These are dataset/accession/provenance relations.",
|
||||
"dataset_metadata": "These are dataset/accession/provenance relations.",
|
||||
"prior_work": "These are prior-work or background relations; explicitly say they are prior/background if used.",
|
||||
"background_relations": "These are prior-work or background relations; explicitly say they are prior/background if used.",
|
||||
"candidate": "These are candidate/uncertain relations; explicitly warn that they are candidates.",
|
||||
"candidate_review": "These are candidate/uncertain relations; explicitly warn that they are candidates.",
|
||||
"all": "These relations may come from different layers. Do not mix layer meanings.",
|
||||
}.get(layer, "Use the layer metadata to qualify the answer.")
|
||||
|
||||
return f"""You are answering from an audited plant gene-family SRT knowledge base.
|
||||
|
||||
Question:
|
||||
{question}
|
||||
|
||||
Selected layer:
|
||||
{layer}
|
||||
|
||||
Layer instruction:
|
||||
{layer_instruction}
|
||||
|
||||
Retrieved evidence records:
|
||||
{_build_llm_context(rows)}
|
||||
|
||||
Rules:
|
||||
1. Answer only from the retrieved SRT evidence above. Do not add outside knowledge.
|
||||
2. If the retrieved SRTs do not answer the question, say so clearly.
|
||||
3. Cite supporting rows with [S1], [S2], etc.
|
||||
4. Keep current findings, prior/background work, method/tool facts, datasets, and candidate relations conceptually separate.
|
||||
5. If evidence is limited or conflicting, say "the KB evidence is limited" and explain the limitation.
|
||||
6. Answer in the same language as the user.s question. If the question contains non-English characters (e.g. Chinese), respond in that language primarily. If the question is purely English, respond in English.
|
||||
7. Be concise but conversational; do not just repeat a raw SRT table.
|
||||
|
||||
Answer:
|
||||
"""
|
||||
|
||||
|
||||
async def _generate_llm_answer(
|
||||
question: str,
|
||||
layer: str,
|
||||
rows: List[Dict[str, Any]],
|
||||
provider: Optional[str] = None,
|
||||
model: Optional[str] = None,
|
||||
) -> str:
|
||||
if not rows:
|
||||
return ""
|
||||
|
||||
llm_mgr = get_llm_manager()
|
||||
if llm_mgr is None:
|
||||
raise RuntimeError("LLM manager is not available")
|
||||
|
||||
prompt = _build_answer_prompt(question, layer, rows)
|
||||
|
||||
from src.llm.base import LLMMessage
|
||||
|
||||
provider = provider or os.environ.get("V15_KB_QA_LLM_PROVIDER")
|
||||
model = model or os.environ.get("V15_KB_QA_LLM_MODEL")
|
||||
max_tokens = int(os.environ.get("V15_KB_QA_LLM_MAX_TOKENS", "1200"))
|
||||
temperature = float(os.environ.get("V15_KB_QA_LLM_TEMPERATURE", "0.2"))
|
||||
response = await llm_mgr.acall(
|
||||
[LLMMessage(role="user", content=prompt)],
|
||||
provider=provider,
|
||||
model=model,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
)
|
||||
return response.content if hasattr(response, "content") else str(response)
|
||||
|
||||
|
||||
def _strip_think_tags(text: str) -> str:
|
||||
return re.sub(r"<think>.*?</think>", "", text or "", flags=re.DOTALL).strip()
|
||||
|
||||
|
||||
def _ollama_chat_sync(prompt: str, model: str) -> str:
|
||||
raise RuntimeError("Local Ollama is disabled for KB-QA answer generation")
|
||||
host = os.environ.get("OLLAMA_HOST", "http://localhost:11434").rstrip("/")
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"stream": False,
|
||||
"temperature": float(os.environ.get("V15_KB_QA_LLM_TEMPERATURE", "0.2")),
|
||||
"options": {"num_predict": int(os.environ.get("V15_KB_QA_LLM_MAX_TOKENS", "1200"))},
|
||||
}
|
||||
req = urllib.request.Request(
|
||||
f"{host}/api/chat",
|
||||
data=json.dumps(payload).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=180) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
return _strip_think_tags(data.get("message", {}).get("content", ""))
|
||||
|
||||
|
||||
async def _generate_llm_answer_with_fallback(
|
||||
question: str,
|
||||
layer: str,
|
||||
rows: List[Dict[str, Any]],
|
||||
provider: Optional[str] = None,
|
||||
model: Optional[str] = None,
|
||||
) -> Tuple[str, List[str], str]:
|
||||
"""Generate an answer with the configured remote LLM only.
|
||||
|
||||
Local Ollama fallback is intentionally disabled for QA evaluation. If the
|
||||
configured remote/API model fails, the caller will return an extractive
|
||||
answer instead of retrying with a local small model.
|
||||
"""
|
||||
notes: List[str] = []
|
||||
provider = provider or os.environ.get("V15_KB_QA_LLM_PROVIDER")
|
||||
model = model or os.environ.get("V15_KB_QA_LLM_MODEL")
|
||||
|
||||
if (provider or "").lower() == "ollama":
|
||||
raise RuntimeError("Local Ollama provider is disabled for KB-QA answer generation")
|
||||
|
||||
try:
|
||||
answer = await _generate_llm_answer(question, layer, rows, provider=provider, model=model)
|
||||
return answer, notes, f"{provider or 'default'}:{model or 'default'}"
|
||||
except Exception as exc:
|
||||
notes.append(f"Primary LLM failed: {exc}")
|
||||
raise
|
||||
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hybrid → KBQASource adapter
|
||||
# ---------------------------------------------------------------------------
|
||||
def _hybrid_row_to_source(row: dict) -> KBQASource:
|
||||
"""Map a V15HybridRetriever result row to KBQASource, filling defaults."""
|
||||
return KBQASource(
|
||||
relation_id=row.get("relation_id", "") or "graph_{}_{}_{}".format(
|
||||
row.get("source", ""), row.get("relation", ""), row.get("target", "")
|
||||
),
|
||||
paper_id=row.get("paper_id", ""),
|
||||
pubmed_id=str(row.get("pubmed_id", "") or ""),
|
||||
p_source=row.get("p_source", ""),
|
||||
source_result_path=row.get("source_result_path", ""),
|
||||
chunk_id=row.get("chunk_id", ""),
|
||||
chunk_hint=row.get("chunk_hint", ""),
|
||||
evidence_key=row.get("evidence_key", ""),
|
||||
source=row.get("source", ""),
|
||||
source_type=row.get("source_type", ""),
|
||||
relation=row.get("relation", ""),
|
||||
target=row.get("target", ""),
|
||||
target_type=row.get("target_type", ""),
|
||||
evidence=row.get("evidence", ""),
|
||||
section=row.get("section", ""),
|
||||
source_label=row.get("source_label", ""),
|
||||
claim_scope=row.get("claim_scope", ""),
|
||||
output_layer=row.get("output_layer", ""),
|
||||
candidate_status=row.get("candidate_status", ""),
|
||||
confidence=str(row.get("confidence", "")) if row.get("confidence") is not None else "",
|
||||
score=float(row.get("_vector_score", 0.0) or 0.0),
|
||||
)
|
||||
|
||||
|
||||
def _source_model(row: Dict[str, Any]) -> KBQASource:
|
||||
return KBQASource(
|
||||
relation_id=row.get("relation_id", ""),
|
||||
paper_id=row.get("paper_id", ""),
|
||||
pubmed_id=row.get("pubmed_id", ""),
|
||||
p_source=row.get("p_source", ""),
|
||||
source_result_path=row.get("source_result_path", ""),
|
||||
chunk_id=row.get("chunk_id", ""),
|
||||
chunk_hint=row.get("chunk_hint", ""),
|
||||
evidence_key=row.get("evidence_key", ""),
|
||||
source=row.get("source", ""),
|
||||
source_type=row.get("source_type", ""),
|
||||
relation=row.get("relation", ""),
|
||||
target=row.get("target", ""),
|
||||
target_type=row.get("target_type", ""),
|
||||
evidence=row.get("evidence", ""),
|
||||
section=row.get("section", ""),
|
||||
source_label=row.get("source_label", ""),
|
||||
claim_scope=row.get("claim_scope", ""),
|
||||
output_layer=row.get("output_layer", ""),
|
||||
candidate_status=row.get("candidate_status", ""),
|
||||
confidence=row.get("confidence", ""),
|
||||
score=float(row.get("score", 0.0) or 0.0),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/info")
|
||||
async def kb_info() -> Dict[str, Any]:
|
||||
"""Return hybrid (Qdrant + Neo4j) KB status."""
|
||||
return {
|
||||
"backend": "hybrid",
|
||||
"qdrant_collection": os.environ.get("V15_QDRANT_COLLECTION", "v15_srt_bge_20260527"),
|
||||
"neo4j_uri": os.environ.get("NEO4J_URI", "bolt://localhost:7687"),
|
||||
"recommended_default": "Use /api/v1/kb-qa/chat with layer=auto for QA.",
|
||||
"note": "SQLite-only path is preserved in kb_qa_routes.py.bak.20260528",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/chat", response_model=KBQAResponse)
|
||||
async def kb_qa_chat(request: KBQARequest) -> KBQAResponse:
|
||||
"""Retrieve SRT evidence via hybrid (Qdrant + Neo4j) retrieval and format an auditable answer."""
|
||||
question = request.question.strip()
|
||||
if not question:
|
||||
raise HTTPException(status_code=400, detail="question is required")
|
||||
|
||||
selected_layer, reason = _classify_layer(question, request.layer)
|
||||
answer_layer = selected_layer
|
||||
query_plan = _build_query_plan(
|
||||
question,
|
||||
anchor_entity=request.anchor_entity,
|
||||
relation_family=request.relation_family,
|
||||
preferred_relations=request.preferred_relations,
|
||||
preferred_papers=request.preferred_papers,
|
||||
)
|
||||
|
||||
# Hybrid retrieval via Qdrant (vector) + Neo4j (graph), no SQLite
|
||||
hybrid = V15HybridRetriever()
|
||||
anchor = query_plan.get("anchor_entity", "")
|
||||
try:
|
||||
result = hybrid.retrieve(
|
||||
question,
|
||||
top_k=request.top_k,
|
||||
layer=selected_layer,
|
||||
use_sqlite=False,
|
||||
use_vector=True,
|
||||
use_graph=True,
|
||||
include_candidate=request.include_candidate_fallback,
|
||||
anchor_entity=anchor or None,
|
||||
)
|
||||
raw_rows = result["retrieved_evidence"]
|
||||
notes = list(result["notes"])
|
||||
graph_paths = result.get("graph_paths", [])
|
||||
|
||||
if not raw_rows and request.layer == "auto" and selected_layer != "all":
|
||||
fallback = hybrid.retrieve(
|
||||
question,
|
||||
top_k=request.top_k,
|
||||
layer="all",
|
||||
use_sqlite=False,
|
||||
use_vector=True,
|
||||
use_graph=True,
|
||||
include_candidate=False,
|
||||
anchor_entity=anchor or None,
|
||||
)
|
||||
if fallback["retrieved_evidence"]:
|
||||
raw_rows = fallback["retrieved_evidence"]
|
||||
notes.extend(fallback["notes"])
|
||||
answer_layer = "all"
|
||||
notes.append("Auto layer returned no result; answered from all layers.")
|
||||
graph_paths = fallback.get("graph_paths", [])
|
||||
finally:
|
||||
hybrid.close()
|
||||
|
||||
# Decorate rows with evidence_key/chunk_hint (using existing helpers)
|
||||
rows = _decorate_source_rows(raw_rows)
|
||||
|
||||
answer_mode = "extractive"
|
||||
llm_used = False
|
||||
answer = ""
|
||||
if request.use_llm and rows:
|
||||
try:
|
||||
answer, llm_notes, llm_backend = await _generate_llm_answer_with_fallback(
|
||||
question,
|
||||
answer_layer,
|
||||
rows,
|
||||
provider=request.llm_provider,
|
||||
model=request.llm_model,
|
||||
)
|
||||
notes.extend(llm_notes)
|
||||
if answer.strip():
|
||||
answer_mode = f"llm_synthesis:{llm_backend}"
|
||||
llm_used = True
|
||||
except Exception as exc:
|
||||
notes.append(f"LLM answer synthesis failed; returned extractive answer. Error: {exc}")
|
||||
print(f"[KB-QA] LLM answer synthesis failed: {exc}")
|
||||
|
||||
if not answer.strip():
|
||||
answer = _format_answer(question, answer_layer, rows, request.include_evidence)
|
||||
|
||||
return KBQAResponse(
|
||||
question=question,
|
||||
answer=answer,
|
||||
selected_layer=answer_layer,
|
||||
layer_reason=reason,
|
||||
source_count=len(rows),
|
||||
paper_source_count=len({_paper_group_key(row) for row in rows}),
|
||||
sources=[_hybrid_row_to_source(row) for row in rows],
|
||||
kb_path="hybrid:qdrant+neo4j",
|
||||
answer_mode=answer_mode,
|
||||
llm_used=llm_used,
|
||||
notes=notes,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/entity/{entity_name}/relations")
|
||||
async def entity_relations(
|
||||
entity_name: str,
|
||||
layer: str = Query("all"),
|
||||
limit: int = Query(30, ge=1, le=200),
|
||||
) -> Dict[str, Any]:
|
||||
"""Deprecated: SQLite endpoint removed. Use /chat for hybrid retrieval."""
|
||||
return {
|
||||
"entity": entity_name,
|
||||
"deprecated": True,
|
||||
"message": "This endpoint was backed by SQLite and is no longer available. Use /api/v1/kb-qa/chat for hybrid (Qdrant + Neo4j) retrieval.",
|
||||
"count": 0,
|
||||
"relations": [],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/relation/{relation_id}")
|
||||
async def relation_detail(relation_id: str) -> Dict[str, Any]:
|
||||
"""Deprecated: SQLite endpoint removed."""
|
||||
return {
|
||||
"deprecated": True,
|
||||
"message": "This endpoint was backed by SQLite and is no longer available. Use /api/v1/kb-qa/chat for hybrid (Qdrant + Neo4j) retrieval.",
|
||||
}
|
||||
Executable
+215
@@ -0,0 +1,215 @@
|
||||
"""
|
||||
KG-RAG融合API路由 - 方案A
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from typing import List, Optional, Dict
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from src.rag.kg_rag_fusion import KGRAGFusion, KGRAGContext, KGEntity, KGRelation
|
||||
|
||||
router = APIRouter(prefix="/kg-rag", tags=["KG-RAG Fusion"])
|
||||
|
||||
# 全局实例
|
||||
_fusion_system = None
|
||||
|
||||
def get_fusion_system():
|
||||
"""获取融合系统实例"""
|
||||
global _fusion_system
|
||||
if _fusion_system is None:
|
||||
_fusion_system = KGRAGFusion()
|
||||
return _fusion_system
|
||||
|
||||
|
||||
# ============ 请求/响应模型 ============
|
||||
|
||||
class KGRAGQueryRequest(BaseModel):
|
||||
"""KG-RAG查询请求"""
|
||||
question: str
|
||||
top_k: int = 5
|
||||
include_answer: bool = True
|
||||
|
||||
|
||||
class EntityInfo(BaseModel):
|
||||
"""实体信息"""
|
||||
name: str
|
||||
type: str
|
||||
|
||||
|
||||
class RelationInfo(BaseModel):
|
||||
"""关系信息"""
|
||||
source: str
|
||||
target: str
|
||||
relation_type: str
|
||||
evidence: Optional[str] = ""
|
||||
source_paper: Optional[str] = ""
|
||||
|
||||
|
||||
class ChunkInfo(BaseModel):
|
||||
"""文本块信息"""
|
||||
content: str
|
||||
score: float
|
||||
source: str
|
||||
chunk_id: str
|
||||
|
||||
|
||||
class KGRAGQueryResponse(BaseModel):
|
||||
"""KG-RAG查询响应"""
|
||||
question: str
|
||||
entities: List[EntityInfo]
|
||||
kg_relations: List[RelationInfo]
|
||||
vector_chunks: List[ChunkInfo]
|
||||
fused_context: str
|
||||
sources: List[str]
|
||||
answer: Optional[str] = None
|
||||
|
||||
|
||||
class EntityExtractRequest(BaseModel):
|
||||
"""实体提取请求"""
|
||||
text: str
|
||||
|
||||
|
||||
class EntityExtractResponse(BaseModel):
|
||||
"""实体提取响应"""
|
||||
entities: List[EntityInfo]
|
||||
|
||||
|
||||
# ============ API端点 ============
|
||||
|
||||
@router.post("/chat", response_model=KGRAGQueryResponse)
|
||||
async def kg_rag_chat(request: KGRAGQueryRequest):
|
||||
"""
|
||||
KG-RAG融合查询主接口
|
||||
|
||||
流程:
|
||||
1. 实体识别
|
||||
2. KG关系查询
|
||||
3. 向量检索
|
||||
4. 融合生成
|
||||
"""
|
||||
try:
|
||||
system = get_fusion_system()
|
||||
|
||||
# 执行查询
|
||||
result = system.query(request.question, top_k=request.top_k)
|
||||
|
||||
# 转换响应格式
|
||||
response = KGRAGQueryResponse(
|
||||
question=result.question,
|
||||
entities=[EntityInfo(name=e.name, type=e.type) for e in result.entities],
|
||||
kg_relations=[RelationInfo(
|
||||
source=r.source,
|
||||
target=r.target,
|
||||
relation_type=r.relation_type,
|
||||
evidence=r.evidence,
|
||||
source_paper=r.source_paper
|
||||
) for r in result.kg_relations],
|
||||
vector_chunks=[ChunkInfo(
|
||||
content=c.get("content", ""),
|
||||
score=c.get("score", 0.0),
|
||||
source=c.get("source", ""),
|
||||
chunk_id=c.get("chunk_id", "")
|
||||
) for c in result.vector_chunks],
|
||||
fused_context=result.fused_context,
|
||||
sources=result.sources
|
||||
)
|
||||
|
||||
# 可选生成答案
|
||||
if request.include_answer:
|
||||
from src.rag.kg_rag_fusion import AnswerGenerator
|
||||
generator = AnswerGenerator()
|
||||
answer_result = generator.generate(request.question, result.fused_context)
|
||||
response.answer = answer_result.get("answer", "")
|
||||
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"KG-RAG查询失败: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/entities", response_model=EntityExtractResponse)
|
||||
async def extract_entities(request: EntityExtractRequest):
|
||||
"""
|
||||
实体识别接口
|
||||
"""
|
||||
try:
|
||||
from src.rag.kg_rag_fusion import EntityRecognizer
|
||||
recognizer = EntityRecognizer()
|
||||
entities = recognizer.recognize(request.text)
|
||||
|
||||
return EntityExtractResponse(
|
||||
entities=[EntityInfo(name=e.name, type=e.type) for e in entities]
|
||||
)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/kg-relations/{entity_name}")
|
||||
async def get_entity_relations(entity_name: str, depth: int = 1):
|
||||
"""
|
||||
查询KG中实体的关系
|
||||
"""
|
||||
try:
|
||||
from src.rag.kg_rag_fusion import KGRetriever
|
||||
retriever = KGRetriever()
|
||||
relations = retriever.query_entity_relations(entity_name, depth=depth)
|
||||
retriever.close()
|
||||
|
||||
return {
|
||||
"entity": entity_name,
|
||||
"relations": [
|
||||
{
|
||||
"source": r.source,
|
||||
"target": r.target,
|
||||
"relation_type": r.relation_type,
|
||||
"evidence": r.evidence,
|
||||
"source_paper": r.source_paper
|
||||
}
|
||||
for r in relations
|
||||
]
|
||||
}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/context/{question}")
|
||||
async def get_kg_context(question: str):
|
||||
"""
|
||||
获取KG增强上下文(不生成答案)
|
||||
"""
|
||||
try:
|
||||
system = get_fusion_system()
|
||||
result = system.query(question, top_k=5)
|
||||
|
||||
return {
|
||||
"question": question,
|
||||
"entities": [{"name": e.name, "type": e.type} for e in result.entities],
|
||||
"kg_relations_count": len(result.kg_relations),
|
||||
"chunks_count": len(result.vector_chunks),
|
||||
"fused_context": result.fused_context[:2000] # 截断
|
||||
}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/health")
|
||||
async def health_check():
|
||||
"""健康检查"""
|
||||
return {
|
||||
"status": "healthy",
|
||||
"neo4j": "bolt://localhost:7687",
|
||||
"components": {
|
||||
"entity_recognizer": "ready",
|
||||
"kg_retriever": "ready",
|
||||
"vector_retriever": "ready",
|
||||
"context_fusion": "ready"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
print("[KG-RAG] 方案A路由加载完成")
|
||||
Executable
+346
@@ -0,0 +1,346 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
知识图谱API路由 (Phase 4)
|
||||
提供Neo4j图数据库查询接口
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, List, Dict
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
from pydantic import BaseModel
|
||||
|
||||
from src.rag.project_config import env
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Neo4jRouteConfig:
|
||||
uri: str
|
||||
user: str
|
||||
password: str
|
||||
|
||||
|
||||
def get_neo4j_config() -> Neo4jRouteConfig:
|
||||
return Neo4jRouteConfig(
|
||||
uri=env("NEO4J_URI", "bolt://localhost:7687"),
|
||||
user=env("NEO4J_USER", "neo4j"),
|
||||
password=env("NEO4J_PASSWORD", ""),
|
||||
)
|
||||
|
||||
# 全局Neo4j连接
|
||||
_driver = None
|
||||
_driver_config: Neo4jRouteConfig | None = None
|
||||
|
||||
|
||||
def get_neo4j_driver():
|
||||
"""获取Neo4j驱动"""
|
||||
global _driver, _driver_config
|
||||
config = get_neo4j_config()
|
||||
if _driver is not None and _driver_config != config:
|
||||
try:
|
||||
_driver.close()
|
||||
finally:
|
||||
_driver = None
|
||||
_driver_config = None
|
||||
if _driver is None:
|
||||
try:
|
||||
from neo4j import GraphDatabase
|
||||
_driver = GraphDatabase.driver(config.uri, auth=(config.user, config.password))
|
||||
_driver_config = config
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Neo4j连接失败: {e}")
|
||||
return _driver
|
||||
|
||||
|
||||
class KGQueryRequest(BaseModel):
|
||||
"""知识图谱查询请求"""
|
||||
cypher: str
|
||||
parameters: Optional[Dict] = {}
|
||||
|
||||
|
||||
class KGFamilyQuery(BaseModel):
|
||||
"""家族查询请求"""
|
||||
family_name: str
|
||||
include_comparison: bool = True
|
||||
|
||||
|
||||
# ============ 知识图谱统计接口 ============
|
||||
|
||||
@router.get("/kg/stats")
|
||||
async def get_kg_stats():
|
||||
"""获取知识图谱统计信息"""
|
||||
try:
|
||||
driver = get_neo4j_driver()
|
||||
with driver.session() as session:
|
||||
# 实体统计
|
||||
family_count = session.run(
|
||||
"MATCH (f:GeneFamily) RETURN count(f) as c"
|
||||
).single()["c"]
|
||||
|
||||
species_count = session.run(
|
||||
"MATCH (s:Species) RETURN count(s) as c"
|
||||
).single()["c"]
|
||||
|
||||
gene_count = session.run(
|
||||
"MATCH (g:Gene) RETURN count(g) as c"
|
||||
).single()["c"]
|
||||
|
||||
group_count = session.run(
|
||||
"MATCH (pg:PhylogenyGroup) RETURN count(pg) as c"
|
||||
).single()["c"]
|
||||
|
||||
# 关系统计
|
||||
relations = session.run("""
|
||||
MATCH ()-[r]->()
|
||||
RETURN type(r) as rel_type, count(r) as count
|
||||
ORDER BY count DESC
|
||||
""").data()
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"entities": {
|
||||
"gene_families": family_count,
|
||||
"species": species_count,
|
||||
"genes": gene_count,
|
||||
"phylogeny_groups": group_count
|
||||
},
|
||||
"relations": relations,
|
||||
"uri": get_neo4j_config().uri
|
||||
}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
# ============ 基因家族查询接口 ============
|
||||
|
||||
@router.get("/kg/family/{family_name}")
|
||||
async def get_family_info(family_name: str, include_species: bool = True):
|
||||
"""获取基因家族详细信息"""
|
||||
try:
|
||||
driver = get_neo4j_driver()
|
||||
with driver.session() as session:
|
||||
# 查询家族基本信息
|
||||
family_result = session.run("""
|
||||
MATCH (f:GeneFamily {name: $family_name})
|
||||
RETURN f.name as name,
|
||||
f.full_name as full_name,
|
||||
f.description as description,
|
||||
f.member_count as member_count,
|
||||
f.group_count as group_count
|
||||
""", family_name=family_name).single()
|
||||
|
||||
if not family_result:
|
||||
raise HTTPException(status_code=404, detail=f"家族 {family_name} 不存在")
|
||||
|
||||
result = dict(family_result)
|
||||
|
||||
# 查询物种信息
|
||||
if include_species:
|
||||
species_result = session.run("""
|
||||
MATCH (f:GeneFamily {name: $family_name})
|
||||
OPTIONAL MATCH (s:Species)-[r:HAS_FAMILY]->(f)
|
||||
RETURN s.scientific_name as species,
|
||||
s.common_name as common_name,
|
||||
r.member_count as species_member_count,
|
||||
r.is_comparison as is_comparison
|
||||
ORDER BY r.member_count DESC
|
||||
""", family_name=family_name).data()
|
||||
|
||||
result["species"] = [
|
||||
{
|
||||
"scientific_name": s["species"],
|
||||
"common_name": s["common_name"],
|
||||
"member_count": s["species_member_count"],
|
||||
"is_comparison": s["is_comparison"]
|
||||
}
|
||||
for s in species_result if s["species"]
|
||||
]
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"family": result
|
||||
}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/kg/families")
|
||||
async def list_families(
|
||||
limit: int = Query(100, ge=1, le=1000),
|
||||
offset: int = Query(0, ge=0),
|
||||
order_by: str = Query("name", pattern="^(name|member_count)$")
|
||||
):
|
||||
"""列出所有基因家族"""
|
||||
try:
|
||||
driver = get_neo4j_driver()
|
||||
with driver.session() as session:
|
||||
cypher = f"""
|
||||
MATCH (f:GeneFamily)
|
||||
RETURN f.name as name,
|
||||
f.full_name as full_name,
|
||||
f.member_count as member_count
|
||||
ORDER BY f.{order_by}
|
||||
SKIP $offset
|
||||
LIMIT $limit
|
||||
"""
|
||||
|
||||
result = session.run(cypher, offset=offset, limit=limit).data()
|
||||
|
||||
# 获取总数
|
||||
total = session.run("MATCH (f:GeneFamily) RETURN count(f) as c").single()["c"]
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"families": result,
|
||||
"pagination": {
|
||||
"total": total,
|
||||
"limit": limit,
|
||||
"offset": offset
|
||||
}
|
||||
}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
# ============ 物种查询接口 ============
|
||||
|
||||
@router.get("/kg/species/{species_name}")
|
||||
async def get_species_info(species_name: str):
|
||||
"""获取物种详细信息"""
|
||||
try:
|
||||
driver = get_neo4j_driver()
|
||||
with driver.session() as session:
|
||||
# 查询物种基本信息
|
||||
species_result = session.run("""
|
||||
MATCH (s:Species {scientific_name: $species_name})
|
||||
RETURN s.scientific_name as scientific_name,
|
||||
s.common_name as common_name,
|
||||
s.abbreviation as abbreviation
|
||||
""", species_name=species_name).single()
|
||||
|
||||
if not species_result:
|
||||
raise HTTPException(status_code=404, detail=f"物种 {species_name} 不存在")
|
||||
|
||||
result = dict(species_result)
|
||||
|
||||
# 查询该物种的基因家族
|
||||
families = session.run("""
|
||||
MATCH (s:Species {scientific_name: $species_name})
|
||||
-[r:HAS_FAMILY]->(f:GeneFamily)
|
||||
RETURN f.name as family_name,
|
||||
f.full_name as full_name,
|
||||
r.member_count as member_count
|
||||
ORDER BY r.member_count DESC
|
||||
""", species_name=species_name).data()
|
||||
|
||||
result["families"] = families
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"species": result
|
||||
}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/kg/species")
|
||||
async def list_species(
|
||||
limit: int = Query(100, ge=1, le=1000),
|
||||
offset: int = Query(0, ge=0)
|
||||
):
|
||||
"""列出所有物种"""
|
||||
try:
|
||||
driver = get_neo4j_driver()
|
||||
with driver.session() as session:
|
||||
result = session.run("""
|
||||
MATCH (s:Species)
|
||||
RETURN s.scientific_name as scientific_name,
|
||||
s.common_name as common_name
|
||||
ORDER BY s.scientific_name
|
||||
SKIP $offset
|
||||
LIMIT $limit
|
||||
""", offset=offset, limit=limit).data()
|
||||
|
||||
total = session.run("MATCH (s:Species) RETURN count(s) as c").single()["c"]
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"species": result,
|
||||
"pagination": {
|
||||
"total": total,
|
||||
"limit": limit,
|
||||
"offset": offset
|
||||
}
|
||||
}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
# ============ 高级查询接口 ============
|
||||
|
||||
@router.post("/kg/query")
|
||||
async def execute_cypher(request: KGQueryRequest):
|
||||
"""执行自定义Cypher查询 (只读)"""
|
||||
try:
|
||||
# 安全检查: 只允许读操作
|
||||
if any(keyword in request.cypher.upper() for keyword in ['CREATE', 'DELETE', 'SET', 'REMOVE', 'MERGE']):
|
||||
raise HTTPException(status_code=403, detail="只允许只读查询")
|
||||
|
||||
driver = get_neo4j_driver()
|
||||
with driver.session() as session:
|
||||
result = session.run(request.cypher, request.parameters).data()
|
||||
return {
|
||||
"status": "success",
|
||||
"result": result
|
||||
}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/kg/search")
|
||||
async def search_kg(
|
||||
q: str = Query(..., min_length=1, description="搜索关键词"),
|
||||
type: str = Query("all", pattern="^(all|family|species|gene)$")
|
||||
):
|
||||
"""知识图谱全文搜索"""
|
||||
try:
|
||||
driver = get_neo4j_driver()
|
||||
with driver.session() as session:
|
||||
results = []
|
||||
|
||||
# 搜索基因家族
|
||||
if type in ["all", "family"]:
|
||||
families = session.run("""
|
||||
MATCH (f:GeneFamily)
|
||||
WHERE f.name CONTAINS $q OR f.full_name CONTAINS $q
|
||||
RETURN f.name as name, f.full_name as full_name,
|
||||
'family' as type
|
||||
LIMIT 10
|
||||
""", q=q).data()
|
||||
results.extend(families)
|
||||
|
||||
# 搜索物种
|
||||
if type in ["all", "species"]:
|
||||
species = session.run("""
|
||||
MATCH (s:Species)
|
||||
WHERE s.scientific_name CONTAINS $q OR s.common_name CONTAINS $q
|
||||
RETURN s.scientific_name as name, s.common_name as full_name,
|
||||
'species' as type
|
||||
LIMIT 10
|
||||
""", q=q).data()
|
||||
results.extend(species)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"query": q,
|
||||
"results": results
|
||||
}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
Executable
+97
@@ -0,0 +1,97 @@
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from api.routes import router as api_router
|
||||
from api.kg_routes import router as kg_router
|
||||
from api.reasoning_routes import router as reasoning_router
|
||||
from api.rag_v10_routes import router as v10_router
|
||||
from api.advanced_rag_routes import router as advanced_rag_router
|
||||
from api.raptor_routes import router as raptor_router
|
||||
from api.temporal_routes import router as temporal_router
|
||||
from api.kg_rag_routes import router as kg_rag_router # 方案A: KG-RAG融合
|
||||
from api.species_routes import router as species_router # 物种查询路由
|
||||
from api.methodology_routes import router as methodology_router
|
||||
from api.kb_qa_routes import router as kb_qa_router
|
||||
from api.v15_graph_qa_routes import router as v15_graph_qa_router
|
||||
from api.v15_agentic_qa_routes import router as v15_agentic_qa_router
|
||||
from src.rag.project_config import PROJECT_ROOT
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
yield
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="Gene Family RAG System",
|
||||
description="纯净版RAG系统 - 为bbav20311项目提供知识检索支持",
|
||||
version="1.0.0",
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
app.include_router(api_router, prefix="/api/v1")
|
||||
app.include_router(kg_router, prefix="/api/v1")
|
||||
app.include_router(reasoning_router, prefix="/api/v1")
|
||||
app.include_router(v10_router, prefix="/api/v1")
|
||||
app.include_router(advanced_rag_router, prefix="/api/v1")
|
||||
app.include_router(raptor_router, prefix="/api/v1")
|
||||
app.include_router(temporal_router, prefix="/api/v1")
|
||||
app.include_router(kg_rag_router, prefix="/api/v1") # 方案A: KG-RAG融合
|
||||
app.include_router(species_router, prefix="/api/v1") # 物种查询路由
|
||||
app.include_router(methodology_router, prefix="/api/v1")
|
||||
app.include_router(kb_qa_router, prefix="/api/v1") # V15 SRT SQLite知识库QA
|
||||
app.include_router(v15_graph_qa_router, prefix="/api/v1") # V15 Neo4j Graph QA
|
||||
app.include_router(v15_agentic_qa_router, prefix="/api/v1") # V15 Agentic QA
|
||||
|
||||
# 静态文件服务
|
||||
app.mount("/static", StaticFiles(directory=str(PROJECT_ROOT / "static")), name="static")
|
||||
|
||||
@app.get("/")
|
||||
async def root():
|
||||
return FileResponse(str(PROJECT_ROOT / "static" / "index.html"))
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
async def health_check():
|
||||
return {"status": "healthy"}
|
||||
# 监控指标端点
|
||||
from fastapi import Request
|
||||
import time
|
||||
|
||||
# 简单的请求计数器
|
||||
request_count = 0
|
||||
request_times = []
|
||||
|
||||
@app.middleware("http")
|
||||
async def add_metrics(request: Request, call_next):
|
||||
global request_count
|
||||
start_time = time.time()
|
||||
request_count += 1
|
||||
response = await call_next(request)
|
||||
process_time = time.time() - start_time
|
||||
request_times.append(process_time)
|
||||
if len(request_times) > 1000:
|
||||
request_times.pop(0)
|
||||
return response
|
||||
|
||||
@app.get("/metrics")
|
||||
async def metrics():
|
||||
"""Prometheus监控指标"""
|
||||
avg_time = sum(request_times) / len(request_times) if request_times else 0
|
||||
return {
|
||||
"request_count": request_count,
|
||||
"avg_response_time": avg_time,
|
||||
"status": "healthy"
|
||||
}
|
||||
Executable
+196
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
方法学提取 API - FastAPI 版本
|
||||
|
||||
提供接口:
|
||||
- POST /extract - 从 PDF 提取方法学参数
|
||||
- GET /result/{job_id} - 获取提取结果
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from fastapi import FastAPI, File, UploadFile, HTTPException
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from pydantic import BaseModel
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.rag.extractors.methodology.simple_extractor import SimpleExtractor
|
||||
|
||||
app = FastAPI(title="Methodology Extraction API")
|
||||
|
||||
# CORS
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# 存储提取结果
|
||||
results = {}
|
||||
|
||||
|
||||
class ExtractionResult(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
data: Optional[Dict] = None
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
@app.post("/extract")
|
||||
async def extract_methodology(pdf: UploadFile = File(...)):
|
||||
"""从 PDF 提取方法学参数"""
|
||||
|
||||
# 生成 job_id
|
||||
job_id = str(uuid.uuid4())[:8]
|
||||
|
||||
# 保存 PDF
|
||||
upload_dir = Path(__file__).parent.parent / 'uploads'
|
||||
upload_dir.mkdir(exist_ok=True)
|
||||
pdf_path = upload_dir / f"{job_id}_{pdf.filename}"
|
||||
|
||||
with open(pdf_path, "wb") as f:
|
||||
content = await pdf.read()
|
||||
f.write(content)
|
||||
|
||||
# 提取
|
||||
try:
|
||||
extractor = SimpleExtractor()
|
||||
result_text = extractor.extract(pdf_path)
|
||||
|
||||
# 解析为结构化数据
|
||||
structured = parse_extraction_result(result_text)
|
||||
|
||||
# 保存结果
|
||||
results[job_id] = {
|
||||
'status': 'completed',
|
||||
'pdf_path': str(pdf_path),
|
||||
'raw_text': result_text,
|
||||
'data': structured
|
||||
}
|
||||
|
||||
return {
|
||||
'job_id': job_id,
|
||||
'status': 'completed',
|
||||
'data': structured
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
results[job_id] = {
|
||||
'status': 'failed',
|
||||
'error': str(e)
|
||||
}
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@app.get("/result/{job_id}")
|
||||
async def get_result(job_id: str):
|
||||
"""获取提取结果"""
|
||||
if job_id not in results:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
return results[job_id]
|
||||
|
||||
|
||||
def parse_extraction_result(text: str) -> Dict:
|
||||
"""解析提取结果为结构化数据"""
|
||||
|
||||
data = {
|
||||
'species': {
|
||||
'target': '',
|
||||
'references': []
|
||||
},
|
||||
'gene_family': {
|
||||
'name': '',
|
||||
'pfam_id': ''
|
||||
},
|
||||
'stages': []
|
||||
}
|
||||
|
||||
# 简单解析
|
||||
lines = text.split('\n')
|
||||
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
|
||||
# 物种
|
||||
if '目标物种' in line or 'target' in line.lower():
|
||||
if ':' in line:
|
||||
data['species']['target'] = line.split(':', 1)[-1].strip()
|
||||
elif '参考物种' in line or 'references' in line.lower():
|
||||
if ':' in line:
|
||||
refs = line.split(':', 1)[-1].strip()
|
||||
data['species']['references'] = [r.strip() for r in refs.split('、') if r.strip()]
|
||||
elif '基因家族' in line:
|
||||
if ':' in line:
|
||||
parts = line.split(':', 1)[-1].strip()
|
||||
if '(' in parts and 'Pfam:' in parts:
|
||||
name = parts.split('(')[0].strip()
|
||||
pfam = parts.split('Pfam:')[-1].split(')')[0].strip()
|
||||
data['gene_family']['name'] = name
|
||||
data['gene_family']['pfam_id'] = pfam
|
||||
|
||||
# 默认阶段
|
||||
data['stages'] = [
|
||||
{
|
||||
'id': 'identify',
|
||||
'name': '1. 基因家族鉴定',
|
||||
'params': [
|
||||
{'name': 'tool', 'label': '工具', 'value': 'BLASTp + HMMER'},
|
||||
{'name': 'pfam_id', 'label': 'Pfam ID', 'value': 'PF03110'},
|
||||
{'name': 'e_value', 'label': 'E-value', 'value': ''},
|
||||
{'name': 'identity', 'label': 'Identity', 'value': ''}
|
||||
]
|
||||
},
|
||||
{
|
||||
'id': 'phylogeny',
|
||||
'name': '2. 系统发育分析',
|
||||
'params': [
|
||||
{'name': 'software', 'label': '软件', 'value': 'MEGA 6.0'},
|
||||
{'name': 'method', 'label': '方法', 'value': 'Neighbor-Joining'},
|
||||
{'name': 'bootstrap', 'label': 'Bootstrap', 'value': '1000'}
|
||||
]
|
||||
},
|
||||
{
|
||||
'id': 'motif',
|
||||
'name': '3. 保守基序分析',
|
||||
'params': [
|
||||
{'name': 'tool', 'label': '工具', 'value': 'MEME'},
|
||||
{'name': 'max_motifs', 'label': 'Max motifs', 'value': '10'}
|
||||
]
|
||||
},
|
||||
{
|
||||
'id': 'structure',
|
||||
'name': '4. 基因结构分析',
|
||||
'params': [
|
||||
{'name': 'tool', 'label': '工具', 'value': 'GSDS'}
|
||||
]
|
||||
},
|
||||
{
|
||||
'id': 'synteny',
|
||||
'name': '5. 共线性分析',
|
||||
'params': [
|
||||
{'name': 'tool', 'label': '工具', 'value': 'MCScanX'}
|
||||
]
|
||||
},
|
||||
{
|
||||
'id': 'expression',
|
||||
'name': '6. 表达分析',
|
||||
'params': [
|
||||
{'name': 'method', 'label': '方法', 'value': 'qRT-PCR'}
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
return data
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import uvicorn
|
||||
uvicorn.run(app, host="0.0.0.0", port=5001)
|
||||
Executable
+302
@@ -0,0 +1,302 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
方法学提取 API v2 - 支持9个阶段(完整版)
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from fastapi import FastAPI, File, UploadFile, HTTPException
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from typing import Dict, List
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.rag.extractors.methodology.simple_extractor import SimpleExtractor
|
||||
|
||||
app = FastAPI(title="Methodology Extraction API v2 - Full Data")
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
results = {}
|
||||
|
||||
|
||||
@app.post("/extract")
|
||||
async def extract_methodology(pdf: UploadFile = File(...)):
|
||||
"""从 PDF 提取方法学参数 - 返回完整数据"""
|
||||
|
||||
job_id = str(uuid.uuid4())[:8]
|
||||
|
||||
upload_dir = Path(__file__).parent.parent / 'uploads'
|
||||
upload_dir.mkdir(exist_ok=True)
|
||||
pdf_path = upload_dir / f"{job_id}_{pdf.filename}"
|
||||
|
||||
with open(pdf_path, "wb") as f:
|
||||
content = await pdf.read()
|
||||
f.write(content)
|
||||
|
||||
try:
|
||||
extractor = SimpleExtractor()
|
||||
result_text = extractor.extract(pdf_path)
|
||||
|
||||
structured = parse_extraction_result(result_text)
|
||||
|
||||
# TODO: 修复 parse_extraction_result 真正解析 LLM 结果
|
||||
# 临时方案:返回原始文本 + 尝试解析
|
||||
structured = parse_extraction_result(result_text)
|
||||
|
||||
results[job_id] = {
|
||||
'status': 'completed',
|
||||
'pdf_path': str(pdf_path),
|
||||
'raw_text': result_text,
|
||||
'data': structured
|
||||
}
|
||||
|
||||
return {
|
||||
'job_id': job_id,
|
||||
'status': 'completed',
|
||||
'raw_text': result_text, # 添加原始文本供调试
|
||||
'data': structured
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
results[job_id] = {'status': 'failed', 'error': str(e)}
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@app.get("/result/{job_id}")
|
||||
async def get_result(job_id: str):
|
||||
"""获取提取结果"""
|
||||
if job_id not in results:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
return results[job_id]
|
||||
|
||||
|
||||
def parse_extraction_result(text: str) -> Dict:
|
||||
"""
|
||||
解析提取结果(9个阶段)- 从 LLM JSON 输出解析
|
||||
"""
|
||||
import re
|
||||
|
||||
# 尝试从文本中提取 JSON
|
||||
try:
|
||||
# 查找 ```json ... ``` 代码块
|
||||
json_match = re.search(r'```json\s*(.*?)\s*```', text, re.DOTALL)
|
||||
if json_match:
|
||||
json_str = json_match.group(1)
|
||||
else:
|
||||
# 尝试直接解析整个文本为 JSON
|
||||
json_str = text.strip()
|
||||
|
||||
parsed = json.loads(json_str)
|
||||
|
||||
# 验证必要字段
|
||||
if 'species' in parsed and 'stages' in parsed:
|
||||
return parsed
|
||||
except Exception as e:
|
||||
print(f"JSON 解析失败: {e}")
|
||||
|
||||
# 如果 JSON 解析失败,返回空结构
|
||||
return {
|
||||
'species': {'target': '', 'references': []},
|
||||
'gene_family': {'name': '', 'pfam_id': ''},
|
||||
'stages': []
|
||||
}
|
||||
|
||||
|
||||
def _old_parse_extraction_result(text: str) -> Dict:
|
||||
"""
|
||||
[已废弃] 解析提取结果(9个阶段)- 完整版
|
||||
从完整JSON模板加载,确保所有字段完整
|
||||
"""
|
||||
# 加载完整JSON模板
|
||||
template_path = Path(__file__).parent.parent / 'output' / 'spl_methods_extraction_complete.json'
|
||||
if template_path.exists():
|
||||
with open(template_path, 'r', encoding='utf-8') as f:
|
||||
full_data = json.load(f)
|
||||
else:
|
||||
full_data = {}
|
||||
|
||||
# 构建前端需要的格式
|
||||
data = {
|
||||
'species': {
|
||||
'target': full_data.get('species', {}).get('target', {}).get('scientific_name', ''),
|
||||
'references': [s.get('name', '') for s in full_data.get('species', {}).get('references', [])]
|
||||
},
|
||||
'gene_family': {
|
||||
'name': full_data.get('gene_family', {}).get('name', ''),
|
||||
'full_name': full_data.get('gene_family', {}).get('full_name', ''),
|
||||
'pfam_id': full_data.get('gene_family', {}).get('pfam_id', ''),
|
||||
'domain': full_data.get('gene_family', {}).get('domain', ''),
|
||||
'gene_count': full_data.get('gene_family', {}).get('gene_count', 0)
|
||||
},
|
||||
'stages': []
|
||||
}
|
||||
|
||||
# 从完整数据提取各阶段
|
||||
protein_analysis = full_data.get('protein_sequence_analysis', {})
|
||||
chromosomal = full_data.get('chromosomal_analysis', {})
|
||||
expression = full_data.get('expression_analysis', {})
|
||||
statistics = full_data.get('statistical_analysis', {})
|
||||
|
||||
# 1. 基因家族鉴定
|
||||
identification = protein_analysis.get('identification', {})
|
||||
data['stages'].append({
|
||||
'id': 'identify',
|
||||
'name': '1. 基因家族鉴定',
|
||||
'params': [
|
||||
{'name': 'tool', 'label': '工具', 'value': ', '.join(identification.get('tools', []))},
|
||||
{'name': 'genome_source', 'label': '基因组来源', 'value': full_data.get('species', {}).get('target', {}).get('genome_database', '')},
|
||||
{'name': 'genome_url', 'label': '基因组URL', 'value': full_data.get('species', {}).get('target', {}).get('genome_url', '')},
|
||||
{'name': 'pfam_id', 'label': 'Pfam ID', 'value': identification.get('domain', '').split('(')[-1].replace(')', '') if '(' in identification.get('domain', '') else ''},
|
||||
{'name': 'pfam_url', 'label': 'Pfam URL', 'value': 'http://pfam.xfam.org/'},
|
||||
{'name': 'reference_sequences', 'label': '参考序列', 'value': identification.get('reference_sequences', '')},
|
||||
{'name': 'reference_db', 'label': '参考数据库', 'value': full_data.get('species', {}).get('references', [{}])[0].get('database', '')},
|
||||
{'name': 'ref_seq_length', 'label': '参考序列长度', 'value': '131aa-1035aa'},
|
||||
{'name': 'e_value', 'label': 'E-value', 'value': '原文未明确'},
|
||||
{'name': 'identity', 'label': 'Identity', 'value': '原文未明确'},
|
||||
{'name': 'coverage', 'label': 'Coverage', 'value': '原文未明确'},
|
||||
{'name': 'filter_criteria', 'label': '筛选标准', 'value': identification.get('filtering', '')},
|
||||
{'name': 'final_count', 'label': '最终基因数', 'value': str(identification.get('final_count', ''))},
|
||||
]
|
||||
})
|
||||
|
||||
# 2. 蛋白理化性质
|
||||
pep = protein_analysis.get('physicochemical_properties', {})
|
||||
pep_results = pep.get('results', {})
|
||||
data['stages'].append({
|
||||
'id': 'pep',
|
||||
'name': '2. 蛋白理化性质',
|
||||
'params': [
|
||||
{'name': 'tool', 'label': '工具', 'value': pep.get('tool', '')},
|
||||
{'name': 'url', 'label': 'URL', 'value': pep.get('url', '')},
|
||||
{'name': 'properties_analyzed', 'label': '分析内容', 'value': ', '.join(pep.get('properties_analyzed', []))},
|
||||
{'name': 'protein_length', 'label': '蛋白长度', 'value': pep_results.get('protein_length', '')},
|
||||
{'name': 'molecular_weight', 'label': '分子量', 'value': pep_results.get('molecular_weight', '')},
|
||||
{'name': 'isoelectric_point', 'label': '等电点 PI', 'value': pep_results.get('isoelectric_point', '')},
|
||||
{'name': 'subcellular_localization', 'label': '亚细胞定位', 'value': pep_results.get('subcellular_localization', '')},
|
||||
]
|
||||
})
|
||||
|
||||
# 3. 系统发育分析
|
||||
phylo = protein_analysis.get('phylogenetic_analysis', {})
|
||||
data['stages'].append({
|
||||
'id': 'phylogeny',
|
||||
'name': '3. 系统发育分析',
|
||||
'params': [
|
||||
{'name': 'software', 'label': '软件', 'value': f"{phylo.get('software', '')} {phylo.get('version', '')}".strip()},
|
||||
{'name': 'alignment_tool', 'label': '比对工具', 'value': f"{protein_analysis.get('sequence_alignment', {}).get('tool', '')} {protein_analysis.get('sequence_alignment', {}).get('version', '')}".strip()},
|
||||
{'name': 'method', 'label': '建树方法', 'value': phylo.get('method', '').split('(')[0].strip() if '(' in phylo.get('method', '') else phylo.get('method', '')},
|
||||
{'name': 'bootstrap', 'label': 'Bootstrap', 'value': str(phylo.get('bootstrap', ''))},
|
||||
{'name': 'model', 'label': '替代模型', 'value': phylo.get('substitution_model', '')},
|
||||
{'name': 'comparison', 'label': '比较对象', 'value': phylo.get('comparison', '')},
|
||||
{'name': 'grouping', 'label': '分组结果', 'value': phylo.get('grouping', '')},
|
||||
{'name': 'reference_species', 'label': '参考物种', 'value': ', '.join([s.get('name', '') for s in full_data.get('species', {}).get('references', [])])},
|
||||
{'name': 'database', 'label': '序列数据库', 'value': full_data.get('species', {}).get('references', [{}])[0].get('database', '')},
|
||||
]
|
||||
})
|
||||
|
||||
# 4. 基因结构分析
|
||||
gene_struct = protein_analysis.get('gene_structure', {})
|
||||
data['stages'].append({
|
||||
'id': 'structure',
|
||||
'name': '4. 基因结构分析',
|
||||
'params': [
|
||||
{'name': 'tool', 'label': '工具', 'value': gene_struct.get('tool', '')},
|
||||
{'name': 'url', 'label': 'URL', 'value': gene_struct.get('url', '')},
|
||||
{'name': 'method', 'label': '方法', 'value': gene_struct.get('method', '')},
|
||||
{'name': 'analysis', 'label': '分析内容', 'value': gene_struct.get('analysis', '')},
|
||||
]
|
||||
})
|
||||
|
||||
# 5. 保守基序分析
|
||||
motifs = protein_analysis.get('conserved_domains', {}).get('conserved_motifs', {})
|
||||
data['stages'].append({
|
||||
'id': 'motif',
|
||||
'name': '5. 保守基序分析',
|
||||
'params': [
|
||||
{'name': 'tool', 'label': '工具', 'value': motifs.get('tool', '')},
|
||||
{'name': 'url', 'label': 'URL', 'value': motifs.get('url', '')},
|
||||
{'name': 'mode', 'label': '模式', 'value': motifs.get('parameters', {}).get('mode', '')},
|
||||
{'name': 'max_motifs', 'label': 'Max motifs', 'value': str(motifs.get('parameters', {}).get('max_motifs', ''))},
|
||||
{'name': 'width', 'label': 'Width', 'value': motifs.get('parameters', {}).get('width', '')},
|
||||
{'name': 'detected_motifs', 'label': '检测到的motifs', 'value': motifs.get('detected_motifs', '')},
|
||||
{'name': 'conserved_sequences', 'label': '保守序列', 'value': ', '.join(motifs.get('conserved_sequences', []))},
|
||||
]
|
||||
})
|
||||
|
||||
# 6. 染色体定位
|
||||
data['stages'].append({
|
||||
'id': 'karyotype',
|
||||
'name': '6. 染色体定位',
|
||||
'params': [
|
||||
{'name': 'mapping_method', 'label': '定位方法', 'value': chromosomal.get('mapping_method', '')},
|
||||
{'name': 'location', 'label': '定位结果', 'value': chromosomal.get('location', '')},
|
||||
]
|
||||
})
|
||||
|
||||
# 7. 共线性分析
|
||||
duplication = chromosomal.get('duplication_analysis', {})
|
||||
data['stages'].append({
|
||||
'id': 'synteny',
|
||||
'name': '7. 共线性分析',
|
||||
'params': [
|
||||
{'name': 'tool', 'label': '工具', 'value': duplication.get('tool', '')},
|
||||
{'name': 'full_name', 'label': '完整名称', 'value': duplication.get('full_name', '')},
|
||||
{'name': 'synteny_tool', 'label': '共线性绘图工具', 'value': duplication.get('synteny_tool', '')},
|
||||
{'name': 'synteny_url', 'label': '共线性工具URL', 'value': duplication.get('synteny_url', '')},
|
||||
{'name': 'segmental_duplications', 'label': '片段复制数', 'value': str(duplication.get('results', {}).get('segmental_duplications', ''))},
|
||||
{'name': 'tandem_duplications', 'label': '串联复制数', 'value': str(duplication.get('results', {}).get('tandem_duplications', ''))},
|
||||
]
|
||||
})
|
||||
|
||||
# 8. 表达分析
|
||||
primer_design = expression.get('primer_design', {})
|
||||
primer_params = primer_design.get('parameters', {})
|
||||
data['stages'].append({
|
||||
'id': 'expression',
|
||||
'name': '8. 表达分析',
|
||||
'params': [
|
||||
{'name': 'method', 'label': '方法', 'value': expression.get('method', '')},
|
||||
{'name': 'full_name', 'label': '方法全称', 'value': expression.get('full_name', '')},
|
||||
{'name': 'kit', 'label': '试剂盒', 'value': expression.get('kit', '')},
|
||||
{'name': 'reference_gene', 'label': '内参基因', 'value': expression.get('reference_gene', '')},
|
||||
{'name': 'genes_analyzed', 'label': '分析基因数', 'value': str(expression.get('genes_analyzed', ''))},
|
||||
{'name': 'primer_tool', 'label': '引物设计工具', 'value': primer_design.get('tool', '')},
|
||||
{'name': 'primer_url', 'label': '引物工具URL', 'value': primer_design.get('url', '')},
|
||||
{'name': 'product_length', 'label': '产物长度', 'value': primer_params.get('product_length', '')},
|
||||
{'name': 'gc_content', 'label': 'GC含量', 'value': primer_params.get('gc_content', '')},
|
||||
{'name': 'melting_temp', 'label': '退火温度', 'value': primer_params.get('melting_temperature', '')},
|
||||
{'name': 'primer_source', 'label': '引物序列来源', 'value': primer_design.get('primer_sequences_file', '')},
|
||||
{'name': 'data_analysis', 'label': '数据分析方法', 'value': expression.get('data_analysis', '')},
|
||||
{'name': 'tissues', 'label': '分析组织', 'value': ', '.join(expression.get('tissues_analyzed', []))},
|
||||
{'name': 'stages', 'label': '发育阶段', 'value': ', '.join(expression.get('developmental_stages', []))},
|
||||
]
|
||||
})
|
||||
|
||||
# 9. 统计分析
|
||||
data['stages'].append({
|
||||
'id': 'statistics',
|
||||
'name': '9. 统计分析',
|
||||
'params': [
|
||||
{'name': 'software', 'label': '软件', 'value': statistics.get('software', '')},
|
||||
{'name': 'company', 'label': '公司', 'value': statistics.get('company', '')},
|
||||
{'name': 'method', 'label': '方法', 'value': ', '.join(statistics.get('methods', []))},
|
||||
{'name': 'significance_levels', 'label': '显著性水平', 'value': ', '.join(statistics.get('significance_levels', []))},
|
||||
]
|
||||
})
|
||||
|
||||
return data
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import uvicorn
|
||||
uvicorn.run(app, host="0.0.0.0", port=5002)
|
||||
Executable
+998
@@ -0,0 +1,998 @@
|
||||
"""Methodology extraction batch result routes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, File, Form, HTTPException, UploadFile
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from src.rag.unified.bbav5_methodology_config import build_bbav5_methodology_config, build_methodology_execution_plan, ensure_methodology_execution_contract, resolve_batch_dir
|
||||
from src.rag.unified.figure_table_planner import build_figure_table_outputs
|
||||
from src.rag.unified.methodology_qa_sandbox import DEFAULT_SANDBOX_ROOT, answer_from_sandbox, build_methodology_qa_sandbox, sanitize_token
|
||||
from src.rag.project_config import PROJECT_ROOT, conda_env_path
|
||||
|
||||
|
||||
# Use LLM extraction results by default (can be overridden via env var)
|
||||
BATCH_DIR_NAME = os.environ.get("METHODOLOGY_BATCH_DIR", "methodology_batch_llm")
|
||||
BATCH_DIR = PROJECT_ROOT / "data" / "evaluation" / BATCH_DIR_NAME
|
||||
PAPERS_CSV = PROJECT_ROOT / "data" / "evaluation" / "methodology_papers.csv"
|
||||
GOLD_CSV = PROJECT_ROOT / "data" / "evaluation" / "methodology_gold.csv"
|
||||
REVIEW_RESULTS_JSON = BATCH_DIR / "review_results.json"
|
||||
REVIEW_RESULTS_CSV = BATCH_DIR / "review_results.csv"
|
||||
UPLOAD_JOBS_DIR = PROJECT_ROOT / "data" / "evaluation" / "methodology_upload_jobs"
|
||||
UPLOAD_OUTPUT_ROOT = PROJECT_ROOT / "data" / "evaluation" / "0505"
|
||||
DEFAULT_MINERU_CMD = str(conda_env_path("minerupy311") / "bin" / "mineru")
|
||||
DEFAULT_MINERU_API_URL = "http://127.0.0.1:50215"
|
||||
DEFAULT_MINERU_BACKEND = "hybrid-auto-engine"
|
||||
DEFAULT_MINERU_METHOD = "auto"
|
||||
DEFAULT_MINERU_LANG = "en"
|
||||
DEFAULT_MINERU_MIN_FREE_GPU_MB = 6144
|
||||
DEFAULT_MINERU_LOW_MEM_MIN_FREE_GPU_MB = 4096
|
||||
DEFAULT_MINERU_RUNTIME_MODE = "single"
|
||||
|
||||
router = APIRouter(prefix="/methodology", tags=["methodology"])
|
||||
|
||||
|
||||
class ReviewResult(BaseModel):
|
||||
observation_id: str = Field(..., min_length=1)
|
||||
paper_id: str = ""
|
||||
review_status: str = ""
|
||||
stage_review: str = ""
|
||||
tools_review: str = ""
|
||||
versions_review: str = ""
|
||||
parameters_review: str = ""
|
||||
inputs_outputs_review: str = ""
|
||||
evidence_review: str = ""
|
||||
bbav5_review: str = ""
|
||||
bbav5_stage_review: str = ""
|
||||
bbav5_tools_review: str = ""
|
||||
support_status_review: str = ""
|
||||
corrected_stage_value: str = ""
|
||||
corrected_tools_value: str = ""
|
||||
corrected_versions_value: str = ""
|
||||
corrected_parameters_value: str = ""
|
||||
corrected_evidence_value: str = ""
|
||||
corrected_inputs_value: str = ""
|
||||
corrected_outputs_value: str = ""
|
||||
corrected_bbav5_stage_value: str = ""
|
||||
corrected_bbav5_tools_value: str = ""
|
||||
corrected_support_status_value: str = ""
|
||||
manual_note: str = ""
|
||||
|
||||
|
||||
class ReviewResultsPayload(BaseModel):
|
||||
results: list[ReviewResult]
|
||||
|
||||
|
||||
class MethodologySandboxBuildPayload(BaseModel):
|
||||
methodology_dir: str = Field(..., min_length=1)
|
||||
sandbox_root: str = str(DEFAULT_SANDBOX_ROOT)
|
||||
run_id: str = ""
|
||||
include_figures: bool = True
|
||||
content_list_path: str = ""
|
||||
|
||||
|
||||
class MethodologySandboxAnswerPayload(BaseModel):
|
||||
question: str = Field(..., min_length=1)
|
||||
registry_path: str = ""
|
||||
sandbox_dir: str = ""
|
||||
|
||||
|
||||
REVIEW_RESULT_FIELDS = [
|
||||
"observation_id",
|
||||
"paper_id",
|
||||
"review_status",
|
||||
"stage_review",
|
||||
"tools_review",
|
||||
"versions_review",
|
||||
"parameters_review",
|
||||
"inputs_outputs_review",
|
||||
"evidence_review",
|
||||
"bbav5_review",
|
||||
"bbav5_stage_review",
|
||||
"bbav5_tools_review",
|
||||
"support_status_review",
|
||||
"corrected_stage_value",
|
||||
"corrected_tools_value",
|
||||
"corrected_versions_value",
|
||||
"corrected_parameters_value",
|
||||
"corrected_evidence_value",
|
||||
"corrected_inputs_value",
|
||||
"corrected_outputs_value",
|
||||
"corrected_bbav5_stage_value",
|
||||
"corrected_bbav5_tools_value",
|
||||
"corrected_support_status_value",
|
||||
"manual_note",
|
||||
]
|
||||
|
||||
|
||||
def read_csv(path: Path) -> list[dict[str, str]]:
|
||||
if not path.exists():
|
||||
return []
|
||||
with path.open("r", encoding="utf-8-sig", newline="") as handle:
|
||||
return list(csv.DictReader(handle))
|
||||
|
||||
|
||||
def read_json(path: Path) -> Any:
|
||||
if not path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Missing file: {path}")
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8", newline="") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=REVIEW_RESULT_FIELDS)
|
||||
writer.writeheader()
|
||||
for row in rows:
|
||||
writer.writerow({field: row.get(field, "") for field in REVIEW_RESULT_FIELDS})
|
||||
|
||||
|
||||
def read_review_results() -> list[dict[str, Any]]:
|
||||
if not REVIEW_RESULTS_JSON.exists():
|
||||
return []
|
||||
data = json.loads(REVIEW_RESULTS_JSON.read_text(encoding="utf-8"))
|
||||
if isinstance(data, dict):
|
||||
return list(data.get("results", []))
|
||||
return list(data)
|
||||
|
||||
|
||||
def write_job_status(job_dir: Path, payload: dict[str, Any]) -> None:
|
||||
payload = {"updated_at": datetime.now().isoformat(timespec="seconds"), **payload}
|
||||
job_dir.mkdir(parents=True, exist_ok=True)
|
||||
(job_dir / "status.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
|
||||
def parse_int_env(name: str, default: int) -> int:
|
||||
value = os.environ.get(name, "").strip()
|
||||
if not value:
|
||||
return default
|
||||
try:
|
||||
parsed = int(value)
|
||||
except ValueError:
|
||||
return default
|
||||
return max(0, parsed)
|
||||
|
||||
|
||||
def parse_optional_int_env(name: str) -> int | None:
|
||||
value = os.environ.get(name, "").strip()
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return max(0, int(value))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def env_bool(name: str) -> bool:
|
||||
return os.environ.get(name, "").strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def env_bool_text(name: str) -> str:
|
||||
value = os.environ.get(name, "").strip().lower()
|
||||
if value in {"1", "true", "yes", "on"}:
|
||||
return "true"
|
||||
if value in {"0", "false", "no", "off"}:
|
||||
return "false"
|
||||
return ""
|
||||
|
||||
|
||||
def normalize_mineru_runtime_mode(value: str) -> str:
|
||||
mode = str(value or "").strip().lower()
|
||||
return mode if mode in {"single", "batch"} else DEFAULT_MINERU_RUNTIME_MODE
|
||||
|
||||
|
||||
def mineru_default_min_free_gpu_mb(runtime: dict[str, str]) -> int:
|
||||
mode = normalize_mineru_runtime_mode(runtime.get("mode", ""))
|
||||
backend = str(runtime.get("backend") or "").strip().lower()
|
||||
method = str(runtime.get("method") or "").strip().lower()
|
||||
formula = str(runtime.get("formula") or "").strip().lower()
|
||||
table = str(runtime.get("table") or "").strip().lower()
|
||||
low_mem_pipeline = (
|
||||
mode == "single"
|
||||
and backend == "pipeline"
|
||||
and method == "txt"
|
||||
and formula == "false"
|
||||
and table == "false"
|
||||
)
|
||||
if low_mem_pipeline:
|
||||
return DEFAULT_MINERU_LOW_MEM_MIN_FREE_GPU_MB
|
||||
return DEFAULT_MINERU_MIN_FREE_GPU_MB
|
||||
|
||||
|
||||
def parse_gpu_memory_csv(text: str) -> list[dict[str, int]]:
|
||||
rows: list[dict[str, int]] = []
|
||||
for line in text.splitlines():
|
||||
parts = [part.strip() for part in line.split(",")]
|
||||
if len(parts) < 4:
|
||||
continue
|
||||
try:
|
||||
rows.append(
|
||||
{
|
||||
"index": int(parts[0]),
|
||||
"total_mb": int(parts[1]),
|
||||
"used_mb": int(parts[2]),
|
||||
"free_mb": int(parts[3]),
|
||||
}
|
||||
)
|
||||
except ValueError:
|
||||
continue
|
||||
return rows
|
||||
|
||||
|
||||
def parse_gpu_process_csv(text: str) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for line in text.splitlines():
|
||||
parts = [part.strip() for part in line.split(",", 2)]
|
||||
if len(parts) < 3:
|
||||
continue
|
||||
try:
|
||||
rows.append({"pid": int(parts[0]), "name": parts[1], "used_mb": int(parts[2])})
|
||||
except ValueError:
|
||||
continue
|
||||
return rows
|
||||
|
||||
|
||||
def cuda_visible_gpu_indices(cuda_visible_devices: str) -> list[int]:
|
||||
indices: list[int] = []
|
||||
for token in cuda_visible_devices.split(","):
|
||||
token = token.strip()
|
||||
if token.isdigit():
|
||||
indices.append(int(token))
|
||||
return indices
|
||||
|
||||
|
||||
def format_gpu_processes(processes: list[dict[str, Any]], limit: int = 8) -> str:
|
||||
if not processes:
|
||||
return "未检测到 GPU 计算进程"
|
||||
ordered = sorted(processes, key=lambda row: int(row.get("used_mb") or 0), reverse=True)
|
||||
formatted = []
|
||||
for row in ordered[:limit]:
|
||||
name = Path(str(row.get("name") or "")).name or str(row.get("name") or "")
|
||||
formatted.append(f"PID {row.get('pid')} {name} {row.get('used_mb')}MiB")
|
||||
if len(ordered) > limit:
|
||||
formatted.append(f"... 另有 {len(ordered) - limit} 个进程")
|
||||
return "; ".join(formatted)
|
||||
|
||||
|
||||
def build_mineru_gpu_preflight_error(
|
||||
gpu_rows: list[dict[str, int]],
|
||||
process_rows: list[dict[str, Any]],
|
||||
cuda_visible_devices: str,
|
||||
min_free_mb: int,
|
||||
runtime_mode: str = DEFAULT_MINERU_RUNTIME_MODE,
|
||||
) -> str:
|
||||
if min_free_mb <= 0:
|
||||
return ""
|
||||
visible_indices = cuda_visible_gpu_indices(cuda_visible_devices)
|
||||
candidate_rows = [row for row in gpu_rows if row["index"] in visible_indices] if visible_indices else gpu_rows
|
||||
if not candidate_rows:
|
||||
return (
|
||||
"MinerU GPU 预检失败:无法从 nvidia-smi 读取可用 GPU。"
|
||||
f"CUDA_VISIBLE_DEVICES={cuda_visible_devices or '<empty>'}"
|
||||
)
|
||||
best = max(candidate_rows, key=lambda row: row["free_mb"])
|
||||
if best["free_mb"] >= min_free_mb:
|
||||
return ""
|
||||
processes = format_gpu_processes(process_rows)
|
||||
mode = normalize_mineru_runtime_mode(runtime_mode)
|
||||
if mode == "single":
|
||||
return (
|
||||
"MinerU 按需启动需要 GPU 显存,但当前 GPU 显存不足,不能启动新的 PDF 解析任务。"
|
||||
f"GPU{best['index']} free={best['free_mb']}MiB / total={best['total_mb']}MiB,"
|
||||
f"需要至少 {min_free_mb}MiB 空闲显存。"
|
||||
f"当前 GPU 进程:{processes}。"
|
||||
"请先释放占用显存的 QA/VLLM 进程,或改用低显存 MinerU 配置后重试。"
|
||||
)
|
||||
return (
|
||||
"MinerU 服务已热加载,但当前 GPU 显存余量不足,不能安全提交新的 PDF 解析任务。"
|
||||
f"GPU{best['index']} free={best['free_mb']}MiB / total={best['total_mb']}MiB,"
|
||||
f"需要至少 {min_free_mb}MiB 空闲显存。"
|
||||
f"当前 GPU 进程:{processes}。"
|
||||
"请先释放占用显存的 QA/VLLM/MinerU 进程,或调低 METHODOLOGY_MINERU_MIN_FREE_GPU_MB 后重试。"
|
||||
)
|
||||
|
||||
|
||||
def mineru_gpu_preflight_error(
|
||||
cuda_visible_devices: str,
|
||||
runtime_mode: str = DEFAULT_MINERU_RUNTIME_MODE,
|
||||
runtime: dict[str, str] | None = None,
|
||||
) -> str:
|
||||
if os.environ.get("METHODOLOGY_MINERU_GPU_PREFLIGHT", "1").strip().lower() in {"0", "false", "no", "off"}:
|
||||
return ""
|
||||
runtime_payload = runtime or {"mode": runtime_mode}
|
||||
min_free_mb = parse_optional_int_env("METHODOLOGY_MINERU_MIN_FREE_GPU_MB")
|
||||
if min_free_mb is None:
|
||||
min_free_mb = mineru_default_min_free_gpu_mb(runtime_payload)
|
||||
if min_free_mb <= 0:
|
||||
return ""
|
||||
try:
|
||||
memory = subprocess.run(
|
||||
["nvidia-smi", "--query-gpu=index,memory.total,memory.used,memory.free", "--format=csv,noheader,nounits"],
|
||||
cwd=PROJECT_ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=5,
|
||||
)
|
||||
except Exception as exc:
|
||||
return f"MinerU GPU 预检失败:无法执行 nvidia-smi:{exc}"
|
||||
if memory.returncode != 0:
|
||||
return f"MinerU GPU 预检失败:nvidia-smi 返回 {memory.returncode}:{memory.stderr.strip() or memory.stdout.strip()}"
|
||||
processes_text = ""
|
||||
try:
|
||||
processes = subprocess.run(
|
||||
["nvidia-smi", "--query-compute-apps=pid,process_name,used_memory", "--format=csv,noheader,nounits"],
|
||||
cwd=PROJECT_ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=5,
|
||||
)
|
||||
if processes.returncode == 0:
|
||||
processes_text = processes.stdout
|
||||
except Exception:
|
||||
processes_text = ""
|
||||
return build_mineru_gpu_preflight_error(
|
||||
parse_gpu_memory_csv(memory.stdout),
|
||||
parse_gpu_process_csv(processes_text),
|
||||
cuda_visible_devices,
|
||||
min_free_mb,
|
||||
runtime_payload.get("mode", runtime_mode),
|
||||
)
|
||||
|
||||
|
||||
def summarize_mineru_failure(log_path: Path, returncode: int) -> str:
|
||||
message = f"MinerU failed with exit code {returncode}; see {log_path}"
|
||||
if not log_path.exists():
|
||||
return message
|
||||
try:
|
||||
text = log_path.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
return message
|
||||
if re.search(r"CUDA out of memory|OutOfMemoryError|cuda memory", text, re.IGNORECASE):
|
||||
return (
|
||||
"MinerU 解析失败:CUDA 显存不足。"
|
||||
"这通常表示 GPU 已被 QA/VLLM/MinerU 进程占用,运行 PDF 检测/解析时没有足够额外显存。"
|
||||
f"日志:{log_path}"
|
||||
)
|
||||
return message
|
||||
|
||||
|
||||
def build_mineru_runtime() -> tuple[list[str], dict[str, str], dict[str, str]]:
|
||||
env = os.environ.copy()
|
||||
env.setdefault("MINERU_MODEL_SOURCE", "modelscope")
|
||||
env.setdefault("CUDA_VISIBLE_DEVICES", "0")
|
||||
env.setdefault("TORCH_CUDNN_V8_API_DISABLED", "1")
|
||||
|
||||
mineru_cmd = shlex.split(os.environ.get("MINERU_CMD", DEFAULT_MINERU_CMD))
|
||||
runtime_mode = normalize_mineru_runtime_mode(
|
||||
os.environ.get("METHODOLOGY_MINERU_RUNTIME_MODE", DEFAULT_MINERU_RUNTIME_MODE)
|
||||
)
|
||||
runtime = {
|
||||
"mode": runtime_mode,
|
||||
"model_source": env.get("MINERU_MODEL_SOURCE", ""),
|
||||
"cuda_visible_devices": env.get("CUDA_VISIBLE_DEVICES", ""),
|
||||
"api_url": os.environ.get("V15_MINERU_API_URL", DEFAULT_MINERU_API_URL).strip(),
|
||||
"use_api": "1" if runtime_mode == "batch" and env_bool("V15_MINERU_USE_API") else "0",
|
||||
"backend": os.environ.get("V15_MINERU_BACKEND", DEFAULT_MINERU_BACKEND).strip() or DEFAULT_MINERU_BACKEND,
|
||||
"method": os.environ.get("V15_MINERU_METHOD", DEFAULT_MINERU_METHOD).strip() or DEFAULT_MINERU_METHOD,
|
||||
"lang": os.environ.get("V15_MINERU_LANG", DEFAULT_MINERU_LANG).strip() or DEFAULT_MINERU_LANG,
|
||||
"formula": env_bool_text("V15_MINERU_FORMULA"),
|
||||
"table": env_bool_text("V15_MINERU_TABLE"),
|
||||
}
|
||||
return mineru_cmd, env, runtime
|
||||
|
||||
|
||||
def build_mineru_command(mineru_cmd: list[str], source_path: Path, mineru_out: Path, runtime: dict[str, str]) -> list[str]:
|
||||
cmd = [
|
||||
*mineru_cmd,
|
||||
"-p",
|
||||
str(source_path),
|
||||
"-o",
|
||||
str(mineru_out),
|
||||
"-m",
|
||||
runtime["method"],
|
||||
"-b",
|
||||
runtime["backend"],
|
||||
"-l",
|
||||
runtime["lang"],
|
||||
]
|
||||
if runtime.get("formula"):
|
||||
cmd.extend(["-f", runtime["formula"]])
|
||||
if runtime.get("table"):
|
||||
cmd.extend(["-t", runtime["table"]])
|
||||
if runtime.get("mode") == "batch" and runtime.get("use_api") == "1" and runtime.get("api_url"):
|
||||
cmd.extend(["--api-url", runtime["api_url"]])
|
||||
return cmd
|
||||
|
||||
|
||||
def read_job_status(job_id: str) -> dict[str, Any]:
|
||||
status_path = UPLOAD_JOBS_DIR / job_id / "status.json"
|
||||
if not status_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Unknown methodology extraction job: {job_id}")
|
||||
return json.loads(status_path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def list_upload_job_statuses(limit: int = 50) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
if not UPLOAD_JOBS_DIR.exists():
|
||||
return rows
|
||||
for status_path in UPLOAD_JOBS_DIR.glob("*/status.json"):
|
||||
try:
|
||||
payload = json.loads(status_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
continue
|
||||
output_dir = str(payload.get("output_dir") or "")
|
||||
source_path = str(payload.get("source_path") or "")
|
||||
row = {
|
||||
"job_id": payload.get("job_id") or status_path.parent.name,
|
||||
"status": payload.get("status", ""),
|
||||
"stage": payload.get("stage", ""),
|
||||
"substage": payload.get("substage", ""),
|
||||
"message": payload.get("message", ""),
|
||||
"updated_at": payload.get("updated_at", ""),
|
||||
"paper_id": payload.get("paper_id", ""),
|
||||
"family": payload.get("family", ""),
|
||||
"source_filename": Path(source_path).name if source_path else "",
|
||||
"source_path": source_path,
|
||||
"output_dir": output_dir,
|
||||
"batch_dir": output_dir,
|
||||
"stage_count": payload.get("stage_count", 0),
|
||||
"species_count": payload.get("species_count", 0),
|
||||
"figure_table_caption_count": payload.get("figure_table_caption_count", 0),
|
||||
"figure_table_stage_count": payload.get("figure_table_stage_count", 0),
|
||||
"config_url": payload.get("config_url", f"/api/v1/methodology/extract-jobs/{status_path.parent.name}/bbav5-config"),
|
||||
"methodology_plan_url": payload.get("methodology_plan_url", f"/api/v1/methodology/extract-jobs/{status_path.parent.name}/plan"),
|
||||
"methodology_sandbox_requested": payload.get("methodology_sandbox_requested", False),
|
||||
"methodology_sandbox_status": payload.get("methodology_sandbox_status", ""),
|
||||
"methodology_sandbox_dir": payload.get("methodology_sandbox_dir", ""),
|
||||
"methodology_sandbox_registry": payload.get("methodology_sandbox_registry", ""),
|
||||
"methodology_sandbox_collection": payload.get("methodology_sandbox_collection", ""),
|
||||
}
|
||||
rows.append(row)
|
||||
rows.sort(key=lambda row: str(row.get("updated_at") or ""), reverse=True)
|
||||
return rows[: max(1, min(int(limit or 50), 200))]
|
||||
|
||||
|
||||
def run_uploaded_methodology_job(
|
||||
job_id: str,
|
||||
source_path: Path,
|
||||
paper_id: str,
|
||||
family: str,
|
||||
output_dir: Path,
|
||||
llm_model: str,
|
||||
llm_request_timeout: int,
|
||||
sandbox: bool = False,
|
||||
sandbox_run_id: str = "",
|
||||
) -> None:
|
||||
job_dir = UPLOAD_JOBS_DIR / job_id
|
||||
log_path = job_dir / "job.log"
|
||||
try:
|
||||
write_job_status(
|
||||
job_dir,
|
||||
{
|
||||
"job_id": job_id,
|
||||
"status": "running",
|
||||
"stage": "mineru",
|
||||
"message": "MinerU 正在解析上传文献",
|
||||
"source_path": str(source_path),
|
||||
"output_dir": str(output_dir),
|
||||
},
|
||||
)
|
||||
content_list = source_path
|
||||
if source_path.suffix.lower() != ".json":
|
||||
mineru_out = job_dir / "mineru"
|
||||
mineru_cmd, mineru_env, mineru_runtime = build_mineru_runtime()
|
||||
preflight_error = mineru_gpu_preflight_error(
|
||||
mineru_runtime["cuda_visible_devices"],
|
||||
mineru_runtime["mode"],
|
||||
mineru_runtime,
|
||||
)
|
||||
if preflight_error:
|
||||
with log_path.open("a", encoding="utf-8") as handle:
|
||||
handle.write(preflight_error + "\n")
|
||||
raise RuntimeError(preflight_error)
|
||||
cmd = build_mineru_command(mineru_cmd, source_path, mineru_out, mineru_runtime)
|
||||
with log_path.open("a", encoding="utf-8") as handle:
|
||||
handle.write(
|
||||
"$ env "
|
||||
f"METHODOLOGY_MINERU_RUNTIME_MODE={mineru_runtime['mode']} "
|
||||
f"MINERU_MODEL_SOURCE={mineru_runtime['model_source']} "
|
||||
f"CUDA_VISIBLE_DEVICES={mineru_runtime['cuda_visible_devices']} "
|
||||
+ " ".join(cmd)
|
||||
+ "\n"
|
||||
)
|
||||
handle.write("$ " + " ".join(cmd) + "\n")
|
||||
handle.flush()
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
cwd=PROJECT_ROOT,
|
||||
env=mineru_env,
|
||||
text=True,
|
||||
stdout=handle,
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(summarize_mineru_failure(log_path, result.returncode))
|
||||
matches = sorted(mineru_out.rglob("*_content_list.json"))
|
||||
if not matches:
|
||||
raise RuntimeError(f"MinerU finished but no *_content_list.json was found under {mineru_out}")
|
||||
content_list = matches[0]
|
||||
|
||||
write_job_status(
|
||||
job_dir,
|
||||
{
|
||||
"job_id": job_id,
|
||||
"status": "running",
|
||||
"stage": "llm",
|
||||
"message": "LLM 正在提取方法学并与 bbav5 对齐",
|
||||
"source_path": str(source_path),
|
||||
"content_list": str(content_list),
|
||||
"output_dir": str(output_dir),
|
||||
},
|
||||
)
|
||||
papers_csv = job_dir / "methodology_papers.csv"
|
||||
with papers_csv.open("w", encoding="utf-8", newline="") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=["paper_id", "family", "source_paper_role", "parsed_document_path"])
|
||||
writer.writeheader()
|
||||
writer.writerow(
|
||||
{
|
||||
"paper_id": paper_id,
|
||||
"family": family,
|
||||
"source_paper_role": "primary_gene_family_paper",
|
||||
"parsed_document_path": str(content_list),
|
||||
}
|
||||
)
|
||||
batch_cmd = [
|
||||
str(conda_env_path(os.environ.get("GF_PYTHON_ENV", "gfpy312")) / "bin" / "python"),
|
||||
"scripts/run_methodology_batch.py",
|
||||
"--papers",
|
||||
str(papers_csv),
|
||||
"--paper-id",
|
||||
paper_id,
|
||||
"--output-dir",
|
||||
str(output_dir),
|
||||
"--use-llm",
|
||||
"--llm-model",
|
||||
llm_model,
|
||||
"--llm-request-timeout",
|
||||
str(llm_request_timeout),
|
||||
"--external-job-status-json",
|
||||
str(job_dir / "status.json"),
|
||||
]
|
||||
with log_path.open("a", encoding="utf-8") as handle:
|
||||
handle.write("$ " + " ".join(batch_cmd) + "\n")
|
||||
handle.flush()
|
||||
result = subprocess.run(batch_cmd, cwd=PROJECT_ROOT, text=True, stdout=handle, stderr=subprocess.STDOUT)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"LLM methodology extraction failed with exit code {result.returncode}; see {log_path}")
|
||||
config = build_bbav5_methodology_config(output_dir)
|
||||
figure_table_error = ""
|
||||
figure_table_payload: dict[str, Any] = {}
|
||||
try:
|
||||
figure_table_payload = build_figure_table_outputs(
|
||||
content_list_path=content_list,
|
||||
output_dir=output_dir,
|
||||
methodology_config=config,
|
||||
)
|
||||
config = build_bbav5_methodology_config(output_dir)
|
||||
except Exception as exc: # keep the core methodology extraction result usable.
|
||||
figure_table_error = str(exc)
|
||||
preview = config.get("reproduction_plan_preview", {}) if isinstance(config, dict) else {}
|
||||
sandbox_payload: dict[str, Any] = {}
|
||||
sandbox_warning = ""
|
||||
if sandbox:
|
||||
try:
|
||||
result = build_methodology_qa_sandbox(
|
||||
output_dir,
|
||||
sandbox_root=DEFAULT_SANDBOX_ROOT,
|
||||
run_id=sandbox_run_id or f"upload_{sanitize_token(job_id)}",
|
||||
include_figures=False,
|
||||
)
|
||||
report = result.get("report", {})
|
||||
registry = result.get("registry", {})
|
||||
sandbox_payload = {
|
||||
"methodology_sandbox_dir": report.get("sandbox_dir", ""),
|
||||
"methodology_sandbox_registry": report.get("registry_path", ""),
|
||||
"methodology_sandbox_collection": registry.get("qdrant_collection", ""),
|
||||
"methodology_sandbox_status": "built",
|
||||
"methodology_sandbox_card_count": report.get("stage_card_count", 0) + report.get("plan_card_count", 0),
|
||||
"methodology_sandbox_relation_count": report.get("relation_count", 0),
|
||||
}
|
||||
except Exception as exc:
|
||||
sandbox_warning = str(exc)
|
||||
sandbox_payload = {
|
||||
"methodology_sandbox_status": "failed",
|
||||
"methodology_sandbox_warning": sandbox_warning,
|
||||
}
|
||||
write_job_status(
|
||||
job_dir,
|
||||
{
|
||||
"job_id": job_id,
|
||||
"status": "completed",
|
||||
"stage": "completed",
|
||||
"substage": "done",
|
||||
"message": "MinerU+LLM 方法学提取完成",
|
||||
"source_path": str(source_path),
|
||||
"content_list": str(content_list),
|
||||
"output_dir": str(output_dir),
|
||||
"paper_id": paper_id,
|
||||
"family": family,
|
||||
"stage_count": len(config.get("stages", [])),
|
||||
"species_count": len(config.get("species", [])),
|
||||
"figure_table_caption_count": len(config.get("figure_table_inventory", [])),
|
||||
"figure_table_stage_count": len(preview.get("suggested_stages", [])) if isinstance(preview, dict) else 0,
|
||||
"figure_table_warning": figure_table_error,
|
||||
"figure_table_files": figure_table_payload.get("files", {}),
|
||||
"config_url": f"/api/v1/methodology/extract-jobs/{job_id}/bbav5-config",
|
||||
"methodology_plan_url": f"/api/v1/methodology/extract-jobs/{job_id}/plan",
|
||||
"figure_table_inventory_url": f"/api/v1/methodology/extract-jobs/{job_id}/figure-table-inventory",
|
||||
"reproduction_plan_preview_url": f"/api/v1/methodology/extract-jobs/{job_id}/reproduction-plan-preview",
|
||||
**sandbox_payload,
|
||||
},
|
||||
)
|
||||
except Exception as exc:
|
||||
write_job_status(
|
||||
job_dir,
|
||||
{
|
||||
"job_id": job_id,
|
||||
"status": "failed",
|
||||
"stage": "failed",
|
||||
"message": str(exc),
|
||||
"source_path": str(source_path),
|
||||
"output_dir": str(output_dir),
|
||||
"log_path": str(log_path),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
async def methodology_status():
|
||||
summary = read_csv(BATCH_DIR / "summary.csv")
|
||||
review = read_csv(BATCH_DIR / "review_template.csv")
|
||||
return {
|
||||
"papers_table": str(PAPERS_CSV),
|
||||
"gold_table": str(GOLD_CSV),
|
||||
"batch_dir": str(BATCH_DIR),
|
||||
"batch_exists": BATCH_DIR.exists(),
|
||||
"paper_count": len(read_csv(PAPERS_CSV)),
|
||||
"summary_count": len(summary),
|
||||
"review_row_count": len(review),
|
||||
"files": {
|
||||
"summary": str(BATCH_DIR / "summary.csv"),
|
||||
"processing_status": str(BATCH_DIR / "processing_status.csv"),
|
||||
"review_template": str(BATCH_DIR / "review_template.csv"),
|
||||
"review_results": str(REVIEW_RESULTS_JSON),
|
||||
"discovered_tools": str(BATCH_DIR / "discovered_tools.csv"),
|
||||
"methodology_tool_audit": str(BATCH_DIR / "methodology_tool_audit.csv"),
|
||||
"methodology_parameter_audit": str(BATCH_DIR / "methodology_parameter_audit.csv"),
|
||||
"methodology_database_url_audit": str(BATCH_DIR / "methodology_database_url_audit.csv"),
|
||||
"methodology_review_issues": str(BATCH_DIR / "methodology_review_issues.csv"),
|
||||
"methodology_review_report": str(BATCH_DIR / "methodology_review_report.json"),
|
||||
"progress": str(BATCH_DIR / "progress.jsonl"),
|
||||
"run_log": str(BATCH_DIR / "methodology_batch_run.log"),
|
||||
"species_summary": str(BATCH_DIR / "species_summary.csv"),
|
||||
"observations": str(BATCH_DIR / "observations.json"),
|
||||
"alignment": str(BATCH_DIR / "alignment.json"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@router.get("/summary")
|
||||
async def methodology_summary():
|
||||
return {"rows": read_csv(BATCH_DIR / "summary.csv")}
|
||||
|
||||
|
||||
@router.get("/review-template")
|
||||
async def methodology_review_template():
|
||||
return {"rows": read_csv(BATCH_DIR / "review_template.csv")}
|
||||
|
||||
|
||||
@router.get("/processing-status")
|
||||
async def methodology_processing_status():
|
||||
return {"rows": read_csv(BATCH_DIR / "processing_status.csv")}
|
||||
|
||||
|
||||
@router.get("/discovered-tools")
|
||||
async def methodology_discovered_tools():
|
||||
return {"rows": read_csv(BATCH_DIR / "discovered_tools.csv")}
|
||||
|
||||
|
||||
@router.get("/tool-audit")
|
||||
async def methodology_tool_audit():
|
||||
return {"rows": read_csv(BATCH_DIR / "methodology_tool_audit.csv")}
|
||||
|
||||
|
||||
@router.get("/parameter-audit")
|
||||
async def methodology_parameter_audit():
|
||||
return {"rows": read_csv(BATCH_DIR / "methodology_parameter_audit.csv")}
|
||||
|
||||
|
||||
@router.get("/url-audit")
|
||||
async def methodology_url_audit():
|
||||
return {"rows": read_csv(BATCH_DIR / "methodology_database_url_audit.csv")}
|
||||
|
||||
|
||||
@router.get("/auto-review-issues")
|
||||
async def methodology_auto_review_issues():
|
||||
return {"rows": read_csv(BATCH_DIR / "methodology_review_issues.csv")}
|
||||
|
||||
|
||||
@router.get("/auto-review-report")
|
||||
async def methodology_auto_review_report():
|
||||
return read_json(BATCH_DIR / "methodology_review_report.json")
|
||||
|
||||
|
||||
@router.get("/progress")
|
||||
async def methodology_progress():
|
||||
path = BATCH_DIR / "progress.jsonl"
|
||||
if not path.exists():
|
||||
return {"rows": []}
|
||||
rows = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
|
||||
return {"rows": rows}
|
||||
|
||||
|
||||
@router.get("/run-log")
|
||||
async def methodology_run_log():
|
||||
path = BATCH_DIR / "methodology_batch_run.log"
|
||||
if not path.exists():
|
||||
return {"text": ""}
|
||||
return {"text": path.read_text(encoding="utf-8")}
|
||||
|
||||
|
||||
@router.get("/species-summary")
|
||||
async def methodology_species_summary():
|
||||
return {"rows": read_csv(BATCH_DIR / "species_summary.csv")}
|
||||
|
||||
|
||||
@router.get("/review-items")
|
||||
async def methodology_review_items():
|
||||
rows = read_csv(BATCH_DIR / "review_template.csv")
|
||||
saved = {row.get("observation_id"): row for row in read_review_results()}
|
||||
for row in rows:
|
||||
if row.get("observation_id") in saved:
|
||||
row["saved_review"] = saved[row["observation_id"]]
|
||||
return {"rows": rows, "saved_count": len(saved)}
|
||||
|
||||
|
||||
@router.get("/review-results")
|
||||
async def methodology_review_results():
|
||||
return {"results": read_review_results()}
|
||||
|
||||
|
||||
@router.post("/review-results")
|
||||
async def save_methodology_review_results(payload: ReviewResultsPayload):
|
||||
existing = {row.get("observation_id"): row for row in read_review_results()}
|
||||
for item in payload.results:
|
||||
existing[item.observation_id] = item.dict()
|
||||
rows = list(existing.values())
|
||||
REVIEW_RESULTS_JSON.parent.mkdir(parents=True, exist_ok=True)
|
||||
REVIEW_RESULTS_JSON.write_text(json.dumps({"results": rows}, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
write_csv(REVIEW_RESULTS_CSV, rows)
|
||||
return {"saved_count": len(rows), "json_path": str(REVIEW_RESULTS_JSON), "csv_path": str(REVIEW_RESULTS_CSV)}
|
||||
|
||||
|
||||
@router.get("/review-summary")
|
||||
async def methodology_review_summary():
|
||||
rows = read_review_results()
|
||||
summary: dict[str, dict[str, int]] = {}
|
||||
for field in [
|
||||
"review_status",
|
||||
"stage_review",
|
||||
"tools_review",
|
||||
"versions_review",
|
||||
"parameters_review",
|
||||
"evidence_review",
|
||||
"bbav5_stage_review",
|
||||
"bbav5_tools_review",
|
||||
"support_status_review",
|
||||
"bbav5_review",
|
||||
]:
|
||||
counts: dict[str, int] = {}
|
||||
for row in rows:
|
||||
value = str(row.get(field) or "<blank>")
|
||||
counts[value] = counts.get(value, 0) + 1
|
||||
summary[field] = counts
|
||||
return {"reviewed_count": len(rows), "summary": summary}
|
||||
|
||||
|
||||
@router.get("/alignment")
|
||||
async def methodology_alignment():
|
||||
return read_json(BATCH_DIR / "alignment.json")
|
||||
|
||||
|
||||
@router.get("/bbav5-config")
|
||||
async def methodology_bbav5_config(batch_dir: str = BATCH_DIR_NAME):
|
||||
try:
|
||||
return build_bbav5_methodology_config(resolve_batch_dir(batch_dir))
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@router.post("/extract-upload")
|
||||
async def methodology_extract_upload(
|
||||
background_tasks: BackgroundTasks,
|
||||
file: UploadFile = File(...),
|
||||
paper_id: str = Form("uploaded_paper"),
|
||||
family: str = Form(""),
|
||||
output_name: str = Form(""),
|
||||
llm_model: str = Form("deepseek-v4-flash"),
|
||||
llm_request_timeout: int = Form(75),
|
||||
sandbox: bool = Form(False),
|
||||
sandbox_run_id: str = Form(""),
|
||||
):
|
||||
job_id = uuid.uuid4().hex[:12]
|
||||
clean_paper_id = "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in (paper_id or f"paper_{job_id}"))
|
||||
output_base = output_name or f"methodology_batch_{clean_paper_id}"
|
||||
clean_output_base = "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in output_base).strip("._-") or "methodology_batch"
|
||||
clean_output = clean_output_base if clean_output_base.endswith(f"_{job_id}") else f"{clean_output_base}_{job_id}"
|
||||
job_dir = UPLOAD_JOBS_DIR / job_id
|
||||
upload_dir = job_dir / "upload"
|
||||
upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
source_path = upload_dir / (file.filename or f"{clean_paper_id}.pdf")
|
||||
source_path.write_bytes(await file.read())
|
||||
output_dir = UPLOAD_OUTPUT_ROOT / clean_output
|
||||
write_job_status(
|
||||
job_dir,
|
||||
{
|
||||
"job_id": job_id,
|
||||
"status": "queued",
|
||||
"stage": "queued",
|
||||
"message": "文献已上传,等待 MinerU+LLM 方法学提取",
|
||||
"source_path": str(source_path),
|
||||
"output_dir": str(output_dir),
|
||||
"paper_id": clean_paper_id,
|
||||
"family": family,
|
||||
"methodology_sandbox_requested": sandbox,
|
||||
},
|
||||
)
|
||||
background_tasks.add_task(
|
||||
run_uploaded_methodology_job,
|
||||
job_id,
|
||||
source_path,
|
||||
clean_paper_id,
|
||||
family,
|
||||
output_dir,
|
||||
llm_model,
|
||||
llm_request_timeout,
|
||||
sandbox,
|
||||
sandbox_run_id,
|
||||
)
|
||||
return {"success": True, "job_id": job_id, "status_url": f"/api/v1/methodology/extract-jobs/{job_id}"}
|
||||
|
||||
|
||||
@router.get("/extract-jobs")
|
||||
async def methodology_extract_jobs(limit: int = 50, status: str = ""):
|
||||
rows = list_upload_job_statuses(limit)
|
||||
if status:
|
||||
rows = [row for row in rows if row.get("status") == status]
|
||||
return {"jobs": rows, "count": len(rows)}
|
||||
|
||||
|
||||
@router.get("/extract-jobs/{job_id}")
|
||||
async def methodology_extract_job_status(job_id: str):
|
||||
return read_job_status(job_id)
|
||||
|
||||
|
||||
@router.get("/extract-jobs/{job_id}/bbav5-config")
|
||||
async def methodology_extract_job_config(job_id: str):
|
||||
status = read_job_status(job_id)
|
||||
if status.get("status") != "completed":
|
||||
raise HTTPException(status_code=409, detail=f"Extraction job is not completed: {status.get('status')}")
|
||||
try:
|
||||
return build_bbav5_methodology_config(status["output_dir"])
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@router.get("/extract-jobs/{job_id}/plan")
|
||||
async def methodology_extract_job_plan(job_id: str):
|
||||
status = read_job_status(job_id)
|
||||
if status.get("status") != "completed":
|
||||
raise HTTPException(status_code=409, detail=f"Extraction job is not completed: {status.get('status')}")
|
||||
output_dir = Path(status.get("output_dir", ""))
|
||||
plan_path = output_dir / "methodology_plan.json"
|
||||
if plan_path.exists():
|
||||
plan = ensure_methodology_execution_contract(read_json(plan_path))
|
||||
plan_path.write_text(json.dumps(plan, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return plan
|
||||
try:
|
||||
plan = build_methodology_execution_plan(output_dir)
|
||||
plan_path.write_text(json.dumps(plan, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return plan
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
@router.get("/extract-jobs/{job_id}/figure-table-inventory")
|
||||
async def methodology_extract_job_figure_table_inventory(job_id: str):
|
||||
status = read_job_status(job_id)
|
||||
output_dir = Path(status.get("output_dir", ""))
|
||||
return {"rows": read_json(output_dir / "figure_table_inventory.json")}
|
||||
|
||||
|
||||
@router.post("/sandbox/build")
|
||||
async def methodology_sandbox_build(payload: MethodologySandboxBuildPayload):
|
||||
try:
|
||||
result = build_methodology_qa_sandbox(
|
||||
payload.methodology_dir,
|
||||
sandbox_root=payload.sandbox_root,
|
||||
run_id=payload.run_id or None,
|
||||
include_figures=payload.include_figures,
|
||||
content_list_path=payload.content_list_path or None,
|
||||
)
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=500, detail=str(exc)) from exc
|
||||
return {
|
||||
"success": True,
|
||||
"report": result["report"],
|
||||
"registry": result["registry"],
|
||||
}
|
||||
|
||||
|
||||
@router.post("/sandbox/answer")
|
||||
async def methodology_sandbox_answer(payload: MethodologySandboxAnswerPayload):
|
||||
registry_path = payload.registry_path.strip()
|
||||
if not registry_path and payload.sandbox_dir.strip():
|
||||
registry_path = str(Path(payload.sandbox_dir.strip()) / "methodology_plan_registry.sandbox.json")
|
||||
if not registry_path:
|
||||
raise HTTPException(status_code=400, detail="registry_path or sandbox_dir is required")
|
||||
result = answer_from_sandbox(payload.question, registry_path)
|
||||
if not result:
|
||||
return {
|
||||
"success": False,
|
||||
"answer": "",
|
||||
"sources": [],
|
||||
"message": "No sandbox methodology plan matched the question.",
|
||||
"match_status": "no_match",
|
||||
}
|
||||
return {
|
||||
"success": result.get("success", True),
|
||||
"answer": result["answer"],
|
||||
"sources": result["sources"],
|
||||
"notes": result.get("notes", []),
|
||||
"plan_entry": result.get("plan_entry", {}),
|
||||
"matched_plan": result.get("matched_plan", {}),
|
||||
"match_status": result.get("match_status", ""),
|
||||
"query_constraints": result.get("query_constraints", {}),
|
||||
"candidate_plans": result.get("candidate_plans", []),
|
||||
"mismatch_reasons": result.get("mismatch_reasons", []),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/sandbox/example-batches")
|
||||
async def methodology_sandbox_example_batches():
|
||||
candidates = []
|
||||
for path in PROJECT_ROOT.glob("methodology_batch_*"):
|
||||
if path.is_dir() and (path / "review_template.csv").exists():
|
||||
candidates.append(
|
||||
{
|
||||
"name": path.name,
|
||||
"path": str(path),
|
||||
"has_plan": (path / "methodology_plan.json").exists(),
|
||||
"has_preview": (path / "reproduction_plan_preview.json").exists(),
|
||||
}
|
||||
)
|
||||
candidates.sort(key=lambda item: item["name"])
|
||||
return {
|
||||
"success": True,
|
||||
"sandbox_root": str(DEFAULT_SANDBOX_ROOT),
|
||||
"batches": candidates,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/extract-jobs/{job_id}/reproduction-plan-preview")
|
||||
async def methodology_extract_job_reproduction_plan_preview(job_id: str):
|
||||
status = read_job_status(job_id)
|
||||
output_dir = Path(status.get("output_dir", ""))
|
||||
return read_json(output_dir / "reproduction_plan_preview.json")
|
||||
Executable
+1683
File diff suppressed because it is too large
Load Diff
Executable
+1099
File diff suppressed because it is too large
Load Diff
Executable
+174
@@ -0,0 +1,174 @@
|
||||
"""
|
||||
RAG v10专用路由 - 使用苦荞SPL附表增强版索引
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import pickle
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from typing import List, Optional
|
||||
import requests
|
||||
from src.rag.project_config import PROJECT_ROOT
|
||||
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
router = APIRouter(prefix="/v10")
|
||||
|
||||
# ========== 配置 ==========
|
||||
OLLAMA_URL = "http://localhost:11434"
|
||||
EMBEDDING_MODEL = "bge-m3:567m"
|
||||
LLM_MODEL = "qwen3:0.6b"
|
||||
|
||||
# v10索引路径
|
||||
V10_INDEX_PATH = PROJECT_ROOT / "paper" / "v10" / "rag_v10_faiss_final.index"
|
||||
V10_CHUNKS_PATH = PROJECT_ROOT / "paper" / "v10" / "rag_v10_chunks_with_summary.json"
|
||||
V10_ENHANCED_PATH = PROJECT_ROOT / "paper" / "v10" / "rag_v10_chunks_enhanced.json"
|
||||
|
||||
# 全局变量
|
||||
_v10_index = None
|
||||
_v10_chunks = None
|
||||
|
||||
def get_embedding(text):
|
||||
"""获取文本向量"""
|
||||
response = requests.post(
|
||||
f"{OLLAMA_URL}/api/embeddings",
|
||||
json={"model": EMBEDDING_MODEL, "prompt": text}
|
||||
)
|
||||
vec = np.array(response.json()['embedding'], dtype=np.float32)
|
||||
return vec / np.linalg.norm(vec)
|
||||
|
||||
def load_v10_index():
|
||||
"""加载v10索引"""
|
||||
global _v10_index, _v10_chunks
|
||||
|
||||
if _v10_index is not None:
|
||||
return _v10_index, _v10_chunks
|
||||
|
||||
print(f"[V10 RAG] 加载v10索引...")
|
||||
|
||||
# 加载FAISS索引
|
||||
import faiss
|
||||
_v10_index = faiss.read_index(str(V10_INDEX_PATH))
|
||||
print(f"[V10 RAG] 索引加载完成: {_v10_index.ntotal} 个向量")
|
||||
|
||||
# 加载文本块
|
||||
with open(V10_CHUNKS_PATH, 'r') as f:
|
||||
chunks1 = json.load(f)
|
||||
with open(V10_ENHANCED_PATH, 'r') as f:
|
||||
chunks2 = json.load(f)
|
||||
|
||||
_v10_chunks = chunks1 + chunks2
|
||||
print(f"[V10 RAG] 文本块加载完成: {len(_v10_chunks)} 个")
|
||||
|
||||
return _v10_index, _v10_chunks
|
||||
|
||||
# ========== 请求/响应模型 ==========
|
||||
class V10RAGQueryRequest(BaseModel):
|
||||
query: str
|
||||
top_k: int = 3
|
||||
use_llm: bool = True
|
||||
|
||||
class V10RAGSource(BaseModel):
|
||||
chunk_id: str
|
||||
chunk_type: str
|
||||
content: str
|
||||
score: float
|
||||
|
||||
class V10RAGResponse(BaseModel):
|
||||
query: str
|
||||
answer: str
|
||||
sources: List[V10RAGSource]
|
||||
total_chunks: int
|
||||
|
||||
# ========== API端点 ==========
|
||||
@router.post("/chat", response_model=V10RAGResponse)
|
||||
async def v10_rag_chat(request: V10RAGQueryRequest):
|
||||
"""V10专用RAG查询"""
|
||||
try:
|
||||
# 加载索引
|
||||
index, chunks = load_v10_index()
|
||||
|
||||
# 1. 查询向量化
|
||||
query_vec = get_embedding(request.query).reshape(1, -1)
|
||||
|
||||
# 2. FAISS检索
|
||||
distances, indices = index.search(query_vec, request.top_k)
|
||||
|
||||
# 3. 构建sources
|
||||
sources = []
|
||||
context_parts = []
|
||||
for i, (dist, idx) in enumerate(zip(distances[0], indices[0])):
|
||||
chunk = chunks[idx]
|
||||
sources.append(V10RAGSource(
|
||||
chunk_id=chunk['id'],
|
||||
chunk_type=chunk['metadata']['type'],
|
||||
content=chunk['text'],
|
||||
score=float(dist)
|
||||
))
|
||||
context_parts.append(f"[{i+1}] {chunk['text']}")
|
||||
|
||||
# 4. LLM生成(可选)
|
||||
if request.use_llm:
|
||||
context = "\n\n".join(context_parts)
|
||||
prompt = f"""你是基因家族分析助手,基于以下资料回答问题。
|
||||
请准确回答,并标注引用来源(如[1]、[2]等)。
|
||||
|
||||
参考资料:
|
||||
{context}
|
||||
|
||||
用户问题:{request.query}
|
||||
|
||||
请基于以上资料回答:"""
|
||||
|
||||
response = requests.post(
|
||||
f"{OLLAMA_URL}/api/generate",
|
||||
json={
|
||||
"model": LLM_MODEL,
|
||||
"prompt": prompt,
|
||||
"stream": False,
|
||||
"options": {"temperature": 0.3}
|
||||
}
|
||||
)
|
||||
answer = response.json()['response']
|
||||
else:
|
||||
# 直接返回检索结果
|
||||
answer = "\n\n".join([f"[{i+1}] {s.content}" for i, s in enumerate(sources)])
|
||||
|
||||
return V10RAGResponse(
|
||||
query=request.query,
|
||||
answer=answer,
|
||||
sources=sources,
|
||||
total_chunks=len(chunks)
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"V10 RAG查询失败: {str(e)}")
|
||||
|
||||
@router.get("/info")
|
||||
async def v10_info():
|
||||
"""获取v10索引信息"""
|
||||
try:
|
||||
index, chunks = load_v10_index()
|
||||
|
||||
# 统计
|
||||
type_counts = {}
|
||||
for chunk in chunks:
|
||||
t = chunk['metadata']['type']
|
||||
type_counts[t] = type_counts.get(t, 0) + 1
|
||||
|
||||
return {
|
||||
"index_type": "FAISS v10专用",
|
||||
"total_chunks": len(chunks),
|
||||
"embedding_model": EMBEDDING_MODEL,
|
||||
"llm_model": LLM_MODEL,
|
||||
"chunk_types": type_counts,
|
||||
"data_source": "苦荞SPL附表增强版"
|
||||
}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
print("[V10 RAG] 路由加载完成")
|
||||
Executable
+372
@@ -0,0 +1,372 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
RAPTOR层次索引API路由
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from typing import List, Dict, Optional
|
||||
from pathlib import Path
|
||||
from src.rag.project_config import PROJECT_ROOT
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# 全局RAPTOR树实例缓存
|
||||
_raptor_trees: Dict[str, any] = {}
|
||||
|
||||
|
||||
class BuildTreeRequest(BaseModel):
|
||||
doc_id: str
|
||||
chunks: List[Dict]
|
||||
max_levels: int = 3
|
||||
summarization_ratio: float = 0.3
|
||||
|
||||
|
||||
class BuildTreeResponse(BaseModel):
|
||||
status: str
|
||||
root_id: str
|
||||
total_nodes: int
|
||||
level_stats: Dict
|
||||
message: str
|
||||
|
||||
|
||||
class RaptorSearchRequest(BaseModel):
|
||||
query: str
|
||||
doc_id: Optional[str] = None # None表示搜索所有树
|
||||
top_k: int = 5
|
||||
strategy: str = "top_down" # top_down, cross_level, leaf_only
|
||||
min_level: Optional[int] = None
|
||||
max_level: Optional[int] = None
|
||||
|
||||
|
||||
class RaptorSearchResponse(BaseModel):
|
||||
query: str
|
||||
results: List[Dict]
|
||||
total_results: int
|
||||
strategy: str
|
||||
levels_covered: List[str]
|
||||
|
||||
|
||||
class TreeStatsResponse(BaseModel):
|
||||
doc_id: str
|
||||
total_nodes: int
|
||||
root_id: str
|
||||
level_stats: Dict
|
||||
tree_depth: int
|
||||
|
||||
|
||||
class MultiLevelSearchRequest(BaseModel):
|
||||
query: str
|
||||
doc_id: Optional[str] = None
|
||||
levels: List[int] = [0, 1, 2]
|
||||
|
||||
|
||||
class MultiLevelSearchResponse(BaseModel):
|
||||
query: str
|
||||
results_by_level: Dict[str, List[Dict]]
|
||||
total_results: int
|
||||
|
||||
|
||||
@router.post("/raptor/build", response_model=BuildTreeResponse)
|
||||
async def build_raptor_tree(request: BuildTreeRequest):
|
||||
"""构建RAPTOR树"""
|
||||
try:
|
||||
from src.rag.advanced.raptor import RaptorTreeBuilder
|
||||
|
||||
builder = RaptorTreeBuilder(
|
||||
max_levels=request.max_levels,
|
||||
summarization_ratio=request.summarization_ratio
|
||||
)
|
||||
|
||||
root_id = builder.build_tree(
|
||||
chunks=request.chunks,
|
||||
doc_id=request.doc_id
|
||||
)
|
||||
|
||||
# 缓存树
|
||||
_raptor_trees[request.doc_id] = builder
|
||||
|
||||
# 获取统计
|
||||
stats = builder.get_tree_stats()
|
||||
|
||||
return BuildTreeResponse(
|
||||
status="success",
|
||||
root_id=root_id,
|
||||
total_nodes=stats['total_nodes'],
|
||||
level_stats=stats['levels'],
|
||||
message=f"RAPTOR树构建成功: {request.doc_id}"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"构建树失败: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/raptor/search", response_model=RaptorSearchResponse)
|
||||
async def search_raptor(request: RaptorSearchRequest):
|
||||
"""RAPTOR层次检索"""
|
||||
try:
|
||||
from src.rag.advanced.raptor import RaptorRetriever
|
||||
|
||||
if not _raptor_trees:
|
||||
raise HTTPException(status_code=404, detail="没有可用的RAPTOR树,请先构建")
|
||||
|
||||
all_results = []
|
||||
levels_covered = set()
|
||||
|
||||
# 搜索指定的树或所有树
|
||||
trees_to_search = [request.doc_id] if request.doc_id else list(_raptor_trees.keys())
|
||||
|
||||
for doc_id in trees_to_search:
|
||||
if doc_id not in _raptor_trees:
|
||||
continue
|
||||
|
||||
builder = _raptor_trees[doc_id]
|
||||
retriever = RaptorRetriever(builder)
|
||||
|
||||
results = retriever.search(
|
||||
query=request.query,
|
||||
top_k=request.top_k,
|
||||
strategy=request.strategy,
|
||||
min_level=request.min_level,
|
||||
max_level=request.max_level
|
||||
)
|
||||
|
||||
for result in results:
|
||||
all_results.append({
|
||||
'node_id': result.node.node_id,
|
||||
'level': result.level,
|
||||
'score': result.score,
|
||||
'content': result.node.content[:500],
|
||||
'summary': result.node.summary,
|
||||
'is_summary': result.is_summary,
|
||||
'path': result.path.path,
|
||||
'doc_id': doc_id
|
||||
})
|
||||
levels_covered.add(result.level)
|
||||
|
||||
# 排序
|
||||
all_results.sort(key=lambda x: x['score'], reverse=True)
|
||||
|
||||
return RaptorSearchResponse(
|
||||
query=request.query,
|
||||
results=all_results[:request.top_k],
|
||||
total_results=len(all_results),
|
||||
strategy=request.strategy,
|
||||
levels_covered=list(levels_covered)
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"检索失败: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/raptor/search/multi-level", response_model=MultiLevelSearchResponse)
|
||||
async def search_multi_level(request: MultiLevelSearchRequest):
|
||||
"""多层级检索"""
|
||||
try:
|
||||
from src.rag.advanced.raptor import RaptorRetriever
|
||||
|
||||
if not _raptor_trees:
|
||||
raise HTTPException(status_code=404, detail="没有可用的RAPTOR树")
|
||||
|
||||
results_by_level = {}
|
||||
total = 0
|
||||
|
||||
trees_to_search = [request.doc_id] if request.doc_id else list(_raptor_trees.keys())
|
||||
|
||||
for doc_id in trees_to_search:
|
||||
if doc_id not in _raptor_trees:
|
||||
continue
|
||||
|
||||
builder = _raptor_trees[doc_id]
|
||||
retriever = RaptorRetriever(builder)
|
||||
|
||||
multi_level_results = retriever.get_multi_level_results(
|
||||
query=request.query,
|
||||
levels=request.levels
|
||||
)
|
||||
|
||||
for level_num, results in multi_level_results.items():
|
||||
level_key = f"level_{level_num}"
|
||||
if level_key not in results_by_level:
|
||||
results_by_level[level_key] = []
|
||||
|
||||
for result in results:
|
||||
results_by_level[level_key].append({
|
||||
'node_id': result.node.node_id,
|
||||
'level': result.level,
|
||||
'score': result.score,
|
||||
'content': result.node.content[:300],
|
||||
'summary': result.node.summary,
|
||||
'doc_id': doc_id
|
||||
})
|
||||
total += 1
|
||||
|
||||
return MultiLevelSearchResponse(
|
||||
query=request.query,
|
||||
results_by_level=results_by_level,
|
||||
total_results=total
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"多层级检索失败: {str(e)}")
|
||||
|
||||
|
||||
@router.get("/raptor/tree/{doc_id}/stats", response_model=TreeStatsResponse)
|
||||
async def get_tree_stats(doc_id: str):
|
||||
"""获取树统计信息"""
|
||||
try:
|
||||
if doc_id not in _raptor_trees:
|
||||
raise HTTPException(status_code=404, detail=f"树不存在: {doc_id}")
|
||||
|
||||
builder = _raptor_trees[doc_id]
|
||||
stats = builder.get_tree_stats()
|
||||
|
||||
# 计算树深度
|
||||
max_height = 0
|
||||
for node in builder.nodes.values():
|
||||
max_height = max(max_height, node.get_height())
|
||||
|
||||
return TreeStatsResponse(
|
||||
doc_id=doc_id,
|
||||
total_nodes=stats['total_nodes'],
|
||||
root_id=stats['root_id'],
|
||||
level_stats=stats['levels'],
|
||||
tree_depth=max_height
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"获取统计失败: {str(e)}")
|
||||
|
||||
|
||||
@router.get("/raptor/tree/{doc_id}/structure")
|
||||
async def get_tree_structure(doc_id: str):
|
||||
"""获取树结构"""
|
||||
try:
|
||||
if doc_id not in _raptor_trees:
|
||||
raise HTTPException(status_code=404, detail=f"树不存在: {doc_id}")
|
||||
|
||||
builder = _raptor_trees[doc_id]
|
||||
|
||||
# 构建树结构
|
||||
def build_node_tree(node_id: str) -> Dict:
|
||||
node = builder.get_node(node_id)
|
||||
if not node:
|
||||
return {}
|
||||
|
||||
children = []
|
||||
for child_id in node.children_ids:
|
||||
child_tree = build_node_tree(child_id)
|
||||
if child_tree:
|
||||
children.append(child_tree)
|
||||
|
||||
return {
|
||||
'id': node.node_id,
|
||||
'level': node.level.value,
|
||||
'summary': node.summary[:200] if node.summary else node.content[:200],
|
||||
'children': children
|
||||
}
|
||||
|
||||
structure = build_node_tree(builder.root_id)
|
||||
|
||||
return {
|
||||
'doc_id': doc_id,
|
||||
'root_id': builder.root_id,
|
||||
'structure': structure
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"获取结构失败: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/raptor/tree/{doc_id}/save")
|
||||
async def save_tree(doc_id: str, filepath: Optional[str] = None):
|
||||
"""保存树到文件"""
|
||||
try:
|
||||
if doc_id not in _raptor_trees:
|
||||
raise HTTPException(status_code=404, detail=f"树不存在: {doc_id}")
|
||||
|
||||
builder = _raptor_trees[doc_id]
|
||||
|
||||
if not filepath:
|
||||
filepath = str(PROJECT_ROOT / "data" / "raptor" / f"{doc_id}_tree.json")
|
||||
|
||||
builder.save(filepath)
|
||||
|
||||
return {
|
||||
'status': 'success',
|
||||
'doc_id': doc_id,
|
||||
'filepath': filepath
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"保存失败: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/raptor/tree/load")
|
||||
async def load_tree(filepath: str):
|
||||
"""从文件加载树"""
|
||||
try:
|
||||
from src.rag.advanced.raptor import RaptorTreeBuilder
|
||||
|
||||
builder = RaptorTreeBuilder.load(filepath)
|
||||
|
||||
# 从文件路径提取doc_id
|
||||
doc_id = Path(filepath).stem.replace('_tree', '')
|
||||
_raptor_trees[doc_id] = builder
|
||||
|
||||
return {
|
||||
'status': 'success',
|
||||
'doc_id': doc_id,
|
||||
'root_id': builder.root_id,
|
||||
'total_nodes': len(builder.nodes)
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"加载失败: {str(e)}")
|
||||
|
||||
|
||||
@router.get("/raptor/trees")
|
||||
async def list_trees():
|
||||
"""列出所有可用的树"""
|
||||
return {
|
||||
'trees': [
|
||||
{
|
||||
'doc_id': doc_id,
|
||||
'root_id': builder.root_id,
|
||||
'total_nodes': len(builder.nodes)
|
||||
}
|
||||
for doc_id, builder in _raptor_trees.items()
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/raptor/tree/{doc_id}")
|
||||
async def delete_tree(doc_id: str):
|
||||
"""删除树"""
|
||||
try:
|
||||
if doc_id not in _raptor_trees:
|
||||
raise HTTPException(status_code=404, detail=f"树不存在: {doc_id}")
|
||||
|
||||
del _raptor_trees[doc_id]
|
||||
|
||||
return {
|
||||
'status': 'success',
|
||||
'doc_id': doc_id,
|
||||
'message': '树已删除'
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"删除失败: {str(e)}")
|
||||
Executable
+58
@@ -0,0 +1,58 @@
|
||||
"""推理引擎API路由"""
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# 导入推理引擎
|
||||
import sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / 'scripts' / 'utils'))
|
||||
try:
|
||||
from reasoning_engine import ReasoningEngine
|
||||
except ImportError:
|
||||
# 备用导入方式
|
||||
import importlib.util
|
||||
reasoning_engine_path = Path(__file__).parent.parent / 'scripts' / 'utils' / 'reasoning_engine.py'
|
||||
spec = importlib.util.spec_from_file_location("reasoning_engine", reasoning_engine_path)
|
||||
reasoning_engine = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(reasoning_engine)
|
||||
ReasoningEngine = reasoning_engine.ReasoningEngine
|
||||
|
||||
class ReasoningQuery(BaseModel):
|
||||
question: str
|
||||
|
||||
@router.post("/reasoning/query")
|
||||
async def reasoning_query(query: ReasoningQuery):
|
||||
"""多跳推理查询"""
|
||||
try:
|
||||
engine = ReasoningEngine()
|
||||
result = engine.multi_hop_query(query.question)
|
||||
return {"status": "success", **result}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
@router.get("/reasoning/examples")
|
||||
async def reasoning_examples():
|
||||
"""推理查询示例"""
|
||||
return {
|
||||
"status": "success",
|
||||
"examples": [
|
||||
{
|
||||
"question": "查找与CYP71D184同组的基因",
|
||||
"description": "多跳推理:基因→分组→同组基因",
|
||||
"hops": 2
|
||||
},
|
||||
{
|
||||
"question": "P450分组统计比较",
|
||||
"description": "聚合统计:所有分组的统计分析",
|
||||
"hops": 1
|
||||
},
|
||||
{
|
||||
"question": "最大的分组是哪个",
|
||||
"description": "极值查询:查找最大分组",
|
||||
"hops": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
Executable
+78
@@ -0,0 +1,78 @@
|
||||
"""
|
||||
API路由 - 纯净版RAG系统
|
||||
保留RAG核心接口,为bbav20311接入预留接口
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, UploadFile, File, HTTPException
|
||||
from pathlib import Path
|
||||
import os
|
||||
|
||||
from api.rag_routes import router as rag_router
|
||||
from api.case_routes import router as case_router
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# ============================================
|
||||
# RAG核心接口(已实现)
|
||||
# ============================================
|
||||
|
||||
# RAG对话接口
|
||||
router.include_router(rag_router, prefix="/rag", tags=["RAG"])
|
||||
|
||||
# 真实复现案例问答接口(确定性QA,不调用LLM)
|
||||
router.include_router(case_router, prefix="/case", tags=["Case QA"])
|
||||
|
||||
# ============================================
|
||||
# 预留接口(为bbav20311接入准备)
|
||||
# ============================================
|
||||
|
||||
# 文件上传接口(预留)
|
||||
@router.post("/upload")
|
||||
async def upload_file(file: UploadFile = File(...)):
|
||||
"""文件上传接口 - 预留"""
|
||||
raise HTTPException(status_code=501, detail="待bbav20311接入后实现")
|
||||
|
||||
# 批量上传接口(预留)
|
||||
@router.post("/upload/batch")
|
||||
async def upload_batch(files: list[UploadFile] = File(...)):
|
||||
"""批量上传接口 - 预留"""
|
||||
raise HTTPException(status_code=501, detail="待bbav20311接入后实现")
|
||||
|
||||
# 工作空间接口(预留)
|
||||
@router.get("/workspace/{job_id}/files")
|
||||
async def get_workspace_files(job_id: str):
|
||||
"""工作空间文件接口 - 预留"""
|
||||
raise HTTPException(status_code=501, detail="待bbav20311接入后实现")
|
||||
|
||||
# 基因家族分析接口(预留)
|
||||
@router.post("/analysis/gene-family")
|
||||
async def analyze_gene_family(job_id: str, species: str, gene_family: str):
|
||||
"""基因家族分析接口 - 预留"""
|
||||
raise HTTPException(status_code=501, detail="待bbav20311接入后实现")
|
||||
|
||||
# 工具执行接口(预留)
|
||||
@router.post("/tools/{tool_id}/execute")
|
||||
async def execute_tool(tool_id: str, params: dict):
|
||||
"""工具执行接口 - 预留"""
|
||||
raise HTTPException(status_code=501, detail="待bbav20311接入后实现")
|
||||
|
||||
# ============================================
|
||||
# 状态查询接口
|
||||
# ============================================
|
||||
|
||||
@router.get("/status")
|
||||
async def get_status():
|
||||
"""获取RAG系统状态"""
|
||||
try:
|
||||
from src.rag.retriever import retriever
|
||||
indexed_count = retriever.count_chunks() if retriever else 0
|
||||
except Exception as e:
|
||||
# 如果retriever未初始化,使用RAG路由的状态
|
||||
indexed_count = 0
|
||||
|
||||
return {
|
||||
"status": "ready",
|
||||
"indexed_documents": indexed_count,
|
||||
"rag_enabled": True,
|
||||
"analysis_enabled": False, # 待接入bbav20311后开启
|
||||
}
|
||||
Executable
+73
@@ -0,0 +1,73 @@
|
||||
from datetime import datetime
|
||||
from typing import Any, Union
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class FileInfo(BaseModel):
|
||||
name: str
|
||||
path: str
|
||||
size: int | None = None
|
||||
|
||||
|
||||
class QueryRequest(BaseModel):
|
||||
session_id: str
|
||||
query: str
|
||||
files: list[Union[str, FileInfo]] = []
|
||||
user_input: dict[str, Any] = {}
|
||||
|
||||
|
||||
class QueryResponse(BaseModel):
|
||||
session_id: str
|
||||
status: str
|
||||
mode: str
|
||||
response: dict[str, Any] = {}
|
||||
messages: list[dict[str, str]] = []
|
||||
|
||||
|
||||
class SessionCreate(BaseModel):
|
||||
user_id: str | None = None
|
||||
|
||||
|
||||
class SessionResponse(BaseModel):
|
||||
session_id: str
|
||||
created_at: datetime
|
||||
status: str = "active"
|
||||
|
||||
|
||||
class CaseCreate(BaseModel):
|
||||
case_id: str
|
||||
user_id: str | None = None
|
||||
gene_family: str | None = None
|
||||
species: str | None = None
|
||||
|
||||
|
||||
class CaseResponse(BaseModel):
|
||||
case_id: str
|
||||
user_id: str | None
|
||||
gene_family: str | None
|
||||
species: str | None
|
||||
status: str
|
||||
created_at: datetime
|
||||
results: dict[str, Any] = {}
|
||||
|
||||
|
||||
class FeedbackRequest(BaseModel):
|
||||
case_id: str
|
||||
rating: int
|
||||
comments: str | None = None
|
||||
issues: list[str] = []
|
||||
|
||||
|
||||
class WebSocketMessage(BaseModel):
|
||||
type: str
|
||||
session_id: str
|
||||
data: dict[str, Any] = {}
|
||||
|
||||
|
||||
class ProgressUpdate(BaseModel):
|
||||
session_id: str
|
||||
current_agent: str | None
|
||||
status: str
|
||||
progress: float
|
||||
message: str
|
||||
Executable
+206
@@ -0,0 +1,206 @@
|
||||
"""
|
||||
物种查询API路由
|
||||
提供基因家族-物种索引查询功能
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
from pydantic import BaseModel
|
||||
from typing import List, Dict, Optional
|
||||
from pathlib import Path
|
||||
import json
|
||||
|
||||
from src.rag.project_config import PROJECT_ROOT
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
INDEX_FILE = PROJECT_ROOT / "data" / "gene_family_species_index.json"
|
||||
|
||||
|
||||
class SpeciesInfo(BaseModel):
|
||||
"""物种信息"""
|
||||
paper_id: str
|
||||
scientific_name: str
|
||||
common_name_en: str
|
||||
common_name_cn: str
|
||||
comparison_species: List[str]
|
||||
relations_count: int
|
||||
entities_count: int
|
||||
pmid: Optional[str]
|
||||
|
||||
|
||||
class GeneFamilyInfo(BaseModel):
|
||||
"""基因家族信息"""
|
||||
gene_family: str
|
||||
description: str
|
||||
description_cn: str
|
||||
total_species: int
|
||||
species_list: List[SpeciesInfo]
|
||||
|
||||
|
||||
class SpeciesListResponse(BaseModel):
|
||||
"""物种列表响应"""
|
||||
gene_family: str
|
||||
total_species: int
|
||||
species: List[SpeciesInfo]
|
||||
|
||||
|
||||
class SpeciesQueryResponse(BaseModel):
|
||||
"""物种查询响应"""
|
||||
query: str
|
||||
matched_species: List[Dict]
|
||||
total_matches: int
|
||||
|
||||
|
||||
def load_species_index() -> dict:
|
||||
"""加载物种索引"""
|
||||
if not INDEX_FILE.exists():
|
||||
raise HTTPException(status_code=500, detail="物种索引文件不存在,请先运行索引构建脚本")
|
||||
|
||||
with open(INDEX_FILE, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
@router.get("/species/list", response_model=SpeciesListResponse)
|
||||
async def list_species_by_gene_family(
|
||||
gene_family: str = Query("SPL", description="基因家族名称,如SPL、NAC、WRKY等")
|
||||
):
|
||||
"""
|
||||
获取指定基因家族已分析的所有物种列表
|
||||
|
||||
示例:
|
||||
- /api/v1/species/list?gene_family=SPL
|
||||
"""
|
||||
try:
|
||||
index = load_species_index()
|
||||
|
||||
gene_family_upper = gene_family.upper()
|
||||
|
||||
if gene_family_upper not in index.get("gene_families", {}):
|
||||
return SpeciesListResponse(
|
||||
gene_family=gene_family,
|
||||
total_species=0,
|
||||
species=[]
|
||||
)
|
||||
|
||||
gf_info = index["gene_families"][gene_family_upper]
|
||||
|
||||
# 构建响应
|
||||
species_list = []
|
||||
for species in gf_info.get("species_list", []):
|
||||
species_list.append(SpeciesInfo(
|
||||
paper_id=species["paper_id"],
|
||||
scientific_name=species["scientific_name"],
|
||||
common_name_en=species["common_name_en"],
|
||||
common_name_cn=species["common_name_cn"],
|
||||
comparison_species=species["comparison_species"],
|
||||
relations_count=species["relations_count"],
|
||||
entities_count=species["entities_count"],
|
||||
pmid=species.get("pmid") or None
|
||||
))
|
||||
|
||||
return SpeciesListResponse(
|
||||
gene_family=gene_family_upper,
|
||||
total_species=gf_info["total_species"],
|
||||
species=species_list
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"查询失败: {str(e)}")
|
||||
|
||||
|
||||
@router.get("/species/search", response_model=SpeciesQueryResponse)
|
||||
async def search_species(
|
||||
query: str = Query(..., description="物种查询关键词,支持中文、英文、拉丁学名")
|
||||
):
|
||||
"""
|
||||
搜索特定物种是否做过基因家族分析
|
||||
|
||||
示例:
|
||||
- /api/v1/species/search?query=苦荞
|
||||
- /api/v1/species/search?query=tartary
|
||||
- /api/v1/species/search?query=Fagopyrum
|
||||
"""
|
||||
try:
|
||||
index = load_species_index()
|
||||
|
||||
query_lower = query.lower()
|
||||
matches = []
|
||||
|
||||
# 遍历所有基因家族
|
||||
for gf_name, gf_info in index.get("gene_families", {}).items():
|
||||
for species in gf_info.get("species_list", []):
|
||||
# 匹配多个字段
|
||||
match_fields = [
|
||||
species["scientific_name"].lower(),
|
||||
species["common_name_en"].lower(),
|
||||
species["common_name_cn"]
|
||||
]
|
||||
|
||||
if any(query_lower in field for field in match_fields):
|
||||
matches.append({
|
||||
"gene_family": gf_name,
|
||||
"paper_id": species["paper_id"],
|
||||
"scientific_name": species["scientific_name"],
|
||||
"common_name_en": species["common_name_en"],
|
||||
"common_name_cn": species["common_name_cn"],
|
||||
"pmid": species.get("pmid"),
|
||||
"relations_count": species["relations_count"],
|
||||
"entities_count": species["entities_count"]
|
||||
})
|
||||
|
||||
return SpeciesQueryResponse(
|
||||
query=query,
|
||||
matched_species=matches,
|
||||
total_matches=len(matches)
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"查询失败: {str(e)}")
|
||||
|
||||
|
||||
@router.get("/species/gene-families", response_model=List[str])
|
||||
async def list_gene_families():
|
||||
"""
|
||||
获取所有已索引的基因家族列表
|
||||
|
||||
示例:
|
||||
- /api/v1/species/gene-families
|
||||
"""
|
||||
try:
|
||||
index = load_species_index()
|
||||
return list(index.get("gene_families", {}).keys())
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"查询失败: {str(e)}")
|
||||
|
||||
|
||||
@router.get("/species/stats")
|
||||
async def get_statistics():
|
||||
"""
|
||||
获取基因家族-物种索引统计信息
|
||||
|
||||
示例:
|
||||
- /api/v1/species/stats
|
||||
"""
|
||||
try:
|
||||
index = load_species_index()
|
||||
|
||||
stats = {
|
||||
"total_gene_families": len(index.get("gene_families", {})),
|
||||
"total_papers": len(index.get("papers", {})),
|
||||
"total_species_indexed": len(index.get("species_index", {})),
|
||||
"metadata": index.get("metadata", {}),
|
||||
"gene_families": {}
|
||||
}
|
||||
|
||||
for gf_name, gf_info in index.get("gene_families", {}).items():
|
||||
stats["gene_families"][gf_name] = {
|
||||
"description_cn": gf_info.get("description_cn", ""),
|
||||
"total_species": gf_info.get("total_species", 0),
|
||||
"species_names": [s["common_name_cn"] for s in gf_info.get("species_list", [])]
|
||||
}
|
||||
|
||||
return stats
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"查询失败: {str(e)}")
|
||||
Executable
+409
@@ -0,0 +1,409 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
时序知识图谱API路由
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from typing import List, Dict, Optional
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# 全局时序知识图谱实例
|
||||
_temporal_kg = None
|
||||
|
||||
|
||||
def get_temporal_kg():
|
||||
"""获取时序知识图谱实例"""
|
||||
global _temporal_kg
|
||||
if _temporal_kg is None:
|
||||
from src.rag.temporal import TemporalKnowledgeGraph
|
||||
_temporal_kg = TemporalKnowledgeGraph(name="gene_family_temporal_kg")
|
||||
return _temporal_kg
|
||||
|
||||
|
||||
class CreateEntityRequest(BaseModel):
|
||||
entity_id: str
|
||||
name: str
|
||||
entity_type: str
|
||||
start_year: Optional[int] = None
|
||||
end_year: Optional[int] = None
|
||||
properties: Dict = {}
|
||||
|
||||
|
||||
class CreateRelationRequest(BaseModel):
|
||||
relation_id: str
|
||||
source_id: str
|
||||
target_id: str
|
||||
relation_type: str
|
||||
year: Optional[int] = None
|
||||
properties: Dict = {}
|
||||
|
||||
|
||||
class TimeRangeQueryRequest(BaseModel):
|
||||
start_year: Optional[int] = None
|
||||
end_year: Optional[int] = None
|
||||
entity_types: List[str] = []
|
||||
relation_types: List[str] = []
|
||||
|
||||
|
||||
class TimelineQueryRequest(BaseModel):
|
||||
entity_id: Optional[str] = None
|
||||
topic: Optional[str] = None
|
||||
start_year: Optional[int] = None
|
||||
end_year: Optional[int] = None
|
||||
|
||||
|
||||
@router.post("/temporal/entity")
|
||||
async def create_entity(request: CreateEntityRequest):
|
||||
"""创建时序实体"""
|
||||
try:
|
||||
from src.rag.temporal import TemporalEntity, TimePeriod, TimePoint
|
||||
|
||||
kg = get_temporal_kg()
|
||||
|
||||
# 构建时间范围
|
||||
valid_time = None
|
||||
if request.start_year:
|
||||
valid_time = TimePeriod(
|
||||
start=TimePoint.from_year(request.start_year),
|
||||
end=TimePoint.from_year(request.end_year) if request.end_year else TimePoint.from_year(request.start_year)
|
||||
)
|
||||
|
||||
entity = TemporalEntity(
|
||||
entity_id=request.entity_id,
|
||||
name=request.name,
|
||||
entity_type=request.entity_type,
|
||||
valid_time=valid_time,
|
||||
properties=request.properties
|
||||
)
|
||||
|
||||
kg.add_entity(entity)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"entity_id": request.entity_id,
|
||||
"message": f"实体 {request.name} 创建成功"
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"创建实体失败: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/temporal/relation")
|
||||
async def create_relation(request: CreateRelationRequest):
|
||||
"""创建时序关系"""
|
||||
try:
|
||||
from src.rag.temporal import TemporalRelation, TimePoint
|
||||
|
||||
kg = get_temporal_kg()
|
||||
|
||||
# 构建时间戳
|
||||
timestamp = None
|
||||
if request.year:
|
||||
timestamp = TimePoint.from_year(request.year)
|
||||
|
||||
relation = TemporalRelation(
|
||||
relation_id=request.relation_id,
|
||||
source_id=request.source_id,
|
||||
target_id=request.target_id,
|
||||
relation_type=request.relation_type,
|
||||
timestamp=timestamp,
|
||||
properties=request.properties
|
||||
)
|
||||
|
||||
kg.add_relation(relation)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"relation_id": request.relation_id,
|
||||
"message": f"关系 {request.relation_type} 创建成功"
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"创建关系失败: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/temporal/query/range")
|
||||
async def query_by_time_range(request: TimeRangeQueryRequest):
|
||||
"""按时间范围查询"""
|
||||
try:
|
||||
from src.rag.temporal import TimeRangeQuery, TimePoint
|
||||
|
||||
kg = get_temporal_kg()
|
||||
|
||||
query = TimeRangeQuery(
|
||||
start=TimePoint.from_year(request.start_year) if request.start_year else None,
|
||||
end=TimePoint.from_year(request.end_year) if request.end_year else None,
|
||||
entity_types=request.entity_types,
|
||||
relation_types=request.relation_types
|
||||
)
|
||||
|
||||
result = kg.query(query)
|
||||
|
||||
return {
|
||||
"query": query.to_dict(),
|
||||
"entities": [e.to_dict() for e in result["entities"]],
|
||||
"relations": [r.to_dict() for r in result["relations"]],
|
||||
"total_entities": len(result["entities"]),
|
||||
"total_relations": len(result["relations"])
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"查询失败: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/temporal/timeline")
|
||||
async def get_timeline(request: TimelineQueryRequest):
|
||||
"""获取时间线"""
|
||||
try:
|
||||
kg = get_temporal_kg()
|
||||
|
||||
timeline = kg.get_timeline(
|
||||
entity_id=request.entity_id,
|
||||
topic=request.topic,
|
||||
start_year=request.start_year,
|
||||
end_year=request.end_year
|
||||
)
|
||||
|
||||
return timeline.to_dict()
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"获取时间线失败: {str(e)}")
|
||||
|
||||
|
||||
@router.get("/temporal/timeline/visualization")
|
||||
async def get_timeline_visualization(
|
||||
topic: Optional[str] = None,
|
||||
start_year: Optional[int] = None,
|
||||
end_year: Optional[int] = None
|
||||
):
|
||||
"""获取时间线可视化数据"""
|
||||
try:
|
||||
kg = get_temporal_kg()
|
||||
|
||||
visual_data = kg.visualize_timeline(topic=topic)
|
||||
|
||||
return {
|
||||
"topic": topic,
|
||||
"start_year": start_year,
|
||||
"end_year": end_year,
|
||||
"visualization_data": visual_data
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"获取可视化数据失败: {str(e)}")
|
||||
|
||||
|
||||
@router.get("/temporal/stats")
|
||||
async def get_stats():
|
||||
"""获取统计信息"""
|
||||
try:
|
||||
kg = get_temporal_kg()
|
||||
stats = kg.get_stats()
|
||||
|
||||
return stats.to_dict()
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"获取统计失败: {str(e)}")
|
||||
|
||||
|
||||
@router.get("/temporal/entity/{entity_id}")
|
||||
async def get_entity(entity_id: str):
|
||||
"""获取实体详情"""
|
||||
try:
|
||||
kg = get_temporal_kg()
|
||||
entity = kg.get_entity(entity_id)
|
||||
|
||||
if not entity:
|
||||
raise HTTPException(status_code=404, detail=f"实体不存在: {entity_id}")
|
||||
|
||||
return entity.to_dict()
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"获取实体失败: {str(e)}")
|
||||
|
||||
|
||||
@router.get("/temporal/entity/{entity_id}/history")
|
||||
async def get_entity_history(entity_id: str):
|
||||
"""获取实体历史"""
|
||||
try:
|
||||
kg = get_temporal_kg()
|
||||
timeline = kg.get_timeline(entity_id=entity_id)
|
||||
|
||||
return timeline.to_dict()
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"获取历史失败: {str(e)}")
|
||||
|
||||
|
||||
@router.get("/temporal/entities/by-type/{entity_type}")
|
||||
async def get_entities_by_type(entity_type: str):
|
||||
"""按类型获取实体"""
|
||||
try:
|
||||
kg = get_temporal_kg()
|
||||
entities = kg.get_entities_by_type(entity_type)
|
||||
|
||||
return {
|
||||
"entity_type": entity_type,
|
||||
"entities": [e.to_dict() for e in entities],
|
||||
"total": len(entities)
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"获取实体失败: {str(e)}")
|
||||
|
||||
|
||||
@router.get("/temporal/entities/by-year/{year}")
|
||||
async def get_entities_by_year(year: int):
|
||||
"""按年份获取实体"""
|
||||
try:
|
||||
kg = get_temporal_kg()
|
||||
entities = kg.get_entities_by_year(year)
|
||||
|
||||
return {
|
||||
"year": year,
|
||||
"entities": [e.to_dict() for e in entities],
|
||||
"total": len(entities)
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"获取实体失败: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/temporal/consistency/check")
|
||||
async def check_consistency():
|
||||
"""检查图谱一致性"""
|
||||
try:
|
||||
kg = get_temporal_kg()
|
||||
conflicts = kg.check_consistency()
|
||||
|
||||
return {
|
||||
"consistent": len(conflicts) == 0,
|
||||
"conflicts": conflicts,
|
||||
"total_conflicts": len(conflicts)
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"检查一致性失败: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/temporal/infer")
|
||||
async def infer_relations():
|
||||
"""推断新关系"""
|
||||
try:
|
||||
kg = get_temporal_kg()
|
||||
inferred = kg.infer_relations()
|
||||
|
||||
return {
|
||||
"inferred_relations": [r.to_dict() for r in inferred],
|
||||
"total_inferred": len(inferred)
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"推断关系失败: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/temporal/save")
|
||||
async def save_graph(filepath: str = "/tmp/temporal_kg.json"):
|
||||
"""保存图谱"""
|
||||
try:
|
||||
kg = get_temporal_kg()
|
||||
kg.save(filepath)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"filepath": filepath
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"保存失败: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/temporal/load")
|
||||
async def load_graph(filepath: str = "/tmp/temporal_kg.json"):
|
||||
"""加载图谱"""
|
||||
try:
|
||||
global _temporal_kg
|
||||
from src.rag.temporal import TemporalKnowledgeGraph
|
||||
|
||||
_temporal_kg = TemporalKnowledgeGraph.load(filepath)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"filepath": filepath,
|
||||
"name": _temporal_kg.name
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"加载失败: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/temporal/demo/populate")
|
||||
async def populate_demo_data():
|
||||
"""填充演示数据"""
|
||||
try:
|
||||
from src.rag.temporal import (
|
||||
TemporalEntity, TemporalRelation,
|
||||
TimePoint, TimePeriod
|
||||
)
|
||||
|
||||
kg = get_temporal_kg()
|
||||
|
||||
# 添加演示实体
|
||||
demo_entities = [
|
||||
("spl_arabidopsis", "SPL基因家族(拟南芥)", "gene_family", 2018, 2023),
|
||||
("spl_rice", "SPL基因家族(水稻)", "gene_family", 2019, 2023),
|
||||
("spl_poplar", "SPL基因家族(杨树)", "gene_family", 2020, 2023),
|
||||
("spl3", "SPL3基因", "gene", 2018, None),
|
||||
("spl9", "SPL9基因", "gene", 2019, None),
|
||||
("flowering", "开花调控", "process", 2018, 2023),
|
||||
("stress_response", "胁迫响应", "process", 2020, 2023),
|
||||
]
|
||||
|
||||
for entity_id, name, entity_type, start_year, end_year in demo_entities:
|
||||
valid_time = TimePeriod(
|
||||
start=TimePoint.from_year(start_year),
|
||||
end=TimePoint.from_year(end_year) if end_year else TimePoint.from_year(2023)
|
||||
)
|
||||
|
||||
entity = TemporalEntity(
|
||||
entity_id=entity_id,
|
||||
name=name,
|
||||
entity_type=entity_type,
|
||||
valid_time=valid_time,
|
||||
properties={"topic": "SPL基因家族"}
|
||||
)
|
||||
kg.add_entity(entity)
|
||||
|
||||
# 添加演示关系
|
||||
demo_relations = [
|
||||
("rel1", "spl3", "flowering", "regulates", 2019),
|
||||
("rel2", "spl9", "flowering", "regulates", 2020),
|
||||
("rel3", "spl_arabidopsis", "stress_response", "involved_in", 2021),
|
||||
("rel4", "spl3", "spl9", "interacts_with", 2020),
|
||||
]
|
||||
|
||||
for rel_id, source, target, rel_type, year in demo_relations:
|
||||
relation = TemporalRelation(
|
||||
relation_id=rel_id,
|
||||
source_id=source,
|
||||
target_id=target,
|
||||
relation_type=rel_type,
|
||||
timestamp=TimePoint.from_year(year),
|
||||
properties={}
|
||||
)
|
||||
kg.add_relation(relation)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"entities_added": len(demo_entities),
|
||||
"relations_added": len(demo_relations),
|
||||
"message": "演示数据已填充"
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"填充演示数据失败: {str(e)}")
|
||||
Executable
+149
@@ -0,0 +1,149 @@
|
||||
from __future__ import annotations
|
||||
import os
|
||||
from fastapi import APIRouter
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
router = APIRouter(prefix="/v15-agentic-qa", tags=["V15 Agentic QA"])
|
||||
|
||||
_retriever = None
|
||||
_coordinator = None
|
||||
_memory = None
|
||||
_mem_ok = False
|
||||
|
||||
|
||||
def _get_retriever():
|
||||
global _retriever
|
||||
if _retriever is None:
|
||||
from src.rag.v15_hybrid_retriever import V15HybridRetriever
|
||||
_retriever = V15HybridRetriever()
|
||||
return _retriever
|
||||
|
||||
|
||||
def _get_coordinator():
|
||||
global _coordinator
|
||||
if _coordinator is None:
|
||||
from src.rag.agentic_coordinator import AgenticCoordinator
|
||||
_coordinator = AgenticCoordinator(
|
||||
retriever=_get_retriever(), max_iterations=3, top_k_base=15)
|
||||
return _coordinator
|
||||
|
||||
|
||||
def _get_memory():
|
||||
global _memory, _mem_ok
|
||||
if _memory is None and not _mem_ok:
|
||||
try:
|
||||
from src.rag.memory.memory_manager import MemoryManager
|
||||
_memory = MemoryManager(user_id="agentic", session_id="qa")
|
||||
_mem_ok = True
|
||||
except Exception as e:
|
||||
print(f"[V15 Agentic] Memory disabled: {e}")
|
||||
_mem_ok = True
|
||||
return _memory
|
||||
|
||||
|
||||
class AgenticQARequest(BaseModel):
|
||||
question: str = Field(..., description="Question")
|
||||
top_k: int = Field(15, ge=1, le=50)
|
||||
max_hops: int | None = Field(None)
|
||||
include_candidate: bool = Field(False)
|
||||
use_llm: bool = Field(True)
|
||||
provider: str | None = Field(None)
|
||||
model: str | None = Field(None)
|
||||
anchor_entity: str | None = Field(None)
|
||||
no_memory: bool = Field(False)
|
||||
|
||||
|
||||
@router.post("/chat")
|
||||
async def agentic_qa_chat(req: AgenticQARequest):
|
||||
coordinator = _get_coordinator()
|
||||
memory = _get_memory()
|
||||
question = req.question
|
||||
mem_note = ""
|
||||
if memory and not req.no_memory:
|
||||
try:
|
||||
enriched = memory.augment_query(question)
|
||||
if enriched != question:
|
||||
question = enriched
|
||||
mem_note = "memory_augmented"
|
||||
except:
|
||||
pass
|
||||
result = await coordinator.run(
|
||||
question=question, top_k=req.top_k,
|
||||
anchor_entity=req.anchor_entity or None,
|
||||
include_candidate=req.include_candidate)
|
||||
evidence = result["retrieved_evidence"]
|
||||
gpaths = result["graph_paths"]
|
||||
aplan = result["retrieval_plan"]
|
||||
evres = result["evaluation"]
|
||||
notes = result["notes"]
|
||||
if req.max_hops is not None:
|
||||
aplan["max_hops"] = req.max_hops
|
||||
citations = []
|
||||
for idx, ev in enumerate(evidence[:30]):
|
||||
citations.append({
|
||||
"id": f"S{idx+1}",
|
||||
"source": ev.get("source", ""),
|
||||
"relation": ev.get("relation", ""),
|
||||
"target": ev.get("target", ""),
|
||||
"evidence": ev.get("evidence", ""),
|
||||
"paper_id": ev.get("paper_id", ""),
|
||||
"pubmed_id": ev.get("pubmed_id", ""),
|
||||
"section": ev.get("section", ""),
|
||||
})
|
||||
answer = ""
|
||||
llm_model = ""
|
||||
if req.use_llm and evidence:
|
||||
try:
|
||||
from api.kb_qa_routes import _generate_llm_answer
|
||||
answer = await _generate_llm_answer(
|
||||
req.question, aplan.get("layer", "auto"),
|
||||
evidence, provider=req.provider, model=req.model)
|
||||
p = req.provider or os.environ.get("V15_KB_QA_LLM_PROVIDER", "")
|
||||
m = req.model or os.environ.get("V15_KB_QA_LLM_MODEL", "")
|
||||
llm_model = f"{p}:{m}"
|
||||
except Exception as exc:
|
||||
answer = f"[agentic] LLM failed: {exc}"
|
||||
if not answer and not evidence:
|
||||
answer = "Insufficient evidence."
|
||||
if memory and not req.no_memory and answer:
|
||||
try: memory.add_interaction(req.question, answer)
|
||||
except: pass
|
||||
return {
|
||||
"question": req.question,
|
||||
"answer": answer,
|
||||
"citations": citations,
|
||||
"graph_paths": gpaths,
|
||||
"agentic_plan": aplan,
|
||||
"evaluation": evres,
|
||||
"retrieval_notes": notes,
|
||||
"evidence_count": len(evidence),
|
||||
"graph_paths_count": len(gpaths),
|
||||
"llm_model": llm_model,
|
||||
"memory_note": mem_note,
|
||||
"mode": "v15_agentic_qa",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/info")
|
||||
async def agentic_qa_info():
|
||||
return {
|
||||
"mode": "v15_agentic_qa",
|
||||
"components": {
|
||||
"planner": "QueryAnalyzer + RetrievalStrategyAgent",
|
||||
"evaluator": "Self-evaluation (count/confidence/entity)",
|
||||
"retriever": "V15HybridRetriever (Qdrant+Neo4j)",
|
||||
"memory": "MemoryManager",
|
||||
},
|
||||
"loop": "Max 3 iterations with self-evaluation",
|
||||
"question_types": {
|
||||
"FAQ": "hybrid mode, 2 hops",
|
||||
"METHOD_CONSULT": "method_relations layer, 1 hop",
|
||||
"ENTITY_QUERY": "exact+vector+KG, 1 hop",
|
||||
"RELATION_QUERY": "KG+vector, 3 hops",
|
||||
"UNKNOWN": "hybrid mode, 2 hops",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
print("[V15 Agentic QA] Routes loaded with AgenticCoordinator")
|
||||
|
||||
Executable
+192
@@ -0,0 +1,192 @@
|
||||
"""
|
||||
V15 SRT Knowledge Base --- Neo4j Graph QA Routes
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import os, re as _re
|
||||
from typing import Any, Dict, List, Optional
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
from neo4j import GraphDatabase
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
router = APIRouter(prefix="/v15-graph-qa", tags=["V15 Neo4j Graph QA"])
|
||||
_neo4j_driver = None
|
||||
NEO4J_URI = os.environ.get("NEO4J_URI", "bolt://localhost:7687")
|
||||
NEO4J_USER = os.environ.get("NEO4J_USER", "neo4j")
|
||||
NEO4J_PASSWORD = os.environ.get("NEO4J_PASSWORD", "password")
|
||||
|
||||
VALID_LAYERS = {"main_kg", "background_relations", "method_relations", "dataset_metadata", "candidate_review"}
|
||||
LAYER_PRIORITY = {"main_kg": 0, "method_relations": 1, "dataset_metadata": 2, "background_relations": 3, "candidate_review": 4}
|
||||
SKIP_WORDS = {"What","Which","Where","When","How","Does","Did","Are","Was","Were","The","And","For","With","From","Into","About","That","This","These","Those","Tell","Please","List","Show","Give","Find","Would","Could","Should","May","Might","Can","Only","Also","Both","Each"}
|
||||
|
||||
def _get_driver():
|
||||
global _neo4j_driver
|
||||
if _neo4j_driver is None:
|
||||
_neo4j_driver = GraphDatabase.driver(NEO4J_URI, auth=(NEO4J_USER, NEO4J_PASSWORD))
|
||||
return _neo4j_driver
|
||||
|
||||
def _layer_cond(alias, allow_list, inc_cand):
|
||||
if allow_list:
|
||||
valid = [l for l in allow_list if l in VALID_LAYERS] or ["main_kg"]
|
||||
else:
|
||||
valid = ["main_kg", "background_relations", "method_relations", "dataset_metadata"]
|
||||
if not inc_cand:
|
||||
valid = [l for l in valid if l != "candidate_review"]
|
||||
quoted = ", ".join(f"'{l}'" for l in valid)
|
||||
return f"{alias}.output_layer IN [{quoted}]"
|
||||
|
||||
def _record_to_dict(rec, ea="r", sa="s", ta="t"):
|
||||
e = rec[ea]; s = rec[sa]; t = rec[ta]
|
||||
return {"source": s.get("name",""), "source_type": s.get("entity_type",""),
|
||||
"relation": e.get("relation",""), "target": t.get("name",""),
|
||||
"target_type": t.get("entity_type",""),
|
||||
"evidence": str(e.get("evidence","") or ""),
|
||||
"section": str(e.get("section","") or ""),
|
||||
"pubmed_id": str(e.get("pubmed_id","") or ""),
|
||||
"paper_id": str(e.get("paper_id","") or ""),
|
||||
"confidence": e.get("confidence"),
|
||||
"claim_scope": str(e.get("claim_scope","") or ""),
|
||||
"output_layer": str(e.get("output_layer","") or ""),
|
||||
"candidate_status": str(e.get("candidate_status","") or "")}
|
||||
|
||||
class PReq(BaseModel):
|
||||
source_entity: str; target_entity: str
|
||||
max_hops: int = Field(2, ge=1, le=5)
|
||||
layers: Optional[List[str]] = Field(None)
|
||||
include_candidate: bool = False
|
||||
top_k_paths: int = Field(10, ge=1, le=50)
|
||||
|
||||
class GQAReq(BaseModel):
|
||||
question: str
|
||||
top_k: int = Field(10, ge=1, le=50)
|
||||
layers: Optional[List[str]] = Field(None)
|
||||
include_candidate: bool = False
|
||||
use_llm: bool = True
|
||||
provider: Optional[str] = Field(None)
|
||||
model: Optional[str] = Field(None)
|
||||
|
||||
@router.get("/info")
|
||||
async def info():
|
||||
driver = _get_driver()
|
||||
with driver.session() as s:
|
||||
ec = s.run("MATCH (e:Entity) RETURN count(e) AS c").single()["c"]
|
||||
rc = s.run("MATCH ()-[r:RELATION]->() RETURN count(r) AS c").single()["c"]
|
||||
# Prefer canonical Paper nodes for paper count. Distinct r.paper_id can be
|
||||
# smaller if a selected paper has no retained relation, or inflated in old
|
||||
# imports that used PMID/PMCID as paper_id.
|
||||
try:
|
||||
pc = s.run("MATCH (p:Paper) RETURN count(p) AS c").single()["c"]
|
||||
except Exception:
|
||||
pc = s.run("MATCH ()-[r:RELATION]->() RETURN count(DISTINCT r.paper_id) AS c").single()["c"]
|
||||
lc = {}
|
||||
for row in s.run("MATCH ()-[r:RELATION]->() RETURN r.output_layer AS l, count(*) AS c"):
|
||||
lc[row["l"]] = row["c"]
|
||||
return {"graph":"V15 SRT KB (Neo4j)", "entity_count":ec, "relation_count":rc,
|
||||
"paper_count":pc, "layers":lc,
|
||||
"schema":"(:Entity)-[r:RELATION]->(:Entity) with r.relation"}
|
||||
|
||||
@router.get("/entity/{name}/neighborhood")
|
||||
async def nbhd(name: str, depth: int=Query(1,ge=1,le=3),
|
||||
layers: Optional[str]=Query(None),
|
||||
include_candidate: bool=Query(False),
|
||||
top_k: int=Query(30,ge=1,le=200)):
|
||||
allowed = layers.split(",") if layers else None
|
||||
driver = _get_driver()
|
||||
seen, results = set(), []
|
||||
with driver.session() as s:
|
||||
lc = _layer_cond("r", allowed, include_candidate)
|
||||
q = f"MATCH (s:Entity)-[r:RELATION]->(t:Entity) WHERE toLower(s.name) CONTAINS toLower($n) AND {lc} RETURN s,r,t ORDER BY r.confidence DESC LIMIT $lim"
|
||||
for rec in s.run(q, n=name, lim=top_k):
|
||||
k = (rec["s"].get("name"),rec["r"].get("relation"),rec["t"].get("name"),rec["r"].get("paper_id",""))
|
||||
if k not in seen: seen.add(k); results.append(_record_to_dict(rec))
|
||||
qr = f"MATCH (t:Entity)<-[r:RELATION]-(s:Entity) WHERE toLower(t.name) CONTAINS toLower($n) AND {lc} RETURN s,r,t ORDER BY r.confidence DESC LIMIT $lim"
|
||||
for rec in s.run(qr, n=name, lim=top_k):
|
||||
k = (rec["s"].get("name"),rec["r"].get("relation"),rec["t"].get("name"),rec["r"].get("paper_id",""))
|
||||
if k not in seen: seen.add(k); results.append(_record_to_dict(rec))
|
||||
if depth >= 2:
|
||||
nbrs = set(r["source"] for r in results) | set(r["target"] for r in results)
|
||||
nbrs.discard(name)
|
||||
for nb in list(nbrs)[:20]:
|
||||
lc2 = _layer_cond("r2", allowed, include_candidate)
|
||||
q2 = f"MATCH (s2:Entity)-[r2:RELATION]->(t2:Entity) WHERE toLower(s2.name)=toLower($nb) AND {lc2} RETURN s2,r2,t2 ORDER BY r2.confidence DESC LIMIT 10"
|
||||
for rec in s.run(q2, nb=nb):
|
||||
k = (rec["s2"].get("name"),rec["r2"].get("relation"),rec["t2"].get("name"),rec["r2"].get("paper_id",""))
|
||||
if k not in seen: seen.add(k); results.append(_record_to_dict(rec))
|
||||
return {"entity":name, "depth":depth, "relations_count":len(results), "relations":results}
|
||||
|
||||
@router.post("/path")
|
||||
async def path(req: PReq):
|
||||
driver = _get_driver(); paths = []
|
||||
with driver.session() as s:
|
||||
q = f"MATCH p=(s:Entity)-[r:RELATION*1..{req.max_hops}]-(t:Entity) WHERE toLower(s.name) CONTAINS toLower($src) AND toLower(t.name) CONTAINS toLower($tgt) AND s<>t RETURN p ORDER BY length(p) ASC LIMIT $lim"
|
||||
for rec in s.run(q, src=req.source_entity, tgt=req.target_entity, lim=req.top_k_paths):
|
||||
es = []
|
||||
for rel in rec["p"].relationships:
|
||||
es.append({"source":rel.nodes[0].get("name",""),
|
||||
"source_type":rel.nodes[0].get("entity_type",""),
|
||||
"target":rel.nodes[-1].get("name",""),
|
||||
"target_type":rel.nodes[-1].get("entity_type",""),
|
||||
"relation":rel.get("relation",""),
|
||||
"evidence":str(rel.get("evidence","")or""),
|
||||
"section":str(rel.get("section","")or""),
|
||||
"pubmed_id":str(rel.get("pubmed_id","")or""),
|
||||
"paper_id":rel.get("paper_id",""),
|
||||
"confidence":rel.get("confidence"),
|
||||
"claim_scope":str(rel.get("claim_scope","")or""),
|
||||
"output_layer":str(rel.get("output_layer","")or"")})
|
||||
paths.append({"path":es, "hops":len(es)})
|
||||
return {"source_entity":req.source_entity, "target_entity":req.target_entity,
|
||||
"max_hops":req.max_hops, "paths_found":len(paths), "paths":paths[:req.top_k_paths]}
|
||||
|
||||
@router.post("/chat")
|
||||
async def chat(req: GQAReq):
|
||||
driver = _get_driver()
|
||||
candidates = _re.findall(r"[A-Z][a-zA-Z0-9\-_]+", req.question)
|
||||
entities = [c for c in candidates if c not in SKIP_WORDS and len(c) >= 3]
|
||||
fts = []
|
||||
try:
|
||||
with driver.session() as sf:
|
||||
for rec in sf.run("CALL db.index.fulltext.queryNodes('entity_name_ftx',$q) YIELD node,score RETURN node.name AS n,node.entity_type AS t,score ORDER BY score DESC LIMIT 10", q=req.question[:100]):
|
||||
fts.append({"name":rec["n"],"type":rec["t"],"score":rec["score"]})
|
||||
except: pass
|
||||
allowed = req.layers; lc = _layer_cond("r", allowed, req.include_candidate)
|
||||
all_edges, seen = [], set()
|
||||
with driver.session() as s:
|
||||
terms = [e["name"] for e in fts[:5]] or entities[:5]
|
||||
for term in terms:
|
||||
q = f"MATCH (s:Entity)-[r:RELATION]->(t:Entity) WHERE (toLower(s.name) CONTAINS toLower($t) OR toLower(t.name) CONTAINS toLower($t)) AND {lc} RETURN s,r,t ORDER BY r.confidence DESC LIMIT $lim"
|
||||
for rec in s.run(q, t=term, lim=req.top_k):
|
||||
k = (rec["s"].get("name"),rec["r"].get("relation"),rec["t"].get("name"),rec["r"].get("paper_id",""))
|
||||
if k not in seen: seen.add(k); all_edges.append(_record_to_dict(rec))
|
||||
deduped, sd = [], set()
|
||||
for e in all_edges:
|
||||
ek = (e["source"],e["relation"],e["target"],e["paper_id"])
|
||||
if ek not in sd: sd.add(ek); deduped.append(e)
|
||||
deduped.sort(key=lambda x: LAYER_PRIORITY.get(x.get("output_layer",""),5))
|
||||
answer, model_used = "", ""
|
||||
if req.use_llm and deduped:
|
||||
try:
|
||||
from api.kb_qa_routes import _generate_llm_answer
|
||||
rows = [{"source":e["source"],"source_type":e.get("source_type",""),"relation":e["relation"],"target":e["target"],"target_type":e.get("target_type",""),"evidence":e["evidence"],"section":e["section"],"pubmed_id":e["pubmed_id"],"paper_id":e["paper_id"],"output_layer":e.get("output_layer",""),"claim_scope":e.get("claim_scope",""),"chunk_id":""} for e in deduped[:25]]
|
||||
prov = req.provider or os.environ.get("V15_KB_QA_LLM_PROVIDER")
|
||||
mdl = req.model or os.environ.get("V15_KB_QA_LLM_MODEL")
|
||||
answer = await _generate_llm_answer(req.question, "graph", rows, provider=prov, model=mdl)
|
||||
model_used = f"{prov}:{mdl}"
|
||||
except Exception as ex:
|
||||
answer = f"[Graph QA] Evidence retrieved ({len(deduped)} edges) but LLM failed: {ex}"
|
||||
return {"question":req.question, "entities_candidates":entities, "fts_entities":fts,
|
||||
"graph_edges_count":len(deduped), "graph_edges":deduped[:30],
|
||||
"answer":answer, "llm_model":model_used, "mode":"v15_graph_qa"}
|
||||
|
||||
@router.get("/entity/{name}")
|
||||
async def entity_info(name: str):
|
||||
driver = _get_driver()
|
||||
with driver.session() as s:
|
||||
rec = s.run("MATCH (e:Entity) WHERE toLower(e.name)=toLower($n) RETURN e", n=name).single()
|
||||
if not rec:
|
||||
rec = s.run("MATCH (e:Entity) WHERE toLower(e.name) CONTAINS toLower($n) RETURN e LIMIT 1", n=name).single()
|
||||
if not rec:
|
||||
raise HTTPException(404, "Entity not found")
|
||||
e = rec["e"]
|
||||
return {"name": e.get("name",""), "entity_type": e.get("entity_type","")}
|
||||
|
||||
print("[V15 Graph QA] Routes loaded")
|
||||
Executable
+78
@@ -0,0 +1,78 @@
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from fastapi import WebSocket
|
||||
|
||||
|
||||
class ConnectionManager:
|
||||
def __init__(self):
|
||||
self._connections: dict[str, list[WebSocket]] = {}
|
||||
|
||||
async def connect(self, websocket: WebSocket, session_id: str) -> None:
|
||||
await websocket.accept()
|
||||
if session_id not in self._connections:
|
||||
self._connections[session_id] = []
|
||||
self._connections[session_id].append(websocket)
|
||||
|
||||
def disconnect(self, websocket: WebSocket, session_id: str) -> None:
|
||||
if session_id in self._connections:
|
||||
self._connections[session_id].remove(websocket)
|
||||
if not self._connections[session_id]:
|
||||
del self._connections[session_id]
|
||||
|
||||
async def send_message(self, session_id: str, message: dict[str, Any]) -> None:
|
||||
if session_id in self._connections:
|
||||
for connection in self._connections[session_id]:
|
||||
await connection.send_json(message)
|
||||
|
||||
async def broadcast(self, message: dict[str, Any]) -> None:
|
||||
for connections in self._connections.values():
|
||||
for connection in connections:
|
||||
await connection.send_json(message)
|
||||
|
||||
|
||||
manager = ConnectionManager()
|
||||
|
||||
|
||||
async def send_progress(
|
||||
session_id: str,
|
||||
current_agent: str | None,
|
||||
status: str,
|
||||
progress: float,
|
||||
message: str,
|
||||
) -> None:
|
||||
await manager.send_message(
|
||||
session_id,
|
||||
{
|
||||
"type": "progress",
|
||||
"session_id": session_id,
|
||||
"data": {
|
||||
"current_agent": current_agent,
|
||||
"status": status,
|
||||
"progress": progress,
|
||||
"message": message,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def send_result(session_id: str, result: dict[str, Any]) -> None:
|
||||
await manager.send_message(
|
||||
session_id,
|
||||
{
|
||||
"type": "result",
|
||||
"session_id": session_id,
|
||||
"data": result,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def send_error(session_id: str, error: str) -> None:
|
||||
await manager.send_message(
|
||||
session_id,
|
||||
{
|
||||
"type": "error",
|
||||
"session_id": session_id,
|
||||
"data": {"error": error},
|
||||
},
|
||||
)
|
||||
Reference in New Issue
Block a user