999 lines
38 KiB
Python
Executable File
999 lines
38 KiB
Python
Executable File
"""Methodology extraction batch result routes."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import csv
|
||
import json
|
||
import os
|
||
import re
|
||
import shlex
|
||
import subprocess
|
||
import uuid
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
from fastapi import APIRouter, BackgroundTasks, File, Form, HTTPException, UploadFile
|
||
from pydantic import BaseModel, Field
|
||
|
||
from src.rag.unified.bbav5_methodology_config import build_bbav5_methodology_config, build_methodology_execution_plan, ensure_methodology_execution_contract, resolve_batch_dir
|
||
from src.rag.unified.figure_table_planner import build_figure_table_outputs
|
||
from src.rag.unified.methodology_qa_sandbox import DEFAULT_SANDBOX_ROOT, answer_from_sandbox, build_methodology_qa_sandbox, sanitize_token
|
||
from src.rag.project_config import PROJECT_ROOT, conda_env_path
|
||
|
||
|
||
# Use LLM extraction results by default (can be overridden via env var)
|
||
BATCH_DIR_NAME = os.environ.get("METHODOLOGY_BATCH_DIR", "methodology_batch_llm")
|
||
BATCH_DIR = PROJECT_ROOT / "data" / "evaluation" / BATCH_DIR_NAME
|
||
PAPERS_CSV = PROJECT_ROOT / "data" / "evaluation" / "methodology_papers.csv"
|
||
GOLD_CSV = PROJECT_ROOT / "data" / "evaluation" / "methodology_gold.csv"
|
||
REVIEW_RESULTS_JSON = BATCH_DIR / "review_results.json"
|
||
REVIEW_RESULTS_CSV = BATCH_DIR / "review_results.csv"
|
||
UPLOAD_JOBS_DIR = PROJECT_ROOT / "data" / "evaluation" / "methodology_upload_jobs"
|
||
UPLOAD_OUTPUT_ROOT = PROJECT_ROOT / "data" / "evaluation" / "0505"
|
||
DEFAULT_MINERU_CMD = str(conda_env_path("minerupy311") / "bin" / "mineru")
|
||
DEFAULT_MINERU_API_URL = "http://127.0.0.1:50215"
|
||
DEFAULT_MINERU_BACKEND = "hybrid-auto-engine"
|
||
DEFAULT_MINERU_METHOD = "auto"
|
||
DEFAULT_MINERU_LANG = "en"
|
||
DEFAULT_MINERU_MIN_FREE_GPU_MB = 6144
|
||
DEFAULT_MINERU_LOW_MEM_MIN_FREE_GPU_MB = 4096
|
||
DEFAULT_MINERU_RUNTIME_MODE = "single"
|
||
|
||
router = APIRouter(prefix="/methodology", tags=["methodology"])
|
||
|
||
|
||
class ReviewResult(BaseModel):
|
||
observation_id: str = Field(..., min_length=1)
|
||
paper_id: str = ""
|
||
review_status: str = ""
|
||
stage_review: str = ""
|
||
tools_review: str = ""
|
||
versions_review: str = ""
|
||
parameters_review: str = ""
|
||
inputs_outputs_review: str = ""
|
||
evidence_review: str = ""
|
||
bbav5_review: str = ""
|
||
bbav5_stage_review: str = ""
|
||
bbav5_tools_review: str = ""
|
||
support_status_review: str = ""
|
||
corrected_stage_value: str = ""
|
||
corrected_tools_value: str = ""
|
||
corrected_versions_value: str = ""
|
||
corrected_parameters_value: str = ""
|
||
corrected_evidence_value: str = ""
|
||
corrected_inputs_value: str = ""
|
||
corrected_outputs_value: str = ""
|
||
corrected_bbav5_stage_value: str = ""
|
||
corrected_bbav5_tools_value: str = ""
|
||
corrected_support_status_value: str = ""
|
||
manual_note: str = ""
|
||
|
||
|
||
class ReviewResultsPayload(BaseModel):
|
||
results: list[ReviewResult]
|
||
|
||
|
||
class MethodologySandboxBuildPayload(BaseModel):
|
||
methodology_dir: str = Field(..., min_length=1)
|
||
sandbox_root: str = str(DEFAULT_SANDBOX_ROOT)
|
||
run_id: str = ""
|
||
include_figures: bool = True
|
||
content_list_path: str = ""
|
||
|
||
|
||
class MethodologySandboxAnswerPayload(BaseModel):
|
||
question: str = Field(..., min_length=1)
|
||
registry_path: str = ""
|
||
sandbox_dir: str = ""
|
||
|
||
|
||
REVIEW_RESULT_FIELDS = [
|
||
"observation_id",
|
||
"paper_id",
|
||
"review_status",
|
||
"stage_review",
|
||
"tools_review",
|
||
"versions_review",
|
||
"parameters_review",
|
||
"inputs_outputs_review",
|
||
"evidence_review",
|
||
"bbav5_review",
|
||
"bbav5_stage_review",
|
||
"bbav5_tools_review",
|
||
"support_status_review",
|
||
"corrected_stage_value",
|
||
"corrected_tools_value",
|
||
"corrected_versions_value",
|
||
"corrected_parameters_value",
|
||
"corrected_evidence_value",
|
||
"corrected_inputs_value",
|
||
"corrected_outputs_value",
|
||
"corrected_bbav5_stage_value",
|
||
"corrected_bbav5_tools_value",
|
||
"corrected_support_status_value",
|
||
"manual_note",
|
||
]
|
||
|
||
|
||
def read_csv(path: Path) -> list[dict[str, str]]:
|
||
if not path.exists():
|
||
return []
|
||
with path.open("r", encoding="utf-8-sig", newline="") as handle:
|
||
return list(csv.DictReader(handle))
|
||
|
||
|
||
def read_json(path: Path) -> Any:
|
||
if not path.exists():
|
||
raise HTTPException(status_code=404, detail=f"Missing file: {path}")
|
||
return json.loads(path.read_text(encoding="utf-8"))
|
||
|
||
|
||
def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
with path.open("w", encoding="utf-8", newline="") as handle:
|
||
writer = csv.DictWriter(handle, fieldnames=REVIEW_RESULT_FIELDS)
|
||
writer.writeheader()
|
||
for row in rows:
|
||
writer.writerow({field: row.get(field, "") for field in REVIEW_RESULT_FIELDS})
|
||
|
||
|
||
def read_review_results() -> list[dict[str, Any]]:
|
||
if not REVIEW_RESULTS_JSON.exists():
|
||
return []
|
||
data = json.loads(REVIEW_RESULTS_JSON.read_text(encoding="utf-8"))
|
||
if isinstance(data, dict):
|
||
return list(data.get("results", []))
|
||
return list(data)
|
||
|
||
|
||
def write_job_status(job_dir: Path, payload: dict[str, Any]) -> None:
|
||
payload = {"updated_at": datetime.now().isoformat(timespec="seconds"), **payload}
|
||
job_dir.mkdir(parents=True, exist_ok=True)
|
||
(job_dir / "status.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
|
||
|
||
def parse_int_env(name: str, default: int) -> int:
|
||
value = os.environ.get(name, "").strip()
|
||
if not value:
|
||
return default
|
||
try:
|
||
parsed = int(value)
|
||
except ValueError:
|
||
return default
|
||
return max(0, parsed)
|
||
|
||
|
||
def parse_optional_int_env(name: str) -> int | None:
|
||
value = os.environ.get(name, "").strip()
|
||
if not value:
|
||
return None
|
||
try:
|
||
return max(0, int(value))
|
||
except ValueError:
|
||
return None
|
||
|
||
|
||
def env_bool(name: str) -> bool:
|
||
return os.environ.get(name, "").strip().lower() in {"1", "true", "yes", "on"}
|
||
|
||
|
||
def env_bool_text(name: str) -> str:
|
||
value = os.environ.get(name, "").strip().lower()
|
||
if value in {"1", "true", "yes", "on"}:
|
||
return "true"
|
||
if value in {"0", "false", "no", "off"}:
|
||
return "false"
|
||
return ""
|
||
|
||
|
||
def normalize_mineru_runtime_mode(value: str) -> str:
|
||
mode = str(value or "").strip().lower()
|
||
return mode if mode in {"single", "batch"} else DEFAULT_MINERU_RUNTIME_MODE
|
||
|
||
|
||
def mineru_default_min_free_gpu_mb(runtime: dict[str, str]) -> int:
|
||
mode = normalize_mineru_runtime_mode(runtime.get("mode", ""))
|
||
backend = str(runtime.get("backend") or "").strip().lower()
|
||
method = str(runtime.get("method") or "").strip().lower()
|
||
formula = str(runtime.get("formula") or "").strip().lower()
|
||
table = str(runtime.get("table") or "").strip().lower()
|
||
low_mem_pipeline = (
|
||
mode == "single"
|
||
and backend == "pipeline"
|
||
and method == "txt"
|
||
and formula == "false"
|
||
and table == "false"
|
||
)
|
||
if low_mem_pipeline:
|
||
return DEFAULT_MINERU_LOW_MEM_MIN_FREE_GPU_MB
|
||
return DEFAULT_MINERU_MIN_FREE_GPU_MB
|
||
|
||
|
||
def parse_gpu_memory_csv(text: str) -> list[dict[str, int]]:
|
||
rows: list[dict[str, int]] = []
|
||
for line in text.splitlines():
|
||
parts = [part.strip() for part in line.split(",")]
|
||
if len(parts) < 4:
|
||
continue
|
||
try:
|
||
rows.append(
|
||
{
|
||
"index": int(parts[0]),
|
||
"total_mb": int(parts[1]),
|
||
"used_mb": int(parts[2]),
|
||
"free_mb": int(parts[3]),
|
||
}
|
||
)
|
||
except ValueError:
|
||
continue
|
||
return rows
|
||
|
||
|
||
def parse_gpu_process_csv(text: str) -> list[dict[str, Any]]:
|
||
rows: list[dict[str, Any]] = []
|
||
for line in text.splitlines():
|
||
parts = [part.strip() for part in line.split(",", 2)]
|
||
if len(parts) < 3:
|
||
continue
|
||
try:
|
||
rows.append({"pid": int(parts[0]), "name": parts[1], "used_mb": int(parts[2])})
|
||
except ValueError:
|
||
continue
|
||
return rows
|
||
|
||
|
||
def cuda_visible_gpu_indices(cuda_visible_devices: str) -> list[int]:
|
||
indices: list[int] = []
|
||
for token in cuda_visible_devices.split(","):
|
||
token = token.strip()
|
||
if token.isdigit():
|
||
indices.append(int(token))
|
||
return indices
|
||
|
||
|
||
def format_gpu_processes(processes: list[dict[str, Any]], limit: int = 8) -> str:
|
||
if not processes:
|
||
return "未检测到 GPU 计算进程"
|
||
ordered = sorted(processes, key=lambda row: int(row.get("used_mb") or 0), reverse=True)
|
||
formatted = []
|
||
for row in ordered[:limit]:
|
||
name = Path(str(row.get("name") or "")).name or str(row.get("name") or "")
|
||
formatted.append(f"PID {row.get('pid')} {name} {row.get('used_mb')}MiB")
|
||
if len(ordered) > limit:
|
||
formatted.append(f"... 另有 {len(ordered) - limit} 个进程")
|
||
return "; ".join(formatted)
|
||
|
||
|
||
def build_mineru_gpu_preflight_error(
|
||
gpu_rows: list[dict[str, int]],
|
||
process_rows: list[dict[str, Any]],
|
||
cuda_visible_devices: str,
|
||
min_free_mb: int,
|
||
runtime_mode: str = DEFAULT_MINERU_RUNTIME_MODE,
|
||
) -> str:
|
||
if min_free_mb <= 0:
|
||
return ""
|
||
visible_indices = cuda_visible_gpu_indices(cuda_visible_devices)
|
||
candidate_rows = [row for row in gpu_rows if row["index"] in visible_indices] if visible_indices else gpu_rows
|
||
if not candidate_rows:
|
||
return (
|
||
"MinerU GPU 预检失败：无法从 nvidia-smi 读取可用 GPU。"
|
||
f"CUDA_VISIBLE_DEVICES={cuda_visible_devices or '<empty>'}"
|
||
)
|
||
best = max(candidate_rows, key=lambda row: row["free_mb"])
|
||
if best["free_mb"] >= min_free_mb:
|
||
return ""
|
||
processes = format_gpu_processes(process_rows)
|
||
mode = normalize_mineru_runtime_mode(runtime_mode)
|
||
if mode == "single":
|
||
return (
|
||
"MinerU 按需启动需要 GPU 显存，但当前 GPU 显存不足，不能启动新的 PDF 解析任务。"
|
||
f"GPU{best['index']} free={best['free_mb']}MiB / total={best['total_mb']}MiB，"
|
||
f"需要至少 {min_free_mb}MiB 空闲显存。"
|
||
f"当前 GPU 进程：{processes}。"
|
||
"请先释放占用显存的 QA/VLLM 进程，或改用低显存 MinerU 配置后重试。"
|
||
)
|
||
return (
|
||
"MinerU 服务已热加载，但当前 GPU 显存余量不足，不能安全提交新的 PDF 解析任务。"
|
||
f"GPU{best['index']} free={best['free_mb']}MiB / total={best['total_mb']}MiB，"
|
||
f"需要至少 {min_free_mb}MiB 空闲显存。"
|
||
f"当前 GPU 进程：{processes}。"
|
||
"请先释放占用显存的 QA/VLLM/MinerU 进程，或调低 METHODOLOGY_MINERU_MIN_FREE_GPU_MB 后重试。"
|
||
)
|
||
|
||
|
||
def mineru_gpu_preflight_error(
|
||
cuda_visible_devices: str,
|
||
runtime_mode: str = DEFAULT_MINERU_RUNTIME_MODE,
|
||
runtime: dict[str, str] | None = None,
|
||
) -> str:
|
||
if os.environ.get("METHODOLOGY_MINERU_GPU_PREFLIGHT", "1").strip().lower() in {"0", "false", "no", "off"}:
|
||
return ""
|
||
runtime_payload = runtime or {"mode": runtime_mode}
|
||
min_free_mb = parse_optional_int_env("METHODOLOGY_MINERU_MIN_FREE_GPU_MB")
|
||
if min_free_mb is None:
|
||
min_free_mb = mineru_default_min_free_gpu_mb(runtime_payload)
|
||
if min_free_mb <= 0:
|
||
return ""
|
||
try:
|
||
memory = subprocess.run(
|
||
["nvidia-smi", "--query-gpu=index,memory.total,memory.used,memory.free", "--format=csv,noheader,nounits"],
|
||
cwd=PROJECT_ROOT,
|
||
text=True,
|
||
capture_output=True,
|
||
timeout=5,
|
||
)
|
||
except Exception as exc:
|
||
return f"MinerU GPU 预检失败：无法执行 nvidia-smi：{exc}"
|
||
if memory.returncode != 0:
|
||
return f"MinerU GPU 预检失败：nvidia-smi 返回 {memory.returncode}：{memory.stderr.strip() or memory.stdout.strip()}"
|
||
processes_text = ""
|
||
try:
|
||
processes = subprocess.run(
|
||
["nvidia-smi", "--query-compute-apps=pid,process_name,used_memory", "--format=csv,noheader,nounits"],
|
||
cwd=PROJECT_ROOT,
|
||
text=True,
|
||
capture_output=True,
|
||
timeout=5,
|
||
)
|
||
if processes.returncode == 0:
|
||
processes_text = processes.stdout
|
||
except Exception:
|
||
processes_text = ""
|
||
return build_mineru_gpu_preflight_error(
|
||
parse_gpu_memory_csv(memory.stdout),
|
||
parse_gpu_process_csv(processes_text),
|
||
cuda_visible_devices,
|
||
min_free_mb,
|
||
runtime_payload.get("mode", runtime_mode),
|
||
)
|
||
|
||
|
||
def summarize_mineru_failure(log_path: Path, returncode: int) -> str:
|
||
message = f"MinerU failed with exit code {returncode}; see {log_path}"
|
||
if not log_path.exists():
|
||
return message
|
||
try:
|
||
text = log_path.read_text(encoding="utf-8", errors="replace")
|
||
except Exception:
|
||
return message
|
||
if re.search(r"CUDA out of memory|OutOfMemoryError|cuda memory", text, re.IGNORECASE):
|
||
return (
|
||
"MinerU 解析失败：CUDA 显存不足。"
|
||
"这通常表示 GPU 已被 QA/VLLM/MinerU 进程占用，运行 PDF 检测/解析时没有足够额外显存。"
|
||
f"日志：{log_path}"
|
||
)
|
||
return message
|
||
|
||
|
||
def build_mineru_runtime() -> tuple[list[str], dict[str, str], dict[str, str]]:
|
||
env = os.environ.copy()
|
||
env.setdefault("MINERU_MODEL_SOURCE", "modelscope")
|
||
env.setdefault("CUDA_VISIBLE_DEVICES", "0")
|
||
env.setdefault("TORCH_CUDNN_V8_API_DISABLED", "1")
|
||
|
||
mineru_cmd = shlex.split(os.environ.get("MINERU_CMD", DEFAULT_MINERU_CMD))
|
||
runtime_mode = normalize_mineru_runtime_mode(
|
||
os.environ.get("METHODOLOGY_MINERU_RUNTIME_MODE", DEFAULT_MINERU_RUNTIME_MODE)
|
||
)
|
||
runtime = {
|
||
"mode": runtime_mode,
|
||
"model_source": env.get("MINERU_MODEL_SOURCE", ""),
|
||
"cuda_visible_devices": env.get("CUDA_VISIBLE_DEVICES", ""),
|
||
"api_url": os.environ.get("V15_MINERU_API_URL", DEFAULT_MINERU_API_URL).strip(),
|
||
"use_api": "1" if runtime_mode == "batch" and env_bool("V15_MINERU_USE_API") else "0",
|
||
"backend": os.environ.get("V15_MINERU_BACKEND", DEFAULT_MINERU_BACKEND).strip() or DEFAULT_MINERU_BACKEND,
|
||
"method": os.environ.get("V15_MINERU_METHOD", DEFAULT_MINERU_METHOD).strip() or DEFAULT_MINERU_METHOD,
|
||
"lang": os.environ.get("V15_MINERU_LANG", DEFAULT_MINERU_LANG).strip() or DEFAULT_MINERU_LANG,
|
||
"formula": env_bool_text("V15_MINERU_FORMULA"),
|
||
"table": env_bool_text("V15_MINERU_TABLE"),
|
||
}
|
||
return mineru_cmd, env, runtime
|
||
|
||
|
||
def build_mineru_command(mineru_cmd: list[str], source_path: Path, mineru_out: Path, runtime: dict[str, str]) -> list[str]:
|
||
cmd = [
|
||
*mineru_cmd,
|
||
"-p",
|
||
str(source_path),
|
||
"-o",
|
||
str(mineru_out),
|
||
"-m",
|
||
runtime["method"],
|
||
"-b",
|
||
runtime["backend"],
|
||
"-l",
|
||
runtime["lang"],
|
||
]
|
||
if runtime.get("formula"):
|
||
cmd.extend(["-f", runtime["formula"]])
|
||
if runtime.get("table"):
|
||
cmd.extend(["-t", runtime["table"]])
|
||
if runtime.get("mode") == "batch" and runtime.get("use_api") == "1" and runtime.get("api_url"):
|
||
cmd.extend(["--api-url", runtime["api_url"]])
|
||
return cmd
|
||
|
||
|
||
def read_job_status(job_id: str) -> dict[str, Any]:
|
||
status_path = UPLOAD_JOBS_DIR / job_id / "status.json"
|
||
if not status_path.exists():
|
||
raise HTTPException(status_code=404, detail=f"Unknown methodology extraction job: {job_id}")
|
||
return json.loads(status_path.read_text(encoding="utf-8"))
|
||
|
||
|
||
def list_upload_job_statuses(limit: int = 50) -> list[dict[str, Any]]:
|
||
rows: list[dict[str, Any]] = []
|
||
if not UPLOAD_JOBS_DIR.exists():
|
||
return rows
|
||
for status_path in UPLOAD_JOBS_DIR.glob("*/status.json"):
|
||
try:
|
||
payload = json.loads(status_path.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
continue
|
||
output_dir = str(payload.get("output_dir") or "")
|
||
source_path = str(payload.get("source_path") or "")
|
||
row = {
|
||
"job_id": payload.get("job_id") or status_path.parent.name,
|
||
"status": payload.get("status", ""),
|
||
"stage": payload.get("stage", ""),
|
||
"substage": payload.get("substage", ""),
|
||
"message": payload.get("message", ""),
|
||
"updated_at": payload.get("updated_at", ""),
|
||
"paper_id": payload.get("paper_id", ""),
|
||
"family": payload.get("family", ""),
|
||
"source_filename": Path(source_path).name if source_path else "",
|
||
"source_path": source_path,
|
||
"output_dir": output_dir,
|
||
"batch_dir": output_dir,
|
||
"stage_count": payload.get("stage_count", 0),
|
||
"species_count": payload.get("species_count", 0),
|
||
"figure_table_caption_count": payload.get("figure_table_caption_count", 0),
|
||
"figure_table_stage_count": payload.get("figure_table_stage_count", 0),
|
||
"config_url": payload.get("config_url", f"/api/v1/methodology/extract-jobs/{status_path.parent.name}/bbav5-config"),
|
||
"methodology_plan_url": payload.get("methodology_plan_url", f"/api/v1/methodology/extract-jobs/{status_path.parent.name}/plan"),
|
||
"methodology_sandbox_requested": payload.get("methodology_sandbox_requested", False),
|
||
"methodology_sandbox_status": payload.get("methodology_sandbox_status", ""),
|
||
"methodology_sandbox_dir": payload.get("methodology_sandbox_dir", ""),
|
||
"methodology_sandbox_registry": payload.get("methodology_sandbox_registry", ""),
|
||
"methodology_sandbox_collection": payload.get("methodology_sandbox_collection", ""),
|
||
}
|
||
rows.append(row)
|
||
rows.sort(key=lambda row: str(row.get("updated_at") or ""), reverse=True)
|
||
return rows[: max(1, min(int(limit or 50), 200))]
|
||
|
||
|
||
def run_uploaded_methodology_job(
|
||
job_id: str,
|
||
source_path: Path,
|
||
paper_id: str,
|
||
family: str,
|
||
output_dir: Path,
|
||
llm_model: str,
|
||
llm_request_timeout: int,
|
||
sandbox: bool = False,
|
||
sandbox_run_id: str = "",
|
||
) -> None:
|
||
job_dir = UPLOAD_JOBS_DIR / job_id
|
||
log_path = job_dir / "job.log"
|
||
try:
|
||
write_job_status(
|
||
job_dir,
|
||
{
|
||
"job_id": job_id,
|
||
"status": "running",
|
||
"stage": "mineru",
|
||
"message": "MinerU 正在解析上传文献",
|
||
"source_path": str(source_path),
|
||
"output_dir": str(output_dir),
|
||
},
|
||
)
|
||
content_list = source_path
|
||
if source_path.suffix.lower() != ".json":
|
||
mineru_out = job_dir / "mineru"
|
||
mineru_cmd, mineru_env, mineru_runtime = build_mineru_runtime()
|
||
preflight_error = mineru_gpu_preflight_error(
|
||
mineru_runtime["cuda_visible_devices"],
|
||
mineru_runtime["mode"],
|
||
mineru_runtime,
|
||
)
|
||
if preflight_error:
|
||
with log_path.open("a", encoding="utf-8") as handle:
|
||
handle.write(preflight_error + "\n")
|
||
raise RuntimeError(preflight_error)
|
||
cmd = build_mineru_command(mineru_cmd, source_path, mineru_out, mineru_runtime)
|
||
with log_path.open("a", encoding="utf-8") as handle:
|
||
handle.write(
|
||
"$ env "
|
||
f"METHODOLOGY_MINERU_RUNTIME_MODE={mineru_runtime['mode']} "
|
||
f"MINERU_MODEL_SOURCE={mineru_runtime['model_source']} "
|
||
f"CUDA_VISIBLE_DEVICES={mineru_runtime['cuda_visible_devices']} "
|
||
+ " ".join(cmd)
|
||
+ "\n"
|
||
)
|
||
handle.write("$ " + " ".join(cmd) + "\n")
|
||
handle.flush()
|
||
result = subprocess.run(
|
||
cmd,
|
||
cwd=PROJECT_ROOT,
|
||
env=mineru_env,
|
||
text=True,
|
||
stdout=handle,
|
||
stderr=subprocess.STDOUT,
|
||
)
|
||
if result.returncode != 0:
|
||
raise RuntimeError(summarize_mineru_failure(log_path, result.returncode))
|
||
matches = sorted(mineru_out.rglob("*_content_list.json"))
|
||
if not matches:
|
||
raise RuntimeError(f"MinerU finished but no *_content_list.json was found under {mineru_out}")
|
||
content_list = matches[0]
|
||
|
||
write_job_status(
|
||
job_dir,
|
||
{
|
||
"job_id": job_id,
|
||
"status": "running",
|
||
"stage": "llm",
|
||
"message": "LLM 正在提取方法学并与 bbav5 对齐",
|
||
"source_path": str(source_path),
|
||
"content_list": str(content_list),
|
||
"output_dir": str(output_dir),
|
||
},
|
||
)
|
||
papers_csv = job_dir / "methodology_papers.csv"
|
||
with papers_csv.open("w", encoding="utf-8", newline="") as handle:
|
||
writer = csv.DictWriter(handle, fieldnames=["paper_id", "family", "source_paper_role", "parsed_document_path"])
|
||
writer.writeheader()
|
||
writer.writerow(
|
||
{
|
||
"paper_id": paper_id,
|
||
"family": family,
|
||
"source_paper_role": "primary_gene_family_paper",
|
||
"parsed_document_path": str(content_list),
|
||
}
|
||
)
|
||
batch_cmd = [
|
||
str(conda_env_path(os.environ.get("GF_PYTHON_ENV", "gfpy312")) / "bin" / "python"),
|
||
"scripts/run_methodology_batch.py",
|
||
"--papers",
|
||
str(papers_csv),
|
||
"--paper-id",
|
||
paper_id,
|
||
"--output-dir",
|
||
str(output_dir),
|
||
"--use-llm",
|
||
"--llm-model",
|
||
llm_model,
|
||
"--llm-request-timeout",
|
||
str(llm_request_timeout),
|
||
"--external-job-status-json",
|
||
str(job_dir / "status.json"),
|
||
]
|
||
with log_path.open("a", encoding="utf-8") as handle:
|
||
handle.write("$ " + " ".join(batch_cmd) + "\n")
|
||
handle.flush()
|
||
result = subprocess.run(batch_cmd, cwd=PROJECT_ROOT, text=True, stdout=handle, stderr=subprocess.STDOUT)
|
||
if result.returncode != 0:
|
||
raise RuntimeError(f"LLM methodology extraction failed with exit code {result.returncode}; see {log_path}")
|
||
config = build_bbav5_methodology_config(output_dir)
|
||
figure_table_error = ""
|
||
figure_table_payload: dict[str, Any] = {}
|
||
try:
|
||
figure_table_payload = build_figure_table_outputs(
|
||
content_list_path=content_list,
|
||
output_dir=output_dir,
|
||
methodology_config=config,
|
||
)
|
||
config = build_bbav5_methodology_config(output_dir)
|
||
except Exception as exc: # keep the core methodology extraction result usable.
|
||
figure_table_error = str(exc)
|
||
preview = config.get("reproduction_plan_preview", {}) if isinstance(config, dict) else {}
|
||
sandbox_payload: dict[str, Any] = {}
|
||
sandbox_warning = ""
|
||
if sandbox:
|
||
try:
|
||
result = build_methodology_qa_sandbox(
|
||
output_dir,
|
||
sandbox_root=DEFAULT_SANDBOX_ROOT,
|
||
run_id=sandbox_run_id or f"upload_{sanitize_token(job_id)}",
|
||
include_figures=False,
|
||
)
|
||
report = result.get("report", {})
|
||
registry = result.get("registry", {})
|
||
sandbox_payload = {
|
||
"methodology_sandbox_dir": report.get("sandbox_dir", ""),
|
||
"methodology_sandbox_registry": report.get("registry_path", ""),
|
||
"methodology_sandbox_collection": registry.get("qdrant_collection", ""),
|
||
"methodology_sandbox_status": "built",
|
||
"methodology_sandbox_card_count": report.get("stage_card_count", 0) + report.get("plan_card_count", 0),
|
||
"methodology_sandbox_relation_count": report.get("relation_count", 0),
|
||
}
|
||
except Exception as exc:
|
||
sandbox_warning = str(exc)
|
||
sandbox_payload = {
|
||
"methodology_sandbox_status": "failed",
|
||
"methodology_sandbox_warning": sandbox_warning,
|
||
}
|
||
write_job_status(
|
||
job_dir,
|
||
{
|
||
"job_id": job_id,
|
||
"status": "completed",
|
||
"stage": "completed",
|
||
"substage": "done",
|
||
"message": "MinerU+LLM 方法学提取完成",
|
||
"source_path": str(source_path),
|
||
"content_list": str(content_list),
|
||
"output_dir": str(output_dir),
|
||
"paper_id": paper_id,
|
||
"family": family,
|
||
"stage_count": len(config.get("stages", [])),
|
||
"species_count": len(config.get("species", [])),
|
||
"figure_table_caption_count": len(config.get("figure_table_inventory", [])),
|
||
"figure_table_stage_count": len(preview.get("suggested_stages", [])) if isinstance(preview, dict) else 0,
|
||
"figure_table_warning": figure_table_error,
|
||
"figure_table_files": figure_table_payload.get("files", {}),
|
||
"config_url": f"/api/v1/methodology/extract-jobs/{job_id}/bbav5-config",
|
||
"methodology_plan_url": f"/api/v1/methodology/extract-jobs/{job_id}/plan",
|
||
"figure_table_inventory_url": f"/api/v1/methodology/extract-jobs/{job_id}/figure-table-inventory",
|
||
"reproduction_plan_preview_url": f"/api/v1/methodology/extract-jobs/{job_id}/reproduction-plan-preview",
|
||
**sandbox_payload,
|
||
},
|
||
)
|
||
except Exception as exc:
|
||
write_job_status(
|
||
job_dir,
|
||
{
|
||
"job_id": job_id,
|
||
"status": "failed",
|
||
"stage": "failed",
|
||
"message": str(exc),
|
||
"source_path": str(source_path),
|
||
"output_dir": str(output_dir),
|
||
"log_path": str(log_path),
|
||
},
|
||
)
|
||
|
||
|
||
@router.get("/status")
|
||
async def methodology_status():
|
||
summary = read_csv(BATCH_DIR / "summary.csv")
|
||
review = read_csv(BATCH_DIR / "review_template.csv")
|
||
return {
|
||
"papers_table": str(PAPERS_CSV),
|
||
"gold_table": str(GOLD_CSV),
|
||
"batch_dir": str(BATCH_DIR),
|
||
"batch_exists": BATCH_DIR.exists(),
|
||
"paper_count": len(read_csv(PAPERS_CSV)),
|
||
"summary_count": len(summary),
|
||
"review_row_count": len(review),
|
||
"files": {
|
||
"summary": str(BATCH_DIR / "summary.csv"),
|
||
"processing_status": str(BATCH_DIR / "processing_status.csv"),
|
||
"review_template": str(BATCH_DIR / "review_template.csv"),
|
||
"review_results": str(REVIEW_RESULTS_JSON),
|
||
"discovered_tools": str(BATCH_DIR / "discovered_tools.csv"),
|
||
"methodology_tool_audit": str(BATCH_DIR / "methodology_tool_audit.csv"),
|
||
"methodology_parameter_audit": str(BATCH_DIR / "methodology_parameter_audit.csv"),
|
||
"methodology_database_url_audit": str(BATCH_DIR / "methodology_database_url_audit.csv"),
|
||
"methodology_review_issues": str(BATCH_DIR / "methodology_review_issues.csv"),
|
||
"methodology_review_report": str(BATCH_DIR / "methodology_review_report.json"),
|
||
"progress": str(BATCH_DIR / "progress.jsonl"),
|
||
"run_log": str(BATCH_DIR / "methodology_batch_run.log"),
|
||
"species_summary": str(BATCH_DIR / "species_summary.csv"),
|
||
"observations": str(BATCH_DIR / "observations.json"),
|
||
"alignment": str(BATCH_DIR / "alignment.json"),
|
||
},
|
||
}
|
||
|
||
|
||
@router.get("/summary")
|
||
async def methodology_summary():
|
||
return {"rows": read_csv(BATCH_DIR / "summary.csv")}
|
||
|
||
|
||
@router.get("/review-template")
|
||
async def methodology_review_template():
|
||
return {"rows": read_csv(BATCH_DIR / "review_template.csv")}
|
||
|
||
|
||
@router.get("/processing-status")
|
||
async def methodology_processing_status():
|
||
return {"rows": read_csv(BATCH_DIR / "processing_status.csv")}
|
||
|
||
|
||
@router.get("/discovered-tools")
|
||
async def methodology_discovered_tools():
|
||
return {"rows": read_csv(BATCH_DIR / "discovered_tools.csv")}
|
||
|
||
|
||
@router.get("/tool-audit")
|
||
async def methodology_tool_audit():
|
||
return {"rows": read_csv(BATCH_DIR / "methodology_tool_audit.csv")}
|
||
|
||
|
||
@router.get("/parameter-audit")
|
||
async def methodology_parameter_audit():
|
||
return {"rows": read_csv(BATCH_DIR / "methodology_parameter_audit.csv")}
|
||
|
||
|
||
@router.get("/url-audit")
|
||
async def methodology_url_audit():
|
||
return {"rows": read_csv(BATCH_DIR / "methodology_database_url_audit.csv")}
|
||
|
||
|
||
@router.get("/auto-review-issues")
|
||
async def methodology_auto_review_issues():
|
||
return {"rows": read_csv(BATCH_DIR / "methodology_review_issues.csv")}
|
||
|
||
|
||
@router.get("/auto-review-report")
|
||
async def methodology_auto_review_report():
|
||
return read_json(BATCH_DIR / "methodology_review_report.json")
|
||
|
||
|
||
@router.get("/progress")
|
||
async def methodology_progress():
|
||
path = BATCH_DIR / "progress.jsonl"
|
||
if not path.exists():
|
||
return {"rows": []}
|
||
rows = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
|
||
return {"rows": rows}
|
||
|
||
|
||
@router.get("/run-log")
|
||
async def methodology_run_log():
|
||
path = BATCH_DIR / "methodology_batch_run.log"
|
||
if not path.exists():
|
||
return {"text": ""}
|
||
return {"text": path.read_text(encoding="utf-8")}
|
||
|
||
|
||
@router.get("/species-summary")
|
||
async def methodology_species_summary():
|
||
return {"rows": read_csv(BATCH_DIR / "species_summary.csv")}
|
||
|
||
|
||
@router.get("/review-items")
|
||
async def methodology_review_items():
|
||
rows = read_csv(BATCH_DIR / "review_template.csv")
|
||
saved = {row.get("observation_id"): row for row in read_review_results()}
|
||
for row in rows:
|
||
if row.get("observation_id") in saved:
|
||
row["saved_review"] = saved[row["observation_id"]]
|
||
return {"rows": rows, "saved_count": len(saved)}
|
||
|
||
|
||
@router.get("/review-results")
|
||
async def methodology_review_results():
|
||
return {"results": read_review_results()}
|
||
|
||
|
||
@router.post("/review-results")
|
||
async def save_methodology_review_results(payload: ReviewResultsPayload):
|
||
existing = {row.get("observation_id"): row for row in read_review_results()}
|
||
for item in payload.results:
|
||
existing[item.observation_id] = item.dict()
|
||
rows = list(existing.values())
|
||
REVIEW_RESULTS_JSON.parent.mkdir(parents=True, exist_ok=True)
|
||
REVIEW_RESULTS_JSON.write_text(json.dumps({"results": rows}, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
write_csv(REVIEW_RESULTS_CSV, rows)
|
||
return {"saved_count": len(rows), "json_path": str(REVIEW_RESULTS_JSON), "csv_path": str(REVIEW_RESULTS_CSV)}
|
||
|
||
|
||
@router.get("/review-summary")
|
||
async def methodology_review_summary():
|
||
rows = read_review_results()
|
||
summary: dict[str, dict[str, int]] = {}
|
||
for field in [
|
||
"review_status",
|
||
"stage_review",
|
||
"tools_review",
|
||
"versions_review",
|
||
"parameters_review",
|
||
"evidence_review",
|
||
"bbav5_stage_review",
|
||
"bbav5_tools_review",
|
||
"support_status_review",
|
||
"bbav5_review",
|
||
]:
|
||
counts: dict[str, int] = {}
|
||
for row in rows:
|
||
value = str(row.get(field) or "<blank>")
|
||
counts[value] = counts.get(value, 0) + 1
|
||
summary[field] = counts
|
||
return {"reviewed_count": len(rows), "summary": summary}
|
||
|
||
|
||
@router.get("/alignment")
|
||
async def methodology_alignment():
|
||
return read_json(BATCH_DIR / "alignment.json")
|
||
|
||
|
||
@router.get("/bbav5-config")
|
||
async def methodology_bbav5_config(batch_dir: str = BATCH_DIR_NAME):
|
||
try:
|
||
return build_bbav5_methodology_config(resolve_batch_dir(batch_dir))
|
||
except FileNotFoundError as exc:
|
||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||
|
||
|
||
@router.post("/extract-upload")
|
||
async def methodology_extract_upload(
|
||
background_tasks: BackgroundTasks,
|
||
file: UploadFile = File(...),
|
||
paper_id: str = Form("uploaded_paper"),
|
||
family: str = Form(""),
|
||
output_name: str = Form(""),
|
||
llm_model: str = Form("deepseek-v4-flash"),
|
||
llm_request_timeout: int = Form(75),
|
||
sandbox: bool = Form(False),
|
||
sandbox_run_id: str = Form(""),
|
||
):
|
||
job_id = uuid.uuid4().hex[:12]
|
||
clean_paper_id = "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in (paper_id or f"paper_{job_id}"))
|
||
output_base = output_name or f"methodology_batch_{clean_paper_id}"
|
||
clean_output_base = "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in output_base).strip("._-") or "methodology_batch"
|
||
clean_output = clean_output_base if clean_output_base.endswith(f"_{job_id}") else f"{clean_output_base}_{job_id}"
|
||
job_dir = UPLOAD_JOBS_DIR / job_id
|
||
upload_dir = job_dir / "upload"
|
||
upload_dir.mkdir(parents=True, exist_ok=True)
|
||
source_path = upload_dir / (file.filename or f"{clean_paper_id}.pdf")
|
||
source_path.write_bytes(await file.read())
|
||
output_dir = UPLOAD_OUTPUT_ROOT / clean_output
|
||
write_job_status(
|
||
job_dir,
|
||
{
|
||
"job_id": job_id,
|
||
"status": "queued",
|
||
"stage": "queued",
|
||
"message": "文献已上传，等待 MinerU+LLM 方法学提取",
|
||
"source_path": str(source_path),
|
||
"output_dir": str(output_dir),
|
||
"paper_id": clean_paper_id,
|
||
"family": family,
|
||
"methodology_sandbox_requested": sandbox,
|
||
},
|
||
)
|
||
background_tasks.add_task(
|
||
run_uploaded_methodology_job,
|
||
job_id,
|
||
source_path,
|
||
clean_paper_id,
|
||
family,
|
||
output_dir,
|
||
llm_model,
|
||
llm_request_timeout,
|
||
sandbox,
|
||
sandbox_run_id,
|
||
)
|
||
return {"success": True, "job_id": job_id, "status_url": f"/api/v1/methodology/extract-jobs/{job_id}"}
|
||
|
||
|
||
@router.get("/extract-jobs")
|
||
async def methodology_extract_jobs(limit: int = 50, status: str = ""):
|
||
rows = list_upload_job_statuses(limit)
|
||
if status:
|
||
rows = [row for row in rows if row.get("status") == status]
|
||
return {"jobs": rows, "count": len(rows)}
|
||
|
||
|
||
@router.get("/extract-jobs/{job_id}")
|
||
async def methodology_extract_job_status(job_id: str):
|
||
return read_job_status(job_id)
|
||
|
||
|
||
@router.get("/extract-jobs/{job_id}/bbav5-config")
|
||
async def methodology_extract_job_config(job_id: str):
|
||
status = read_job_status(job_id)
|
||
if status.get("status") != "completed":
|
||
raise HTTPException(status_code=409, detail=f"Extraction job is not completed: {status.get('status')}")
|
||
try:
|
||
return build_bbav5_methodology_config(status["output_dir"])
|
||
except FileNotFoundError as exc:
|
||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||
|
||
|
||
@router.get("/extract-jobs/{job_id}/plan")
|
||
async def methodology_extract_job_plan(job_id: str):
|
||
status = read_job_status(job_id)
|
||
if status.get("status") != "completed":
|
||
raise HTTPException(status_code=409, detail=f"Extraction job is not completed: {status.get('status')}")
|
||
output_dir = Path(status.get("output_dir", ""))
|
||
plan_path = output_dir / "methodology_plan.json"
|
||
if plan_path.exists():
|
||
plan = ensure_methodology_execution_contract(read_json(plan_path))
|
||
plan_path.write_text(json.dumps(plan, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
return plan
|
||
try:
|
||
plan = build_methodology_execution_plan(output_dir)
|
||
plan_path.write_text(json.dumps(plan, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
return plan
|
||
except FileNotFoundError as exc:
|
||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||
|
||
@router.get("/extract-jobs/{job_id}/figure-table-inventory")
|
||
async def methodology_extract_job_figure_table_inventory(job_id: str):
|
||
status = read_job_status(job_id)
|
||
output_dir = Path(status.get("output_dir", ""))
|
||
return {"rows": read_json(output_dir / "figure_table_inventory.json")}
|
||
|
||
|
||
@router.post("/sandbox/build")
|
||
async def methodology_sandbox_build(payload: MethodologySandboxBuildPayload):
|
||
try:
|
||
result = build_methodology_qa_sandbox(
|
||
payload.methodology_dir,
|
||
sandbox_root=payload.sandbox_root,
|
||
run_id=payload.run_id or None,
|
||
include_figures=payload.include_figures,
|
||
content_list_path=payload.content_list_path or None,
|
||
)
|
||
except FileNotFoundError as exc:
|
||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||
except Exception as exc:
|
||
raise HTTPException(status_code=500, detail=str(exc)) from exc
|
||
return {
|
||
"success": True,
|
||
"report": result["report"],
|
||
"registry": result["registry"],
|
||
}
|
||
|
||
|
||
@router.post("/sandbox/answer")
|
||
async def methodology_sandbox_answer(payload: MethodologySandboxAnswerPayload):
|
||
registry_path = payload.registry_path.strip()
|
||
if not registry_path and payload.sandbox_dir.strip():
|
||
registry_path = str(Path(payload.sandbox_dir.strip()) / "methodology_plan_registry.sandbox.json")
|
||
if not registry_path:
|
||
raise HTTPException(status_code=400, detail="registry_path or sandbox_dir is required")
|
||
result = answer_from_sandbox(payload.question, registry_path)
|
||
if not result:
|
||
return {
|
||
"success": False,
|
||
"answer": "",
|
||
"sources": [],
|
||
"message": "No sandbox methodology plan matched the question.",
|
||
"match_status": "no_match",
|
||
}
|
||
return {
|
||
"success": result.get("success", True),
|
||
"answer": result["answer"],
|
||
"sources": result["sources"],
|
||
"notes": result.get("notes", []),
|
||
"plan_entry": result.get("plan_entry", {}),
|
||
"matched_plan": result.get("matched_plan", {}),
|
||
"match_status": result.get("match_status", ""),
|
||
"query_constraints": result.get("query_constraints", {}),
|
||
"candidate_plans": result.get("candidate_plans", []),
|
||
"mismatch_reasons": result.get("mismatch_reasons", []),
|
||
}
|
||
|
||
|
||
@router.get("/sandbox/example-batches")
|
||
async def methodology_sandbox_example_batches():
|
||
candidates = []
|
||
for path in PROJECT_ROOT.glob("methodology_batch_*"):
|
||
if path.is_dir() and (path / "review_template.csv").exists():
|
||
candidates.append(
|
||
{
|
||
"name": path.name,
|
||
"path": str(path),
|
||
"has_plan": (path / "methodology_plan.json").exists(),
|
||
"has_preview": (path / "reproduction_plan_preview.json").exists(),
|
||
}
|
||
)
|
||
candidates.sort(key=lambda item: item["name"])
|
||
return {
|
||
"success": True,
|
||
"sandbox_root": str(DEFAULT_SANDBOX_ROOT),
|
||
"batches": candidates,
|
||
}
|
||
|
||
|
||
@router.get("/extract-jobs/{job_id}/reproduction-plan-preview")
|
||
async def methodology_extract_job_reproduction_plan_preview(job_id: str):
|
||
status = read_job_status(job_id)
|
||
output_dir = Path(status.get("output_dir", ""))
|
||
return read_json(output_dir / "reproduction_plan_preview.json")
|