Compare commits
19
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
faa8686461 | ||
|
|
8e4fce062a | ||
|
|
04bf099faf | ||
|
|
d54419f040 | ||
|
|
0252d52425 | ||
|
|
2895cd8497 | ||
|
|
aec01ca0df | ||
|
|
1fe80b293e | ||
|
|
8d04eb6845 | ||
|
|
c620461794 | ||
|
|
025e2d516c | ||
|
|
f18286f022 | ||
|
|
28bee31d00 | ||
|
|
00eb0bc1d0 | ||
|
|
b5dc12e8a5 | ||
|
|
60f7360cec | ||
|
|
3889a2a29b | ||
|
|
75877ecab7 | ||
|
|
7840527f82 |
@@ -29,7 +29,7 @@ PAPERS_CSV = PROJECT_ROOT / "data" / "evaluation" / "methodology_papers.csv"
|
||||
GOLD_CSV = PROJECT_ROOT / "data" / "evaluation" / "methodology_gold.csv"
|
||||
REVIEW_RESULTS_JSON = BATCH_DIR / "review_results.json"
|
||||
REVIEW_RESULTS_CSV = BATCH_DIR / "review_results.csv"
|
||||
# 2026-08-31 新工程适配:上传目录/产出根支持 env 覆盖
|
||||
# 2026-08-31 新工程适配:上传目录/产出根支持 env 覆盖(数据根迁移时无需改代码)
|
||||
UPLOAD_JOBS_DIR = Path(os.environ.get("METHODOLOGY_UPLOAD_JOBS_DIR", PROJECT_ROOT / "data" / "evaluation" / "methodology_upload_jobs"))
|
||||
UPLOAD_OUTPUT_ROOT = Path(os.environ.get("METHODOLOGY_UPLOAD_OUTPUT_ROOT", PROJECT_ROOT / "data" / "evaluation" / "0505"))
|
||||
DEFAULT_MINERU_CMD = str(conda_env_path("minerupy312") / "bin" / "mineru")
|
||||
@@ -351,6 +351,22 @@ def mineru_gpu_preflight_error(
|
||||
)
|
||||
|
||||
|
||||
# 缺陷 #38(2026-09-11):成功任务的 job.log 必然有 doc_analyze_streaming 行(176 实测)。
|
||||
# returncode≠0 且日志中无此行 = 启动期失败(pipeline 起来前就死了,真因被块缓冲吞掉)。
|
||||
# 判定条件(methodology 与 legacy-rag 两处 build_mineru_runtime/run_uploaded_methodology_job 同改)
|
||||
MINERU_PIPELINE_LOG_MARKER = "doc_analyze_streaming"
|
||||
MINERU_STARTUP_RETRIES = 1
|
||||
|
||||
|
||||
def job_log_has_pipeline_line(log_path: Path) -> bool:
|
||||
"""job.log 是否已出现 pipeline 日志行(def _job_log_has_pipeline_line)。"""
|
||||
try:
|
||||
text = log_path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
return False
|
||||
return MINERU_PIPELINE_LOG_MARKER in text
|
||||
|
||||
|
||||
def summarize_mineru_failure(log_path: Path, returncode: int) -> str:
|
||||
message = f"MinerU failed with exit code {returncode}; see {log_path}"
|
||||
if not log_path.exists():
|
||||
@@ -368,6 +384,28 @@ def summarize_mineru_failure(log_path: Path, returncode: int) -> str:
|
||||
return message
|
||||
|
||||
|
||||
def summarize_mineru_startup_failure(log_path: Path, returncode: int, retried: bool) -> str:
|
||||
"""启动期失败(无 pipeline 行)的中文可读错误(缺陷 #38)。
|
||||
|
||||
附上 job.log 中可见的原始错误尾部,让用户能看到真因(修复前真因
|
||||
被 stdout 块缓冲 + CLI 立即杀 API 进程永久丢失)。
|
||||
"""
|
||||
tail = ""
|
||||
try:
|
||||
text = log_path.read_text(encoding="utf-8", errors="replace")
|
||||
lines = [ln for ln in text.splitlines() if ln.strip()]
|
||||
if lines:
|
||||
tail = " | ".join(ln.strip() for ln in lines[-3:])[:500]
|
||||
except OSError:
|
||||
tail = ""
|
||||
detail = f"原始日志尾部:{tail}" if tail else f"原始错误见日志:{log_path}"
|
||||
action = "重试后" if retried else ""
|
||||
return (
|
||||
f"提取启动阶段失败{action},可能受主机 GPU/显示重配置干扰,请重试"
|
||||
f"(MinerU 退出码 {returncode};{detail})。"
|
||||
)
|
||||
|
||||
|
||||
def build_mineru_runtime() -> tuple[list[str], dict[str, str], dict[str, str]]:
|
||||
env = os.environ.copy()
|
||||
env.setdefault("MINERU_MODEL_SOURCE", "modelscope")
|
||||
@@ -384,6 +422,9 @@ def build_mineru_runtime() -> tuple[list[str], dict[str, str], dict[str, str]]:
|
||||
else:
|
||||
env.setdefault("CUDA_VISIBLE_DEVICES", "0")
|
||||
env.setdefault("TORCH_CUDNN_V8_API_DISABLED", "1")
|
||||
# 缺陷 #38(2026-09-11):子进程 API 的异常日志即写即落(不再被 stdout
|
||||
# 块缓冲吞掉)——CLI 失败后立即杀 API 进程前真因已落到 job.log。
|
||||
env["PYTHONUNBUFFERED"] = "1"
|
||||
# mineru 3.1.13 CLI 陷阱(部署 agent 实测):带 MINERU_API_SHUTDOWN_ON_STDIN_EOF=1
|
||||
# 时,stdin EOF 会让 API 进程静默 shutdown——subprocess 调用必须 stdin=DEVNULL
|
||||
env.setdefault("MINERU_API_SHUTDOWN_ON_STDIN_EOF", "0")
|
||||
@@ -566,7 +607,34 @@ def run_uploaded_methodology_job(
|
||||
stdout=handle,
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
# 缺陷 #38:启动期失败自动重试 1 次——returncode≠0 且 job.log 无
|
||||
# doc_analyze_streaming 行 = pipeline 起来前就失败(真因曾因块缓冲
|
||||
# 丢失)。重试的 stdout 继续 append 到同一 job.log,日志/状态留
|
||||
# retry 标记(G6-2:已有 pipeline 行的中途失败不得触发重试)。
|
||||
startup_retried = False
|
||||
if result.returncode != 0 and not job_log_has_pipeline_line(log_path):
|
||||
startup_retried = True
|
||||
with log_path.open("a", encoding="utf-8") as handle:
|
||||
handle.write(
|
||||
f"[retry] MinerU 启动期失败(exit {result.returncode},"
|
||||
"无 pipeline 阶段日志),自动重试 1/1\n"
|
||||
)
|
||||
handle.write("$ " + " ".join(cmd) + "\n")
|
||||
handle.flush()
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
cwd=PROJECT_ROOT,
|
||||
env=mineru_env,
|
||||
text=True,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=handle,
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
if not job_log_has_pipeline_line(log_path):
|
||||
raise RuntimeError(
|
||||
summarize_mineru_startup_failure(log_path, result.returncode, retried=startup_retried)
|
||||
)
|
||||
raise RuntimeError(summarize_mineru_failure(log_path, result.returncode))
|
||||
matches = sorted(mineru_out.rglob("*_content_list.json"))
|
||||
if not matches:
|
||||
|
||||
@@ -0,0 +1,328 @@
|
||||
"""test_mineru_retry.py —— MinerU 启动期失败重试与可读错误(缺陷 #38,验收 §4 + 回归 G6)。
|
||||
|
||||
运行(worktree 根目录,无需部署、不触服务):
|
||||
/home/xzq/5090/teamWorkspace/zh/plantai-rag-new-venv/bin/python -m pytest agent/methodology/api/test_mineru_retry.py -v
|
||||
/home/xzq/5090/teamWorkspace/zh/plantai-rag-new-venv/bin/python -m pytest agent/legacy-rag/api/test_mineru_retry.py -v
|
||||
说明:agent/methodology 与 agent/legacy-rag 共用同一份 routes(文件名同、内容同),
|
||||
同一测试文件复制两处、两处同跑(各用各树的 src/)。
|
||||
|
||||
断言:
|
||||
1. build_mineru_runtime 返回 env 含 PYTHONUNBUFFERED=1;
|
||||
2. subprocess.run 首次失败 + job.log 无 pipeline 行 → 自动重试 1 次
|
||||
(两次启动行、job.log 有 [retry] 标记,第二次 stdout 追加同一 job.log);
|
||||
3. 重试仍失败 → status failed + 中文可读「提取启动阶段失败」+ 原始错误尾部;
|
||||
4. 已有 pipeline 行的中途失败 → 不重试,走原错误路径(含 CUDA OOM 中文错误)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import api.methodology_routes as mr
|
||||
|
||||
from api.methodology_routes import (
|
||||
MINERU_PIPELINE_LOG_MARKER,
|
||||
build_mineru_runtime,
|
||||
job_log_has_pipeline_line,
|
||||
run_uploaded_methodology_job,
|
||||
summarize_mineru_startup_failure,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 固定夹具
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass
|
||||
class FakeResult:
|
||||
returncode: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunnerRecorder:
|
||||
"""按调用顺序返回预设 returncode;每次调用把 stdout 写进所给 handle(模拟子进程日志)。"""
|
||||
|
||||
sequence: list
|
||||
calls: list = field(default_factory=list)
|
||||
|
||||
def run(self, cmd, **kwargs):
|
||||
n = len(self.calls)
|
||||
self.calls.append({"cmd": cmd, "kwargs": kwargs})
|
||||
rc = int(self.sequence[min(n, len(self.sequence) - 1)])
|
||||
handle = kwargs.get("stdout")
|
||||
if handle is not None and kwargs.get("text"):
|
||||
handle.write(f"[fake stdout call#{n}] rc={rc}\n")
|
||||
handle.flush()
|
||||
return FakeResult(returncode=rc)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_success(monkeypatch):
|
||||
"""LLM 批次与 bbav5 配置阶段全部假成功(不触真实脚本/目录/真实函数)。
|
||||
|
||||
- gf_python_path / _preflight_executable 桩掉(不依赖真实 python 路径)
|
||||
- build_bbav5_methodology_config 桩掉(不要求真实批次产物目录)
|
||||
- figure_table 阶段走 routes 内部 try/except,不阻断
|
||||
"""
|
||||
captured = {}
|
||||
|
||||
def fake_batch(cmd, **kwargs):
|
||||
captured["batch_cmd"] = cmd
|
||||
return FakeResult(returncode=0)
|
||||
|
||||
monkeypatch.setattr(mr, "gf_python_path", lambda: Path("/usr/bin/python3"))
|
||||
monkeypatch.setattr(mr, "_preflight_executable", lambda cmd, hint, name: None)
|
||||
monkeypatch.setattr(mr.subprocess, "run", fake_batch)
|
||||
monkeypatch.setattr(
|
||||
mr,
|
||||
"build_bbav5_methodology_config",
|
||||
lambda output_dir: captured.setdefault("config", {}),
|
||||
)
|
||||
return captured
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def job_root(monkeypatch, tmp_path):
|
||||
"""把 UPLOAD_JOBS_DIR 重定向到 tmp(绝不写真实数据目录)。"""
|
||||
root = tmp_path / "upload_jobs"
|
||||
monkeypatch.setattr(mr, "UPLOAD_JOBS_DIR", root)
|
||||
return root
|
||||
|
||||
|
||||
def read_status(job_dir: Path) -> dict:
|
||||
return json.loads((job_dir / "status.json").read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def run_job(monkeypatch, tmp_path, runner: RunnerRecorder, source_path: Path, fake_success, job_root):
|
||||
"""以 runner 重绑 subprocess.run 跑完整任务流(fake_success 保证后续阶段假成功)。
|
||||
|
||||
注意:fake_success 已把 mr.subprocess.run 桩为假批次;这里再次 setattr
|
||||
为 runner(同一模块对象),此后所有子进程调用(mineru 两次 + LLM 批次)
|
||||
都由 runner 按 sequence 依次应答——测试 sequence 需覆盖全部调用。
|
||||
"""
|
||||
job_dir = job_root / "testjob"
|
||||
# 模拟 mineru 成功产出:在 job/mineru 下伪造 *_content_list.json
|
||||
# (测试不真正执行 mineru)
|
||||
(job_dir / "mineru").mkdir(parents=True, exist_ok=True)
|
||||
(job_dir / "mineru" / "fake_content_list.json").write_text("{}", encoding="utf-8")
|
||||
# GPU 预检禁用(否则 nvidia-smi 会消耗 runner 的 sequence 槽位)
|
||||
monkeypatch.setenv("METHODOLOGY_MINERU_GPU_PREFLIGHT", "0")
|
||||
monkeypatch.setattr(subprocess, "run", runner.run)
|
||||
run_uploaded_methodology_job(
|
||||
job_id="testjob",
|
||||
source_path=source_path,
|
||||
paper_id="paper1",
|
||||
family="",
|
||||
output_dir=tmp_path / "out",
|
||||
llm_model="deepseek-v4-flash",
|
||||
llm_request_timeout=75,
|
||||
sandbox=False,
|
||||
sandbox_run_id="",
|
||||
)
|
||||
return job_dir
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pdf_source(tmp_path):
|
||||
src = tmp_path / "sample.pdf"
|
||||
src.write_bytes(b"%PDF-1.4\nfake pdf bytes for startup-failure test\n")
|
||||
return src
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def json_source(tmp_path):
|
||||
src = tmp_path / "sample.json"
|
||||
src.write_text("{}", encoding="utf-8")
|
||||
return src
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 修复点 1:env 注入 PYTHONUNBUFFERED
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_build_mineru_runtime_env_unbuffered(monkeypatch):
|
||||
monkeypatch.delenv("MINERU_CMD", raising=False)
|
||||
cmd, env, runtime = build_mineru_runtime()
|
||||
assert env.get("PYTHONUNBUFFERED") == "1"
|
||||
assert cmd # 默认 MINERU_CMD 路径(不要求存在,build 阶段不校验)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 修复点 2:启动期失败自动重试 1 次
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_startup_failure_retries_once(monkeypatch, tmp_path, pdf_source, fake_success, job_root):
|
||||
runner = RunnerRecorder(sequence=[1, 0, 0]) # mineru 首调失败 → 重试成功 → LLM 批次成功
|
||||
job_dir = run_job(monkeypatch, tmp_path, runner, pdf_source, fake_success, job_root)
|
||||
|
||||
log_text = (job_dir / "job.log").read_text(encoding="utf-8")
|
||||
startup_lines = [ln for ln in log_text.splitlines() if ln.startswith("$ ")]
|
||||
assert len(startup_lines) >= 2, f"应出现两次 mineru 启动(1 次自动重试),实际:{startup_lines}"
|
||||
assert "[retry]" in log_text, "job.log 应有可识别重试标记"
|
||||
# 第二次 subprocess.run 的 stdout 追加到同一 job.log
|
||||
assert "[fake stdout call#0]" in log_text
|
||||
assert "[fake stdout call#1]" in log_text
|
||||
|
||||
status = read_status(job_dir)
|
||||
assert status.get("status") == "completed"
|
||||
|
||||
|
||||
def test_startup_failure_retry_then_fail_readable_error(monkeypatch, tmp_path, pdf_source, fake_success, job_root):
|
||||
runner = RunnerRecorder(sequence=[1, 1]) # 两次都失败,且无 pipeline 行
|
||||
job_dir = run_job(monkeypatch, tmp_path, runner, pdf_source, fake_success, job_root)
|
||||
|
||||
log_text = (job_dir / "job.log").read_text(encoding="utf-8")
|
||||
assert len([ln for ln in log_text.splitlines() if ln.startswith("$ /")]) == 2 # 两次 mineru 启动
|
||||
assert "[retry]" in log_text
|
||||
# pipeline 判定:仅检查子进程输出行(启动命令行本身含 marker 关键字)
|
||||
stdout_lines = [ln for ln in log_text.splitlines() if "[fake stdout" in ln or ln.startswith("[retry]")]
|
||||
assert MINERU_PIPELINE_LOG_MARKER not in "\n".join(stdout_lines)
|
||||
|
||||
status = read_status(job_dir)
|
||||
assert status.get("status") == "failed"
|
||||
message = str(status.get("message") or "")
|
||||
assert "提取启动阶段失败" in message, f"中文可读错误缺失:{message}"
|
||||
assert "退出码" in message and "1" in message
|
||||
assert "原始日志尾部" in message or "原始错误见日志" in message
|
||||
assert message # 非空
|
||||
|
||||
|
||||
def test_startup_failure_with_marker_retries_once_then_no_second_retry(monkeypatch, tmp_path, pdf_source, job_root):
|
||||
"""G6-2 边界:第一次失败无 marker → 触发重试;重试后才有 marker 但 rc≠0 →
|
||||
不二次重试,走 summarize_mineru_failure(有 marker 的中途失败路径)。"""
|
||||
sequence = [1, 1]
|
||||
calls = []
|
||||
|
||||
def run_with_marker(cmd, **kwargs):
|
||||
n = len(calls)
|
||||
rc = int(sequence[min(n, len(sequence) - 1)])
|
||||
calls.append({"cmd": cmd, "kwargs": kwargs})
|
||||
handle = kwargs.get("stdout")
|
||||
if handle is not None and kwargs.get("text"):
|
||||
if n == 1: # 第二次调用(重试)时写入 pipeline 行
|
||||
handle.write(f"[fake] {MINERU_PIPELINE_LOG_MARKER} started\n")
|
||||
handle.write(f"[fake stdout call#{n}] rc={rc}\n")
|
||||
handle.flush()
|
||||
return FakeResult(returncode=rc)
|
||||
|
||||
monkeypatch.setenv("METHODOLOGY_MINERU_GPU_PREFLIGHT", "0")
|
||||
monkeypatch.setattr(subprocess, "run", run_with_marker)
|
||||
monkeypatch.setattr(mr, "_preflight_executable", lambda cmd, hint, name: None)
|
||||
monkeypatch.setattr(
|
||||
mr,
|
||||
"summarize_mineru_failure",
|
||||
lambda log_path, rc: f"CUDA out of memory fake path rc={rc}",
|
||||
)
|
||||
job_dir = job_root / "testjob"
|
||||
run_uploaded_methodology_job(
|
||||
job_id="testjob",
|
||||
source_path=pdf_source,
|
||||
paper_id="paper1",
|
||||
family="",
|
||||
output_dir=tmp_path / "out",
|
||||
llm_model="deepseek-v4-flash",
|
||||
llm_request_timeout=75,
|
||||
sandbox=False,
|
||||
sandbox_run_id="",
|
||||
)
|
||||
log_text = (job_dir / "job.log").read_text(encoding="utf-8")
|
||||
assert MINERU_PIPELINE_LOG_MARKER in log_text
|
||||
assert log_text.count("[retry]") == 1, "有 pipeline 行后不得再次重试"
|
||||
# 中途失败(有 marker)→ 原错误路径(CUDA 中文错误)
|
||||
status = read_status(job_dir)
|
||||
assert status.get("status") == "failed"
|
||||
assert "CUDA" in str(status.get("message") or "")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 修复点 4:有 pipeline 行的中途失败不重试
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_mid_failure_with_pipeline_line_no_retry(monkeypatch, tmp_path, pdf_source, job_root):
|
||||
sequence = [1]
|
||||
calls = []
|
||||
|
||||
def run_with_marker(cmd, **kwargs):
|
||||
n = len(calls)
|
||||
rc = int(sequence[min(n, len(sequence) - 1)])
|
||||
calls.append({"cmd": cmd, "kwargs": kwargs})
|
||||
handle = kwargs.get("stdout")
|
||||
if handle is not None and kwargs.get("text"):
|
||||
if n == 0: # 第一次调用即写 pipeline 行
|
||||
handle.write(f"[fake] {MINERU_PIPELINE_LOG_MARKER} started\n")
|
||||
handle.write(f"[fake stdout call#{n}] rc={rc}\n")
|
||||
handle.flush()
|
||||
return FakeResult(returncode=rc)
|
||||
|
||||
monkeypatch.setenv("METHODOLOGY_MINERU_GPU_PREFLIGHT", "0")
|
||||
monkeypatch.setattr(subprocess, "run", run_with_marker)
|
||||
monkeypatch.setattr(mr, "_preflight_executable", lambda cmd, hint, name: None)
|
||||
monkeypatch.setattr(
|
||||
mr,
|
||||
"summarize_mineru_failure",
|
||||
lambda log_path, rc: f"CUDA out of memory fake path rc={rc}",
|
||||
)
|
||||
job_dir = job_root / "testjob"
|
||||
run_uploaded_methodology_job(
|
||||
job_id="testjob",
|
||||
source_path=pdf_source,
|
||||
paper_id="paper1",
|
||||
family="",
|
||||
output_dir=tmp_path / "out",
|
||||
llm_model="deepseek-v4-flash",
|
||||
llm_request_timeout=75,
|
||||
sandbox=False,
|
||||
sandbox_run_id="",
|
||||
)
|
||||
log_text = (job_dir / "job.log").read_text(encoding="utf-8")
|
||||
assert MINERU_PIPELINE_LOG_MARKER in log_text
|
||||
assert "[retry]" not in log_text, "已有 pipeline 行的中途失败不得触发重试"
|
||||
# 原始启动块 = 1 条 `$ env ...` + 1 条 `$ <cmd>`;无重试追加
|
||||
assert len([ln for ln in log_text.splitlines() if ln.startswith("$ /")]) == 1
|
||||
status = read_status(job_dir)
|
||||
assert status.get("status") == "failed"
|
||||
assert "CUDA" in str(status.get("message") or "")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# JSON 直接投递(跳过 mineru 段)不重试、不破坏状态机
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_json_source_skips_mineru_no_retry(monkeypatch, tmp_path, json_source, fake_success, job_root):
|
||||
runner = RunnerRecorder(sequence=[0]) # 仅 LLM 批次一次
|
||||
job_dir = run_job(monkeypatch, tmp_path, runner, json_source, fake_success, job_root)
|
||||
log_text = (job_dir / "job.log").read_text(encoding="utf-8")
|
||||
assert "[retry]" not in log_text
|
||||
assert MINERU_PIPELINE_LOG_MARKER not in log_text
|
||||
status = read_status(job_dir)
|
||||
assert status.get("status") == "completed"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 工具函数单元
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_job_log_has_pipeline_line(tmp_path):
|
||||
log_path = tmp_path / "job.log"
|
||||
assert job_log_has_pipeline_line(log_path) is False
|
||||
log_path.write_text("noise\n", encoding="utf-8")
|
||||
assert job_log_has_pipeline_line(log_path) is False
|
||||
log_path.write_text(f"2026-09-11 [INFO] {MINERU_PIPELINE_LOG_MARKER} begin\n", encoding="utf-8")
|
||||
assert job_log_has_pipeline_line(log_path) is True
|
||||
|
||||
|
||||
def test_summarize_startup_failure_readable(tmp_path):
|
||||
log_path = tmp_path / "job.log"
|
||||
log_path.write_text("line one\nline two\nTraceback: boom\n", encoding="utf-8")
|
||||
msg = summarize_mineru_startup_failure(log_path, 1, retried=True)
|
||||
assert "提取启动阶段失败" in msg
|
||||
assert "重试后" in msg
|
||||
assert "退出码 1" in msg
|
||||
assert "Traceback: boom" in msg
|
||||
missing = summarize_mineru_startup_failure(tmp_path / "nope.log", 1, retried=False)
|
||||
assert "提取启动阶段失败" in missing
|
||||
assert "重试后" not in missing
|
||||
@@ -351,6 +351,22 @@ def mineru_gpu_preflight_error(
|
||||
)
|
||||
|
||||
|
||||
# 缺陷 #38(2026-09-11):成功任务的 job.log 必然有 doc_analyze_streaming 行(176 实测)。
|
||||
# returncode≠0 且日志中无此行 = 启动期失败(pipeline 起来前就死了,真因被块缓冲吞掉)。
|
||||
# 判定条件(methodology 与 legacy-rag 两处 build_mineru_runtime/run_uploaded_methodology_job 同改)
|
||||
MINERU_PIPELINE_LOG_MARKER = "doc_analyze_streaming"
|
||||
MINERU_STARTUP_RETRIES = 1
|
||||
|
||||
|
||||
def job_log_has_pipeline_line(log_path: Path) -> bool:
|
||||
"""job.log 是否已出现 pipeline 日志行(def _job_log_has_pipeline_line)。"""
|
||||
try:
|
||||
text = log_path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
return False
|
||||
return MINERU_PIPELINE_LOG_MARKER in text
|
||||
|
||||
|
||||
def summarize_mineru_failure(log_path: Path, returncode: int) -> str:
|
||||
message = f"MinerU failed with exit code {returncode}; see {log_path}"
|
||||
if not log_path.exists():
|
||||
@@ -368,6 +384,28 @@ def summarize_mineru_failure(log_path: Path, returncode: int) -> str:
|
||||
return message
|
||||
|
||||
|
||||
def summarize_mineru_startup_failure(log_path: Path, returncode: int, retried: bool) -> str:
|
||||
"""启动期失败(无 pipeline 行)的中文可读错误(缺陷 #38)。
|
||||
|
||||
附上 job.log 中可见的原始错误尾部,让用户能看到真因(修复前真因
|
||||
被 stdout 块缓冲 + CLI 立即杀 API 进程永久丢失)。
|
||||
"""
|
||||
tail = ""
|
||||
try:
|
||||
text = log_path.read_text(encoding="utf-8", errors="replace")
|
||||
lines = [ln for ln in text.splitlines() if ln.strip()]
|
||||
if lines:
|
||||
tail = " | ".join(ln.strip() for ln in lines[-3:])[:500]
|
||||
except OSError:
|
||||
tail = ""
|
||||
detail = f"原始日志尾部:{tail}" if tail else f"原始错误见日志:{log_path}"
|
||||
action = "重试后" if retried else ""
|
||||
return (
|
||||
f"提取启动阶段失败{action},可能受主机 GPU/显示重配置干扰,请重试"
|
||||
f"(MinerU 退出码 {returncode};{detail})。"
|
||||
)
|
||||
|
||||
|
||||
def build_mineru_runtime() -> tuple[list[str], dict[str, str], dict[str, str]]:
|
||||
env = os.environ.copy()
|
||||
env.setdefault("MINERU_MODEL_SOURCE", "modelscope")
|
||||
@@ -384,6 +422,9 @@ def build_mineru_runtime() -> tuple[list[str], dict[str, str], dict[str, str]]:
|
||||
else:
|
||||
env.setdefault("CUDA_VISIBLE_DEVICES", "0")
|
||||
env.setdefault("TORCH_CUDNN_V8_API_DISABLED", "1")
|
||||
# 缺陷 #38(2026-09-11):子进程 API 的异常日志即写即落(不再被 stdout
|
||||
# 块缓冲吞掉)——CLI 失败后立即杀 API 进程前真因已落到 job.log。
|
||||
env["PYTHONUNBUFFERED"] = "1"
|
||||
# mineru 3.1.13 CLI 陷阱(部署 agent 实测):带 MINERU_API_SHUTDOWN_ON_STDIN_EOF=1
|
||||
# 时,stdin EOF 会让 API 进程静默 shutdown——subprocess 调用必须 stdin=DEVNULL
|
||||
env.setdefault("MINERU_API_SHUTDOWN_ON_STDIN_EOF", "0")
|
||||
@@ -566,7 +607,34 @@ def run_uploaded_methodology_job(
|
||||
stdout=handle,
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
# 缺陷 #38:启动期失败自动重试 1 次——returncode≠0 且 job.log 无
|
||||
# doc_analyze_streaming 行 = pipeline 起来前就失败(真因曾因块缓冲
|
||||
# 丢失)。重试的 stdout 继续 append 到同一 job.log,日志/状态留
|
||||
# retry 标记(G6-2:已有 pipeline 行的中途失败不得触发重试)。
|
||||
startup_retried = False
|
||||
if result.returncode != 0 and not job_log_has_pipeline_line(log_path):
|
||||
startup_retried = True
|
||||
with log_path.open("a", encoding="utf-8") as handle:
|
||||
handle.write(
|
||||
f"[retry] MinerU 启动期失败(exit {result.returncode},"
|
||||
"无 pipeline 阶段日志),自动重试 1/1\n"
|
||||
)
|
||||
handle.write("$ " + " ".join(cmd) + "\n")
|
||||
handle.flush()
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
cwd=PROJECT_ROOT,
|
||||
env=mineru_env,
|
||||
text=True,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=handle,
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
if not job_log_has_pipeline_line(log_path):
|
||||
raise RuntimeError(
|
||||
summarize_mineru_startup_failure(log_path, result.returncode, retried=startup_retried)
|
||||
)
|
||||
raise RuntimeError(summarize_mineru_failure(log_path, result.returncode))
|
||||
matches = sorted(mineru_out.rglob("*_content_list.json"))
|
||||
if not matches:
|
||||
|
||||
@@ -0,0 +1,328 @@
|
||||
"""test_mineru_retry.py —— MinerU 启动期失败重试与可读错误(缺陷 #38,验收 §4 + 回归 G6)。
|
||||
|
||||
运行(worktree 根目录,无需部署、不触服务):
|
||||
/home/xzq/5090/teamWorkspace/zh/plantai-rag-new-venv/bin/python -m pytest agent/methodology/api/test_mineru_retry.py -v
|
||||
/home/xzq/5090/teamWorkspace/zh/plantai-rag-new-venv/bin/python -m pytest agent/legacy-rag/api/test_mineru_retry.py -v
|
||||
说明:agent/methodology 与 agent/legacy-rag 共用同一份 routes(文件名同、内容同),
|
||||
同一测试文件复制两处、两处同跑(各用各树的 src/)。
|
||||
|
||||
断言:
|
||||
1. build_mineru_runtime 返回 env 含 PYTHONUNBUFFERED=1;
|
||||
2. subprocess.run 首次失败 + job.log 无 pipeline 行 → 自动重试 1 次
|
||||
(两次启动行、job.log 有 [retry] 标记,第二次 stdout 追加同一 job.log);
|
||||
3. 重试仍失败 → status failed + 中文可读「提取启动阶段失败」+ 原始错误尾部;
|
||||
4. 已有 pipeline 行的中途失败 → 不重试,走原错误路径(含 CUDA OOM 中文错误)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import api.methodology_routes as mr
|
||||
|
||||
from api.methodology_routes import (
|
||||
MINERU_PIPELINE_LOG_MARKER,
|
||||
build_mineru_runtime,
|
||||
job_log_has_pipeline_line,
|
||||
run_uploaded_methodology_job,
|
||||
summarize_mineru_startup_failure,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 固定夹具
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass
|
||||
class FakeResult:
|
||||
returncode: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunnerRecorder:
|
||||
"""按调用顺序返回预设 returncode;每次调用把 stdout 写进所给 handle(模拟子进程日志)。"""
|
||||
|
||||
sequence: list
|
||||
calls: list = field(default_factory=list)
|
||||
|
||||
def run(self, cmd, **kwargs):
|
||||
n = len(self.calls)
|
||||
self.calls.append({"cmd": cmd, "kwargs": kwargs})
|
||||
rc = int(self.sequence[min(n, len(self.sequence) - 1)])
|
||||
handle = kwargs.get("stdout")
|
||||
if handle is not None and kwargs.get("text"):
|
||||
handle.write(f"[fake stdout call#{n}] rc={rc}\n")
|
||||
handle.flush()
|
||||
return FakeResult(returncode=rc)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_success(monkeypatch):
|
||||
"""LLM 批次与 bbav5 配置阶段全部假成功(不触真实脚本/目录/真实函数)。
|
||||
|
||||
- gf_python_path / _preflight_executable 桩掉(不依赖真实 python 路径)
|
||||
- build_bbav5_methodology_config 桩掉(不要求真实批次产物目录)
|
||||
- figure_table 阶段走 routes 内部 try/except,不阻断
|
||||
"""
|
||||
captured = {}
|
||||
|
||||
def fake_batch(cmd, **kwargs):
|
||||
captured["batch_cmd"] = cmd
|
||||
return FakeResult(returncode=0)
|
||||
|
||||
monkeypatch.setattr(mr, "gf_python_path", lambda: Path("/usr/bin/python3"))
|
||||
monkeypatch.setattr(mr, "_preflight_executable", lambda cmd, hint, name: None)
|
||||
monkeypatch.setattr(mr.subprocess, "run", fake_batch)
|
||||
monkeypatch.setattr(
|
||||
mr,
|
||||
"build_bbav5_methodology_config",
|
||||
lambda output_dir: captured.setdefault("config", {}),
|
||||
)
|
||||
return captured
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def job_root(monkeypatch, tmp_path):
|
||||
"""把 UPLOAD_JOBS_DIR 重定向到 tmp(绝不写真实数据目录)。"""
|
||||
root = tmp_path / "upload_jobs"
|
||||
monkeypatch.setattr(mr, "UPLOAD_JOBS_DIR", root)
|
||||
return root
|
||||
|
||||
|
||||
def read_status(job_dir: Path) -> dict:
|
||||
return json.loads((job_dir / "status.json").read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def run_job(monkeypatch, tmp_path, runner: RunnerRecorder, source_path: Path, fake_success, job_root):
|
||||
"""以 runner 重绑 subprocess.run 跑完整任务流(fake_success 保证后续阶段假成功)。
|
||||
|
||||
注意:fake_success 已把 mr.subprocess.run 桩为假批次;这里再次 setattr
|
||||
为 runner(同一模块对象),此后所有子进程调用(mineru 两次 + LLM 批次)
|
||||
都由 runner 按 sequence 依次应答——测试 sequence 需覆盖全部调用。
|
||||
"""
|
||||
job_dir = job_root / "testjob"
|
||||
# 模拟 mineru 成功产出:在 job/mineru 下伪造 *_content_list.json
|
||||
# (测试不真正执行 mineru)
|
||||
(job_dir / "mineru").mkdir(parents=True, exist_ok=True)
|
||||
(job_dir / "mineru" / "fake_content_list.json").write_text("{}", encoding="utf-8")
|
||||
# GPU 预检禁用(否则 nvidia-smi 会消耗 runner 的 sequence 槽位)
|
||||
monkeypatch.setenv("METHODOLOGY_MINERU_GPU_PREFLIGHT", "0")
|
||||
monkeypatch.setattr(subprocess, "run", runner.run)
|
||||
run_uploaded_methodology_job(
|
||||
job_id="testjob",
|
||||
source_path=source_path,
|
||||
paper_id="paper1",
|
||||
family="",
|
||||
output_dir=tmp_path / "out",
|
||||
llm_model="deepseek-v4-flash",
|
||||
llm_request_timeout=75,
|
||||
sandbox=False,
|
||||
sandbox_run_id="",
|
||||
)
|
||||
return job_dir
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pdf_source(tmp_path):
|
||||
src = tmp_path / "sample.pdf"
|
||||
src.write_bytes(b"%PDF-1.4\nfake pdf bytes for startup-failure test\n")
|
||||
return src
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def json_source(tmp_path):
|
||||
src = tmp_path / "sample.json"
|
||||
src.write_text("{}", encoding="utf-8")
|
||||
return src
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 修复点 1:env 注入 PYTHONUNBUFFERED
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_build_mineru_runtime_env_unbuffered(monkeypatch):
|
||||
monkeypatch.delenv("MINERU_CMD", raising=False)
|
||||
cmd, env, runtime = build_mineru_runtime()
|
||||
assert env.get("PYTHONUNBUFFERED") == "1"
|
||||
assert cmd # 默认 MINERU_CMD 路径(不要求存在,build 阶段不校验)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 修复点 2:启动期失败自动重试 1 次
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_startup_failure_retries_once(monkeypatch, tmp_path, pdf_source, fake_success, job_root):
|
||||
runner = RunnerRecorder(sequence=[1, 0, 0]) # mineru 首调失败 → 重试成功 → LLM 批次成功
|
||||
job_dir = run_job(monkeypatch, tmp_path, runner, pdf_source, fake_success, job_root)
|
||||
|
||||
log_text = (job_dir / "job.log").read_text(encoding="utf-8")
|
||||
startup_lines = [ln for ln in log_text.splitlines() if ln.startswith("$ ")]
|
||||
assert len(startup_lines) >= 2, f"应出现两次 mineru 启动(1 次自动重试),实际:{startup_lines}"
|
||||
assert "[retry]" in log_text, "job.log 应有可识别重试标记"
|
||||
# 第二次 subprocess.run 的 stdout 追加到同一 job.log
|
||||
assert "[fake stdout call#0]" in log_text
|
||||
assert "[fake stdout call#1]" in log_text
|
||||
|
||||
status = read_status(job_dir)
|
||||
assert status.get("status") == "completed"
|
||||
|
||||
|
||||
def test_startup_failure_retry_then_fail_readable_error(monkeypatch, tmp_path, pdf_source, fake_success, job_root):
|
||||
runner = RunnerRecorder(sequence=[1, 1]) # 两次都失败,且无 pipeline 行
|
||||
job_dir = run_job(monkeypatch, tmp_path, runner, pdf_source, fake_success, job_root)
|
||||
|
||||
log_text = (job_dir / "job.log").read_text(encoding="utf-8")
|
||||
assert len([ln for ln in log_text.splitlines() if ln.startswith("$ /")]) == 2 # 两次 mineru 启动
|
||||
assert "[retry]" in log_text
|
||||
# pipeline 判定:仅检查子进程输出行(启动命令行本身含 marker 关键字)
|
||||
stdout_lines = [ln for ln in log_text.splitlines() if "[fake stdout" in ln or ln.startswith("[retry]")]
|
||||
assert MINERU_PIPELINE_LOG_MARKER not in "\n".join(stdout_lines)
|
||||
|
||||
status = read_status(job_dir)
|
||||
assert status.get("status") == "failed"
|
||||
message = str(status.get("message") or "")
|
||||
assert "提取启动阶段失败" in message, f"中文可读错误缺失:{message}"
|
||||
assert "退出码" in message and "1" in message
|
||||
assert "原始日志尾部" in message or "原始错误见日志" in message
|
||||
assert message # 非空
|
||||
|
||||
|
||||
def test_startup_failure_with_marker_retries_once_then_no_second_retry(monkeypatch, tmp_path, pdf_source, job_root):
|
||||
"""G6-2 边界:第一次失败无 marker → 触发重试;重试后才有 marker 但 rc≠0 →
|
||||
不二次重试,走 summarize_mineru_failure(有 marker 的中途失败路径)。"""
|
||||
sequence = [1, 1]
|
||||
calls = []
|
||||
|
||||
def run_with_marker(cmd, **kwargs):
|
||||
n = len(calls)
|
||||
rc = int(sequence[min(n, len(sequence) - 1)])
|
||||
calls.append({"cmd": cmd, "kwargs": kwargs})
|
||||
handle = kwargs.get("stdout")
|
||||
if handle is not None and kwargs.get("text"):
|
||||
if n == 1: # 第二次调用(重试)时写入 pipeline 行
|
||||
handle.write(f"[fake] {MINERU_PIPELINE_LOG_MARKER} started\n")
|
||||
handle.write(f"[fake stdout call#{n}] rc={rc}\n")
|
||||
handle.flush()
|
||||
return FakeResult(returncode=rc)
|
||||
|
||||
monkeypatch.setenv("METHODOLOGY_MINERU_GPU_PREFLIGHT", "0")
|
||||
monkeypatch.setattr(subprocess, "run", run_with_marker)
|
||||
monkeypatch.setattr(mr, "_preflight_executable", lambda cmd, hint, name: None)
|
||||
monkeypatch.setattr(
|
||||
mr,
|
||||
"summarize_mineru_failure",
|
||||
lambda log_path, rc: f"CUDA out of memory fake path rc={rc}",
|
||||
)
|
||||
job_dir = job_root / "testjob"
|
||||
run_uploaded_methodology_job(
|
||||
job_id="testjob",
|
||||
source_path=pdf_source,
|
||||
paper_id="paper1",
|
||||
family="",
|
||||
output_dir=tmp_path / "out",
|
||||
llm_model="deepseek-v4-flash",
|
||||
llm_request_timeout=75,
|
||||
sandbox=False,
|
||||
sandbox_run_id="",
|
||||
)
|
||||
log_text = (job_dir / "job.log").read_text(encoding="utf-8")
|
||||
assert MINERU_PIPELINE_LOG_MARKER in log_text
|
||||
assert log_text.count("[retry]") == 1, "有 pipeline 行后不得再次重试"
|
||||
# 中途失败(有 marker)→ 原错误路径(CUDA 中文错误)
|
||||
status = read_status(job_dir)
|
||||
assert status.get("status") == "failed"
|
||||
assert "CUDA" in str(status.get("message") or "")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 修复点 4:有 pipeline 行的中途失败不重试
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_mid_failure_with_pipeline_line_no_retry(monkeypatch, tmp_path, pdf_source, job_root):
|
||||
sequence = [1]
|
||||
calls = []
|
||||
|
||||
def run_with_marker(cmd, **kwargs):
|
||||
n = len(calls)
|
||||
rc = int(sequence[min(n, len(sequence) - 1)])
|
||||
calls.append({"cmd": cmd, "kwargs": kwargs})
|
||||
handle = kwargs.get("stdout")
|
||||
if handle is not None and kwargs.get("text"):
|
||||
if n == 0: # 第一次调用即写 pipeline 行
|
||||
handle.write(f"[fake] {MINERU_PIPELINE_LOG_MARKER} started\n")
|
||||
handle.write(f"[fake stdout call#{n}] rc={rc}\n")
|
||||
handle.flush()
|
||||
return FakeResult(returncode=rc)
|
||||
|
||||
monkeypatch.setenv("METHODOLOGY_MINERU_GPU_PREFLIGHT", "0")
|
||||
monkeypatch.setattr(subprocess, "run", run_with_marker)
|
||||
monkeypatch.setattr(mr, "_preflight_executable", lambda cmd, hint, name: None)
|
||||
monkeypatch.setattr(
|
||||
mr,
|
||||
"summarize_mineru_failure",
|
||||
lambda log_path, rc: f"CUDA out of memory fake path rc={rc}",
|
||||
)
|
||||
job_dir = job_root / "testjob"
|
||||
run_uploaded_methodology_job(
|
||||
job_id="testjob",
|
||||
source_path=pdf_source,
|
||||
paper_id="paper1",
|
||||
family="",
|
||||
output_dir=tmp_path / "out",
|
||||
llm_model="deepseek-v4-flash",
|
||||
llm_request_timeout=75,
|
||||
sandbox=False,
|
||||
sandbox_run_id="",
|
||||
)
|
||||
log_text = (job_dir / "job.log").read_text(encoding="utf-8")
|
||||
assert MINERU_PIPELINE_LOG_MARKER in log_text
|
||||
assert "[retry]" not in log_text, "已有 pipeline 行的中途失败不得触发重试"
|
||||
# 原始启动块 = 1 条 `$ env ...` + 1 条 `$ <cmd>`;无重试追加
|
||||
assert len([ln for ln in log_text.splitlines() if ln.startswith("$ /")]) == 1
|
||||
status = read_status(job_dir)
|
||||
assert status.get("status") == "failed"
|
||||
assert "CUDA" in str(status.get("message") or "")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# JSON 直接投递(跳过 mineru 段)不重试、不破坏状态机
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_json_source_skips_mineru_no_retry(monkeypatch, tmp_path, json_source, fake_success, job_root):
|
||||
runner = RunnerRecorder(sequence=[0]) # 仅 LLM 批次一次
|
||||
job_dir = run_job(monkeypatch, tmp_path, runner, json_source, fake_success, job_root)
|
||||
log_text = (job_dir / "job.log").read_text(encoding="utf-8")
|
||||
assert "[retry]" not in log_text
|
||||
assert MINERU_PIPELINE_LOG_MARKER not in log_text
|
||||
status = read_status(job_dir)
|
||||
assert status.get("status") == "completed"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 工具函数单元
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_job_log_has_pipeline_line(tmp_path):
|
||||
log_path = tmp_path / "job.log"
|
||||
assert job_log_has_pipeline_line(log_path) is False
|
||||
log_path.write_text("noise\n", encoding="utf-8")
|
||||
assert job_log_has_pipeline_line(log_path) is False
|
||||
log_path.write_text(f"2026-09-11 [INFO] {MINERU_PIPELINE_LOG_MARKER} begin\n", encoding="utf-8")
|
||||
assert job_log_has_pipeline_line(log_path) is True
|
||||
|
||||
|
||||
def test_summarize_startup_failure_readable(tmp_path):
|
||||
log_path = tmp_path / "job.log"
|
||||
log_path.write_text("line one\nline two\nTraceback: boom\n", encoding="utf-8")
|
||||
msg = summarize_mineru_startup_failure(log_path, 1, retried=True)
|
||||
assert "提取启动阶段失败" in msg
|
||||
assert "重试后" in msg
|
||||
assert "退出码 1" in msg
|
||||
assert "Traceback: boom" in msg
|
||||
missing = summarize_mineru_startup_failure(tmp_path / "nope.log", 1, retried=False)
|
||||
assert "提取启动阶段失败" in missing
|
||||
assert "重试后" not in missing
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Tuple
|
||||
|
||||
@@ -16,6 +17,7 @@ from src.rag.normalized_release import (
|
||||
)
|
||||
|
||||
from .schemas import EvidenceItem, RetrievalHypothesis
|
||||
from .soft_gate_policy import doi_from_paper_id, pmcid_from_paper_id
|
||||
|
||||
ALLOWED_QDRANT_COLLECTIONS = {
|
||||
"plant_srt_direct_v1_bge_m3_20260702",
|
||||
@@ -36,6 +38,12 @@ def _norm(value: Any) -> str:
|
||||
return " ".join(text.split())
|
||||
|
||||
|
||||
# doi: 前缀形态 paper_id 的 DOI 全文校验(回改 v4,2026-09-11)
|
||||
_DOI_LITERAL = re.compile(
|
||||
r"10\.\d{4,9}/[A-Za-z0-9][A-Za-z0-9._;()/:+\-]*"
|
||||
)
|
||||
|
||||
|
||||
def load_paper_metadata_registry(path: Path) -> Dict[str, Dict[str, Any]]:
|
||||
records: Dict[str, Dict[str, Any]] = {}
|
||||
if not path.is_file():
|
||||
@@ -117,6 +125,7 @@ class DirectFlowRetriever:
|
||||
use_qdrant: bool = True,
|
||||
use_neo4j: bool = True,
|
||||
use_graph_expansion: bool = True,
|
||||
qdrant_score_threshold: float | None = None,
|
||||
) -> Tuple[List[EvidenceItem], Dict[str, Any]]:
|
||||
effective_use_neo4j = bool(use_neo4j) and self.graph_schema in {
|
||||
"canonical",
|
||||
@@ -138,6 +147,7 @@ class DirectFlowRetriever:
|
||||
"use_graph_expansion": use_graph_expansion,
|
||||
"effective_use_neo4j": effective_use_neo4j,
|
||||
"effective_use_graph_expansion": effective_use_graph_expansion,
|
||||
"qdrant_score_threshold": qdrant_score_threshold,
|
||||
},
|
||||
"neo4j_status": (
|
||||
"enabled"
|
||||
@@ -153,7 +163,12 @@ class DirectFlowRetriever:
|
||||
per_query = max(10, min(qdrant_top_k // max(1, len(all_queries)), 50))
|
||||
if self.vector_query_workers == 1 or len(all_queries) == 1:
|
||||
query_rows = [
|
||||
(query, self.qdrant_search(query, per_query))
|
||||
(
|
||||
query,
|
||||
self.qdrant_search(
|
||||
query, per_query, score_threshold=qdrant_score_threshold
|
||||
),
|
||||
)
|
||||
for query in all_queries
|
||||
]
|
||||
else:
|
||||
@@ -162,7 +177,12 @@ class DirectFlowRetriever:
|
||||
thread_name_prefix="qa-vector",
|
||||
) as executor:
|
||||
futures = [
|
||||
executor.submit(self.qdrant_search, query, per_query)
|
||||
executor.submit(
|
||||
self.qdrant_search,
|
||||
query,
|
||||
per_query,
|
||||
score_threshold=qdrant_score_threshold,
|
||||
)
|
||||
for query in all_queries
|
||||
]
|
||||
query_rows = [
|
||||
@@ -200,14 +220,20 @@ class DirectFlowRetriever:
|
||||
}
|
||||
return merged, audit
|
||||
|
||||
def qdrant_search(self, query: str, limit: int) -> List[EvidenceItem]:
|
||||
def qdrant_search(
|
||||
self, query: str, limit: int, *, score_threshold: float | None = None
|
||||
) -> List[EvidenceItem]:
|
||||
try:
|
||||
vector = self.embed(query)
|
||||
# 缺陷 #28 回改 v2(2026-09-11):向量域阈值仅 search/retrieval-only
|
||||
# 路径传入(pipeline 传 QA_MIN_VECTOR_SCORE=0.45);chat 模式 None
|
||||
# 保召回,rerank 后由 final 域 MIN_SOURCE_SCORE=0.25 兜底。
|
||||
response = self.qdrant.query_points(
|
||||
collection_name=self.collection,
|
||||
query=vector,
|
||||
limit=limit,
|
||||
with_payload=True,
|
||||
score_threshold=score_threshold,
|
||||
)
|
||||
except Exception:
|
||||
return []
|
||||
@@ -324,6 +350,20 @@ class DirectFlowRetriever:
|
||||
metadata = payload.get("metadata") if isinstance(payload.get("metadata"), dict) else {}
|
||||
paper_id = _text(payload.get("paper_id") or payload.get("best_paper_id"))
|
||||
registry_metadata = getattr(self, "paper_metadata", {}).get(paper_id, {})
|
||||
# 缺陷 #28 回改 v4(2026-09-11):175 无 paper metadata registry,
|
||||
# Qdrant payload 又无 pmid/pmcid/doi 键——chat 的 public_citation
|
||||
# 过滤会把有正文的 qdrant 条目全滤掉,只剩无 evidence 的 neo4j
|
||||
# 归一化图断言(bge 全 0.02-0.047),final 门 0.25 全拒 sources=[]。
|
||||
# 从 paper_id 派生 DOI/PMCID 回填引用字段(payload 显式字段 >
|
||||
# registry > paper_id 派生)。
|
||||
_derived_doi = doi_from_paper_id(paper_id)
|
||||
_derived_pmcid = pmcid_from_paper_id(paper_id)
|
||||
# doi: 前缀形态(如 doi:10.1002/ps.6526)直接剥前缀(DOI 已带斜杠,
|
||||
# doi_from_paper_id 不覆盖此形态)。
|
||||
if not _derived_doi and paper_id.lower().startswith("doi:"):
|
||||
_candidate = paper_id[4:].strip()
|
||||
if _DOI_LITERAL.fullmatch(_candidate):
|
||||
_derived_doi = _candidate
|
||||
return EvidenceItem(
|
||||
evidence_id="",
|
||||
kb="V16/V15",
|
||||
@@ -334,8 +374,8 @@ class DirectFlowRetriever:
|
||||
paper_id=paper_id,
|
||||
title=_text(payload.get("title") or payload.get("paper_title") or metadata.get("title") or registry_metadata.get("title")),
|
||||
pmid=_text(payload.get("pmid") or payload.get("pubmed_id") or metadata.get("pmid") or registry_metadata.get("pmid")),
|
||||
pmcid=_text(payload.get("pmcid") or metadata.get("pmcid") or registry_metadata.get("pmcid")),
|
||||
doi=_text(payload.get("doi") or metadata.get("doi") or registry_metadata.get("doi")),
|
||||
pmcid=_text(payload.get("pmcid") or metadata.get("pmcid") or registry_metadata.get("pmcid") or _derived_pmcid),
|
||||
doi=_text(payload.get("doi") or metadata.get("doi") or registry_metadata.get("doi") or _derived_doi),
|
||||
species=_text(payload.get("species") or metadata.get("species")),
|
||||
section=_text(payload.get("section") or metadata.get("section")),
|
||||
output_layer=_text(payload.get("output_layer") or "main_kg"),
|
||||
|
||||
@@ -43,6 +43,7 @@ from .question_decomposition import QuestionDecompositionPlanner
|
||||
from .query_builder import QueryBuilder
|
||||
from .reranker import BGEReranker
|
||||
from .schemas import EvidenceItem, HypothesisEntity, RetrievalHypothesis
|
||||
from .score_gate import min_source_score, min_vector_score
|
||||
from .soft_gate_policy import (
|
||||
LEGACY,
|
||||
doi_from_paper_id,
|
||||
@@ -1254,7 +1255,11 @@ class QAV1Pipeline:
|
||||
use_qdrant = bool(retrieval_ablation.get("use_qdrant", True))
|
||||
use_neo4j = bool(retrieval_ablation.get("use_neo4j", True))
|
||||
use_graph_expansion = bool(retrieval_ablation.get("use_graph_expansion", True)) and use_neo4j
|
||||
use_bge_rerank = bool(ablation.get("use_bge_rerank", True)) and self.config.enable_bge_rerank
|
||||
use_bge_rerank = (
|
||||
bool(ablation.get("use_bge_rerank", True))
|
||||
and self.config.enable_bge_rerank
|
||||
and not retrieval_only # 回改 v2:search 免 CPU rerank(性能根治)
|
||||
)
|
||||
use_final_llm = not retrieval_only
|
||||
graph_min_evidence = int(ablation.get("graph_min_evidence") or os.environ.get("QA_V115_GRAPH_MIN_EVIDENCE", "15"))
|
||||
answer_context_mode = str(ablation.get("answer_context") or "top30_plus_graph_chains")
|
||||
@@ -1285,14 +1290,22 @@ class QAV1Pipeline:
|
||||
1.0, min(float(endpoint_path_time_budget_seconds or 5.0), 30.0)
|
||||
)
|
||||
|
||||
if retrieval_only and not self.llm.available:
|
||||
# V1.1.711 supports retrieval-only operation. Without an LLM key,
|
||||
# disable provisional drafting, structured planning, and final answer
|
||||
# generation, but still return reranked top30 evidence and native
|
||||
# Neo4j graph chains from the raw question.
|
||||
if retrieval_only:
|
||||
# 缺陷 #28 回改 v2(2026-09-11):检索模式绝不产生答案——无条件
|
||||
# 关闭草稿/规划/最终 LLM(不看 llm.available;LLM 调用纯浪费且
|
||||
# 拖慢 search)。仍返回 top30 向量证据。
|
||||
use_draft = False
|
||||
use_planner = False
|
||||
use_final_llm = False
|
||||
# 缺陷 #28 回改 v3(2026-09-11):检索模式只走 Qdrant 向量——
|
||||
# r3 语义补完。图证据种子检索(use_neo4j,graph_top_k=80)在
|
||||
# 13M 边库上 150s+(TL 实测 arginine 150s+ vs 关 neo4j 后
|
||||
# 0.787s),不可接受。use_graph_expansion 依赖 use_neo4j 一并
|
||||
# 显式关(本块在所有 ablation 解析之后:检索模式用户 ablation
|
||||
# 不得重开图检索——性能红线)。native_graph_chains 空数组在
|
||||
# search 分支已安全。
|
||||
use_neo4j = False
|
||||
use_graph_expansion = False
|
||||
|
||||
t0 = time.perf_counter()
|
||||
if use_draft:
|
||||
@@ -1391,7 +1404,15 @@ class QAV1Pipeline:
|
||||
"use_neo4j": use_neo4j,
|
||||
"use_graph_expansion": use_graph_expansion,
|
||||
}
|
||||
if self.agentic_controller is not None:
|
||||
if self.agentic_controller is None:
|
||||
# 回改 v2(2026-09-11):agentic_controller 禁用(含单测)时,
|
||||
# 给确定性 plan——query_plan.agentic_plan 键不缺失。
|
||||
agentic_plan = {
|
||||
"vector_plan": {"enabled": bool(use_qdrant)},
|
||||
"graph_plan": {"enabled": bool(use_neo4j)},
|
||||
"plan_source": "deterministic_retrieval_only_plan",
|
||||
}
|
||||
elif self.agentic_controller is not None:
|
||||
try:
|
||||
agentic_plan = self.agentic_controller.plan(
|
||||
question,
|
||||
@@ -1550,6 +1571,9 @@ class QAV1Pipeline:
|
||||
hypothesis,
|
||||
qdrant_top_k=qdrant_top_k,
|
||||
graph_top_k=graph_top_k,
|
||||
qdrant_score_threshold=(
|
||||
min_vector_score() if retrieval_only else None
|
||||
),
|
||||
**direct_flow_switches,
|
||||
)
|
||||
timings["retrieval"] = time.perf_counter() - t0
|
||||
@@ -1565,6 +1589,14 @@ class QAV1Pipeline:
|
||||
clustered_input, cluster_audit = build_clusters(ranked_input)
|
||||
if use_bge_rerank:
|
||||
ranked, rerank_audit = self._rerank(question, clustered_input)
|
||||
elif retrieval_only:
|
||||
# 回改 v2(2026-09-11):search 免 BGE rerank,候选按 vector_score
|
||||
# 排序直接输出,final_score 落在 vector_score 域——与 Qdrant
|
||||
# score_threshold(min_vector_score=0.45)同域同语义。
|
||||
ranked = sorted(clustered_input, key=lambda x: -x.vector_score)
|
||||
for item in ranked:
|
||||
item.final_score = item.vector_score
|
||||
rerank_audit = [{"status": "disabled_retrieval_only"}]
|
||||
else:
|
||||
ranked = sorted(clustered_input, key=lambda x: -(x.rrf_score or x.vector_score))
|
||||
for item in ranked:
|
||||
@@ -1574,6 +1606,8 @@ class QAV1Pipeline:
|
||||
ranked,
|
||||
top_k_for_llm=top_k_for_llm,
|
||||
graph_min_evidence=graph_min_evidence,
|
||||
# 回改 v2:search 预筛用向量域 0.45;chat 走 final 域 0.25
|
||||
min_score=(min_vector_score() if retrieval_only else None),
|
||||
)
|
||||
assign_evidence_ids(selected)
|
||||
timings["cluster_bge_rerank_and_select"] = time.perf_counter() - t0
|
||||
@@ -3316,7 +3350,19 @@ class QAV1Pipeline:
|
||||
*,
|
||||
top_k_for_llm: int,
|
||||
graph_min_evidence: int,
|
||||
min_score: float | None = None,
|
||||
) -> tuple[List[EvidenceItem], Dict[str, Any]]:
|
||||
# 缺陷 #28 回改 v2(2026-09-11):相关性下限双域。
|
||||
# - min_score 显式传入时用它(search/retrieval-only 传向量域
|
||||
# min_vector_score()=0.45);
|
||||
# - 缺省为 final 域 min_source_score()=0.25(chat 模式语义不变)。
|
||||
# 低于下限的候选不入选;全部低于下限 -> 返回空(前端呈现未命中)。
|
||||
_min_score = min_source_score() if min_score is None else min_score
|
||||
_min_score_domain = "vector" if min_score is not None else "final"
|
||||
_below_min = len(ranked) - sum(
|
||||
1 for item in ranked if item.final_score >= _min_score
|
||||
) # 被下限过滤的候选数(审计用,见 return 处)
|
||||
ranked = [item for item in ranked if item.final_score >= _min_score]
|
||||
graph_items = [item for item in ranked if cls._is_graph_item(item)]
|
||||
other_items = [item for item in ranked if not cls._is_graph_item(item)]
|
||||
selected: List[EvidenceItem] = []
|
||||
@@ -3365,6 +3411,9 @@ class QAV1Pipeline:
|
||||
"graph_selected": graph_selected,
|
||||
"non_graph_selected": len(selected) - graph_selected,
|
||||
"graph_quota_satisfied": graph_selected >= graph_min_evidence,
|
||||
"min_source_score": _min_score,
|
||||
"below_min_source_score": _below_min,
|
||||
"min_score_domain": _min_score_domain,
|
||||
"note": "Graph quota is enforced after BGE rerank. If fewer graph candidates exist, all available graph candidates are retained and the deficit is reported.",
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
"""score_gate.py —— 检索相关性下限(缺陷 #28 回改 v2,2026-09-11)。
|
||||
|
||||
双域双常量(TL 2026-09-11 在 175 裸进程 Qdrant 27433 + Ollama bge-m3:567m 实测):
|
||||
- 垃圾问题「景甜是谁」top10 vector_score(cosine) 全部 0.32-0.34;
|
||||
- arginine 正常问题 top10 vector_score 0.63+;
|
||||
- 垃圾 final_score(BGE rerank 后)仍为 0.19-0.20(测试 agent 实测)。
|
||||
|
||||
MIN_VECTOR_SCORE = 0.45(env QA_MIN_VECTOR_SCORE)
|
||||
- 语义:向量域(Qdrant Cosine 分数)相关性下限。0.45 位于垃圾带
|
||||
(0.32-0.34)与正常带(0.63+)之间并各留余量。
|
||||
- 用途:仅 search / retrieval-only 路径——Qdrant score_threshold 与
|
||||
检索模式预筛。chat 模式不使用(保召回,rerank 后用 MIN_SOURCE_SCORE 兜底)。
|
||||
|
||||
MIN_SOURCE_SCORE = 0.25(env QA_MIN_SOURCE_SCORE)
|
||||
- 语义:最终域(final_score = 0.8*sigmoid(bge) + 0.2*rrf/max_rrf)相关性
|
||||
下限。垃圾 final 0.19-0.20 与其留 ~25% 余量,正常命中 final >0.7。
|
||||
- 用途:chat 模式 _select_graph_balanced 的后置过滤(r4-28 语义不变)。
|
||||
|
||||
本模块无第三方依赖、无 IO,可被 v16_retriever / direct_flow_retriever /
|
||||
pipeline 直接 import(编译期与运行期均安全)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
DEFAULT_MIN_VECTOR_SCORE = 0.45
|
||||
DEFAULT_MIN_SOURCE_SCORE = 0.25
|
||||
_ENV_VECTOR = "QA_MIN_VECTOR_SCORE"
|
||||
_ENV_SOURCE = "QA_MIN_SOURCE_SCORE"
|
||||
|
||||
|
||||
def _env_value(env_key: str, default: float) -> float:
|
||||
"""env 覆盖:值 ∈ [0,1] 才生效,否则回落默认。"""
|
||||
raw = os.environ.get(env_key, "").strip()
|
||||
if not raw:
|
||||
return default
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
return default
|
||||
if not (0.0 <= value <= 1.0):
|
||||
return default
|
||||
return value
|
||||
|
||||
|
||||
def min_vector_score() -> float:
|
||||
"""向量域下限(Qdrant score_threshold / search 预筛)。"""
|
||||
return min_vector_score.value
|
||||
|
||||
|
||||
def min_source_score() -> float:
|
||||
"""最终域下限(chat 模式 final_score 后置过滤)。"""
|
||||
return min_source_score.value
|
||||
|
||||
|
||||
min_vector_score.value = _env_value(_ENV_VECTOR, DEFAULT_MIN_VECTOR_SCORE) # type: ignore[attr-defined]
|
||||
min_source_score.value = _env_value(_ENV_SOURCE, DEFAULT_MIN_SOURCE_SCORE) # type: ignore[attr-defined]
|
||||
@@ -0,0 +1,335 @@
|
||||
"""test_score_gate.py —— 相关性下限单测(缺陷 #28 回改 v2,2026-09-11)。
|
||||
|
||||
运行(venv 内,源码树):
|
||||
cd agent/qa/src/rag/qa_v1_1_73
|
||||
python -m pytest test_score_gate.py -v
|
||||
|
||||
覆盖:
|
||||
- search(retrieval_only)模式:无 LLM 调用、无 rerank 调用;
|
||||
vector 域 0.45 预筛滤掉 0.32 垃圾、保留 0.63 正常;
|
||||
sources 的 score 字段取 vector_score(final_score=vector_score 域)。
|
||||
- chat 模式:final 域 0.25 后置过滤语义不变(5 高+25 低 -> 5、30 低 -> 0);
|
||||
Qdrant score_threshold 不传(None,保召回)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# 本文件位于 agent/qa/src/rag/qa_v1_1_73/,parents[2] = agent/qa/src
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
from rag.qa_v1_1_73 import score_gate # noqa: E402
|
||||
from rag.qa_v1_1_73.schemas import EvidenceItem # noqa: E402
|
||||
from rag.qa_v1_1_73.pipeline import QAV1Pipeline # noqa: E402
|
||||
|
||||
|
||||
def _item(
|
||||
idx: int, *, final: float = 0.0, vector: float = 0.0, graph: bool = False, key: str = ""
|
||||
) -> EvidenceItem:
|
||||
return EvidenceItem(
|
||||
evidence_id=f"E{idx}",
|
||||
kb="V16",
|
||||
source=f"src{idx}",
|
||||
relation="has",
|
||||
target=f"tgt{idx}",
|
||||
evidence=f"evidence body {idx}",
|
||||
relation_id=(key or f"rid-{idx}"),
|
||||
retrieval_methods=(["neo4j_bounded_expand"] if graph else ["qdrant_vector"]),
|
||||
vector_score=vector,
|
||||
final_score=final,
|
||||
)
|
||||
|
||||
|
||||
def _select(items, *, top_k=30, graph_min=0):
|
||||
selected, audit = QAV1Pipeline._select_graph_balanced(
|
||||
items, top_k_for_llm=top_k, graph_min_evidence=graph_min
|
||||
)
|
||||
return selected, audit
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# chat 模式 final 域 0.25(r4-28 语义不变)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_chat_all_below_source_threshold_selects_zero():
|
||||
"""30 条 final 全部低于 0.25 -> 选入 0。"""
|
||||
items = [_item(i, final=0.20) for i in range(30)]
|
||||
selected, audit = _select(items)
|
||||
assert selected == []
|
||||
assert audit["below_min_source_score"] == 30
|
||||
assert audit["min_source_score"] == score_gate.min_source_score()
|
||||
|
||||
|
||||
def test_chat_five_high_twentyfive_low_selects_only_five():
|
||||
"""5 高(final>=0.25)+ 25 低 -> 仅选入 5(保 rerank 序)。"""
|
||||
items = [_item(i, final=0.90 - i * 0.01) for i in range(5)]
|
||||
items += [_item(5 + i, final=0.20) for i in range(25)]
|
||||
selected, audit = _select(items)
|
||||
assert len(selected) == 5
|
||||
assert [item.evidence_id for item in selected] == [f"E{i}" for i in range(5)]
|
||||
assert audit["below_min_source_score"] == 25
|
||||
|
||||
|
||||
def test_chat_below_threshold_graph_candidates_excluded():
|
||||
"""低分 graph 候选不占用 graph 配额。"""
|
||||
items = [
|
||||
_item(1, final=0.30, graph=True),
|
||||
_item(2, final=0.21, graph=True),
|
||||
_item(3, final=0.95, graph=False),
|
||||
_item(4, final=0.19, graph=False),
|
||||
]
|
||||
selected, audit = _select(items, top_k=30, graph_min=1)
|
||||
assert [item.evidence_id for item in selected] == ["E1", "E3"]
|
||||
assert audit["graph_selected"] == 1
|
||||
assert audit["below_min_source_score"] == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# search / retrieval-only 模式:免 LLM、免 rerank、vector 域 0.45 预筛
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class _StubLLM:
|
||||
available = True # 可用也必须不调用(回改核心断言)
|
||||
|
||||
def complete_json(self, *args, **kwargs):
|
||||
raise AssertionError("search 模式不得调用 LLM")
|
||||
|
||||
def complete(self, *args, **kwargs):
|
||||
raise AssertionError("search 模式不得调用 LLM")
|
||||
|
||||
|
||||
class _StubDirectRetriever:
|
||||
"""捕获 retrieve 入参,供断言 Qdrant score_threshold 传递。"""
|
||||
|
||||
def __init__(self, items):
|
||||
self.items = items
|
||||
self.last_kwargs = {}
|
||||
|
||||
def retrieve(self, question, queries, hypothesis, **kwargs):
|
||||
self.last_kwargs = kwargs
|
||||
return list(self.items), {
|
||||
"qdrant": [],
|
||||
"seed_extraction": {},
|
||||
"neo4j": [],
|
||||
"dedup": {"qdrant_items": len(self.items)},
|
||||
"options": dict(kwargs),
|
||||
}
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
|
||||
def _make_pipeline(graph_schema):
|
||||
from rag.qa_v1_1_73.config import QAConfig
|
||||
|
||||
cfg = QAConfig.from_env()
|
||||
if graph_schema == "legacy":
|
||||
object.__setattr__(cfg, "graph_schema", "legacy")
|
||||
import threading
|
||||
|
||||
p = QAV1Pipeline.__new__(QAV1Pipeline)
|
||||
p.config = cfg
|
||||
p._reranker_lock = threading.Lock()
|
||||
p.llm = _StubLLM()
|
||||
p.provisional = _RaisingComposer("provisional")
|
||||
p.hypothesis_extractor = _RaisingComposer("hypothesis")
|
||||
p.query_builder = _StubQueryBuilder()
|
||||
p.reranker = _StubReranker()
|
||||
p.agentic_controller = None
|
||||
p.question_decomposer = None
|
||||
p.subquestion_orchestrator = None
|
||||
p._retriever_stub = _StubDirectRetriever(ITEMS)
|
||||
p.direct_retriever_factory = lambda: p._retriever_stub
|
||||
p.audit = _NullAudit()
|
||||
p.domain = _StubDomain()
|
||||
return p
|
||||
|
||||
|
||||
class _RaisingComposer:
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
def generate(self, *args, **kwargs):
|
||||
raise AssertionError(f"{self.name} 不得被调用")
|
||||
|
||||
def extract(self, *args, **kwargs):
|
||||
raise AssertionError(f"{self.name} 不得被调用")
|
||||
|
||||
|
||||
class _StubQueryBuilder:
|
||||
def build(self, question, provisional_answer, hypothesis):
|
||||
return {"hypothesis_lane": [], "direct": [], "concept": []}
|
||||
|
||||
|
||||
class _StubReranker:
|
||||
def rerank(self, question, items):
|
||||
raise AssertionError("search 模式不得调用 rerank")
|
||||
|
||||
|
||||
class _NullAudit:
|
||||
def new_query_id(self):
|
||||
return "qid"
|
||||
|
||||
def write(self, query_id, payload):
|
||||
return None
|
||||
|
||||
|
||||
class _StubDomain:
|
||||
def normalize_hypothesis(self, hypothesis):
|
||||
return {"entities": []}
|
||||
|
||||
def direct_entities_from_question(self, question):
|
||||
return []
|
||||
|
||||
|
||||
# 集成(answer())路径:search 免 rerank,final_score = vector_score(同域),
|
||||
# 按 TL 实测向量域构造:垃圾 0.32(< 0.45 滤掉)、正常 0.63(保留)。
|
||||
JUNK = _item(1, vector=0.32)
|
||||
GOOD = _item(2, vector=0.63)
|
||||
ITEMS = [JUNK, GOOD]
|
||||
|
||||
|
||||
def test_search_mode_no_llm_no_rerank_keeps_good_only():
|
||||
p = _make_pipeline("legacy")
|
||||
out = p.answer(
|
||||
"景甜是谁",
|
||||
retrieval_only=True,
|
||||
native_chain_enabled=False,
|
||||
progress_callback=None,
|
||||
)
|
||||
assert out["sources"] != [] # vector 0.63 保留
|
||||
titles = [src["source"] for src in out["sources"]]
|
||||
assert "src2" in titles and "src1" not in titles # 0.32 垃圾被 0.45 预筛滤掉
|
||||
assert p._retriever_stub.last_kwargs["qdrant_score_threshold"] == 0.45
|
||||
assert out["retrieval_audit"]["options"]["qdrant_score_threshold"] == 0.45
|
||||
assert out["query_plan"]["agentic_plan"]["plan_source"] == "deterministic_retrieval_only_plan"
|
||||
assert out["degraded_execution"] is False
|
||||
# 跳过 rerank 后:final_score = vector_score(TL 回改语义,与 Qdrant 阈值
|
||||
# 同域);rerank_score 未精排(0.0)。BFF normalize 的 score 取 final_score。
|
||||
for src in out["sources"]:
|
||||
assert src["rerank_score"] == 0.0
|
||||
assert src["final_score"] == src["vector_score"]
|
||||
|
||||
|
||||
def test_search_mode_use_neo4j_forced_off():
|
||||
"""回改 v3:search 模式 retriever 收到的检索选项 use_neo4j=False
|
||||
(r3 语义补完——图证据种子检索在 13M 边库上 150s+,检索模式只走 Qdrant)。
|
||||
"""
|
||||
p = _make_pipeline("legacy")
|
||||
out = p.answer(
|
||||
"arginine biosynthesis",
|
||||
retrieval_only=True,
|
||||
native_chain_enabled=False,
|
||||
progress_callback=None,
|
||||
)
|
||||
kwargs = p._retriever_stub.last_kwargs
|
||||
assert kwargs["use_neo4j"] is False
|
||||
assert kwargs["use_graph_expansion"] is False # 依赖 use_neo4j 一并关
|
||||
assert kwargs["use_qdrant"] is True
|
||||
# 即使 ablation 显式开图检索,检索模式也不得重开(性能红线)
|
||||
p2 = _make_pipeline("legacy")
|
||||
out2 = p2.answer(
|
||||
"arginine biosynthesis",
|
||||
retrieval_only=True,
|
||||
native_chain_enabled=False,
|
||||
ablation={"retrieval": {"use_neo4j": True, "use_graph_expansion": True}},
|
||||
progress_callback=None,
|
||||
)
|
||||
kwargs2 = p2._retriever_stub.last_kwargs
|
||||
assert kwargs2["use_neo4j"] is False
|
||||
assert kwargs2["use_graph_expansion"] is False
|
||||
assert out2["success"] is True
|
||||
|
||||
|
||||
def test_chat_mode_passes_no_threshold():
|
||||
p = _make_pipeline("legacy")
|
||||
object.__setattr__(p.config, "enable_bge_rerank", False)
|
||||
out = p.answer(
|
||||
"景甜是谁",
|
||||
retrieval_only=False,
|
||||
native_chain_enabled=False,
|
||||
progress_callback=None,
|
||||
)
|
||||
assert p._retriever_stub.last_kwargs["qdrant_score_threshold"] is None
|
||||
|
||||
|
||||
def test_direct_flow_retriever_threshold_passthrough():
|
||||
"""真实 retriever 的 qdrant_score_threshold 参数链路(不触发 IO)。
|
||||
|
||||
用对象伪造构造 DirectFlowRetriever 实例,直接调用 qdrant_search:
|
||||
- 传阈值时透传给 qdrant.query_points(同域向量阈值);
|
||||
- 不传阈值时 score_threshold=None(chat 保召回)。
|
||||
"""
|
||||
import rag.qa_v1_1_73.direct_flow_retriever as dfr_mod
|
||||
|
||||
class _FakeQdrant:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def query_points(self, **kwargs):
|
||||
self.calls.append(kwargs)
|
||||
from qdrant_client.http import models
|
||||
|
||||
return models.QueryResponse(points=[])
|
||||
|
||||
class _MinimalRetriever(dfr_mod.DirectFlowRetriever):
|
||||
"""绕过 __init__(不建连接),只测 qdrant_search 参数透传。"""
|
||||
|
||||
def __init__(self):
|
||||
self.qdrant = _FakeQdrant()
|
||||
self.collection = "plant_srt_pc2025_v3_20260817_normalized"
|
||||
self.embed_url = ""
|
||||
self.embed_model = ""
|
||||
|
||||
def embed(self, text):
|
||||
return [0.0] * 1024
|
||||
|
||||
r = _MinimalRetriever()
|
||||
r.qdrant_search("景甜是谁", 10, score_threshold=0.45)
|
||||
assert r.qdrant.calls[-1]["score_threshold"] == 0.45
|
||||
assert r.qdrant.calls[-1]["collection_name"] == r.collection
|
||||
|
||||
r.qdrant_search("arginine", 10)
|
||||
assert r.qdrant.calls[-1]["score_threshold"] is None
|
||||
|
||||
|
||||
def test_item_from_payload_derives_citation_from_paper_id():
|
||||
"""回改 v4 根因单测:Qdrant payload 无 pmid/pmcid/doi 时从 paper_id
|
||||
派生 DOI——chat 模式 public_citation 过滤不得把有正文的 qdrant 条目
|
||||
全滤掉(此前 rerank 输入全是无 evidence 的 neo4j 图断言,bge 全
|
||||
0.02-0.047,final 门 0.25 全拒 sources=[])。"""
|
||||
import rag.qa_v1_1_73.direct_flow_retriever as dfr_mod
|
||||
|
||||
r = dfr_mod.DirectFlowRetriever.__new__(dfr_mod.DirectFlowRetriever)
|
||||
r.paper_metadata = {}
|
||||
# 内部 DOI 形态
|
||||
it = r.item_from_payload(
|
||||
{"paper_id": "10.3390_f12050650", "source": "s", "relation": "r",
|
||||
"target": "t", "evidence": "e"},
|
||||
"qdrant_vector", "q",
|
||||
)
|
||||
assert it.doi == "10.3390/f12050650"
|
||||
assert it.retrieval_methods == ["qdrant_vector"]
|
||||
# doi: 前缀形态
|
||||
it2 = r.item_from_payload(
|
||||
{"paper_id": "doi:10.1002/ps.6526", "source": "s", "relation": "r",
|
||||
"target": "t", "evidence": "e"},
|
||||
"qdrant_vector", "q",
|
||||
)
|
||||
assert it2.doi == "10.1002/ps.6526"
|
||||
# 派生出的 DOI 能通过 public_citation 校验(过滤不再误杀)
|
||||
from rag.qa_v1_1_73.citation_contract import public_citation
|
||||
cite = public_citation(evidence_id="", pmid="", pmcid="", doi=it.doi)
|
||||
assert cite == "[DOI: 10.3390/f12050650]"
|
||||
|
||||
|
||||
def test_env_overrides(monkeypatch):
|
||||
monkeypatch.setenv("QA_MIN_VECTOR_SCORE", "0.5")
|
||||
assert score_gate._env_value("QA_MIN_VECTOR_SCORE", 0.45) == 0.5
|
||||
monkeypatch.setenv("QA_MIN_VECTOR_SCORE", "abc")
|
||||
assert score_gate._env_value("QA_MIN_VECTOR_SCORE", 0.45) == 0.45
|
||||
monkeypatch.setenv("QA_MIN_SOURCE_SCORE", "1.5")
|
||||
assert score_gate._env_value("QA_MIN_SOURCE_SCORE", 0.25) == 0.25
|
||||
monkeypatch.delenv("QA_MIN_VECTOR_SCORE", raising=False)
|
||||
monkeypatch.delenv("QA_MIN_SOURCE_SCORE", raising=False)
|
||||
@@ -334,6 +334,60 @@ VSTORE_EMBED_MODEL=bge-m3:567m
|
||||
- 短句(12 字)单次嵌入 4.1s;780 字长文本 1.0s(2026-09-10 实测,模型常驻后);
|
||||
- 单篇文档全量嵌入(多 chunk 累计)约 52s 属历史实测值——功能正常,纯性能差距,无 GPU 机器属预期。
|
||||
|
||||
### 5e. 论文图片本地源部署(figures 双源,2026-09-11 起部署必做)
|
||||
|
||||
> BFF 答案配图走双源:本地 MinerU 图片优先,PMC 外网兜底。本地源三个条件缺一
|
||||
> 即**静默降级**为纯 PMC(无报错、无日志,唯一症状是配图变少或全走外网)。
|
||||
|
||||
1. **env 两键**(sdk/.env,与 BFF 同 env,BFF 重启后生效):
|
||||
|
||||
```bash
|
||||
PLANTAI_FIGURES_MINERU_ROOT=/srv/plantai/data/evaluation/methodology_upload_jobs
|
||||
PLANTAI_FIGURES_REGISTRY=/srv/plantai/data/knowledge_base/paper_metadata_registry_v1_20260717/paper_metadata_registry_full.jsonl
|
||||
```
|
||||
|
||||
2. **registry 文件传输**:源机 `knowledge_base/paper_metadata_registry_v1_20260717/
|
||||
paper_metadata_registry_full.jsonl`(16.9MB)经 p2p 传到目标机上述路径;该文件
|
||||
是 pmcid→paper_id 映射,缺失时本地索引即使有图也映射不出 PMCID。
|
||||
3. **BFF 重启**:必须手动 kill 旧 BFF(见 §5f 陷阱)。
|
||||
|
||||
**验证**:用 /tmp 假任务目录受控测试——`demo_job/mineru/paper/txt/*_content_list.json`
|
||||
(type=image 条目)+ `methodology_papers.csv`(paper_id 取 registry 内带 pmcid 的),
|
||||
加载 figures 模块 `_build_local_index()` 应产出 `source=local` 条目;或部署后真实上传
|
||||
一篇 registry 内论文后提问验证。
|
||||
|
||||
**已知限制(2026-09-11 实测)**:本地图片只对「paper_id 在 registry 中」的任务命中;
|
||||
历史 demo_paper/e2e_check 测试任务不产生本地图片索引。
|
||||
|
||||
### 5f. 部署陷阱:BFF 重启必须手动 kill(start_all 不替换在跑进程)
|
||||
|
||||
**start_all.sh 不杀已运行的 BFF**:8090 被旧进程占住时,脚本内 uvicorn 新进程因端口
|
||||
占用静默退出,自检看到的仍是旧进程(新 env 不生效)。三步:
|
||||
|
||||
```bash
|
||||
pgrep -f "uvicorn app.main:app" | xargs kill # 1. 杀旧 BFF
|
||||
cd install/sdk && set -a && . .env && set +a # 2. source env
|
||||
setsid venv/bin/python -m uvicorn app.main:app --host 127.0.0.1 --port 8090 > /tmp/plantai-sdk.log 2>&1 < /dev/null &
|
||||
tr "\0" "\n" < /proc/$(pgrep -f "port 8090" | head -1)/environ | grep FIGURES # 3. 验证新 env 注入
|
||||
```
|
||||
|
||||
### 5g. r4 部署清单(2026-09-12,六项缺陷修复轮)
|
||||
|
||||
1. **QA 双域相关性阈值**(qa/env/prod.env,可选;不设则用代码默认值):
|
||||
|
||||
```bash
|
||||
QA_MIN_VECTOR_SCORE=0.45 # 检索模式向量域(Qdrant score_threshold 同源)
|
||||
QA_MIN_SOURCE_SCORE=0.25 # chat 模式最终域(rerank 后过滤)
|
||||
```
|
||||
|
||||
2. **MinerU 依赖 accelerate**:目标机 mineru 环境需 `pip install accelerate`——缺失
|
||||
症状为启动期失败「requires accelerate」(r4 的自动重试会报中文可读错误但不会自愈)。
|
||||
3. **figures 两键 + registry**:见 §5e。
|
||||
4. **vstore 内容去重**:vault.add 按 SHA256 覆盖旧 doc,无需 env;存量重复数据如需清理
|
||||
走 rebuild-index/rebuild-chunks 端点。
|
||||
5. **r4 前端 dist**:ui/dist 重构建(含 intent 词典「总结/概括/归纳」、KgPane 空态提示、
|
||||
SourceRow 标题 fallback、DocPane 状态机修复)。
|
||||
|
||||
## 6. 部署后完整质检(agent 逐条执行并记录结果,全过才算部署成功)
|
||||
|
||||
```bash
|
||||
@@ -477,4 +531,5 @@ NEO4J_URI=bolt://127.0.0.1:19888 NEO4J_PASSWORD='<密码>' bash deploy/qa-db/cre
|
||||
4. **构建机与目标机若为同一台机器**(内网 192.168.0.175 现状),部署就是替换现有 `plantai-rag-new-deploy-unified/` 部署树;执行前先备份、执行后按 §6 全量质检。
|
||||
5. 本手册基于 2026-09-08 服务器实测编写;如工具链或目录结构有变,先对照仓库实际脚本修正本手册再执行。
|
||||
6. **当前生产服务器(192.168.0.175)仍是 0.0.0.0:8090 直连形态,尚未执行 §5b 的 nginx 迁移**——启动脚本默认值已改为 127.0.0.1,但现网进程仍是旧绑定;迁移时按 §5b 顺序执行(先 nginx 后改绑),不要直接无参重启 bff_restart.sh 以免客户访问断档。
|
||||
8. **start_all.sh 不会替换在跑的 BFF**(2026-09-11 实测):BFF 更新 env/代码后必须手动 kill + 重启,详见 §5f。
|
||||
7. **175 上 QA 数据库 docker 容器(27333/19788)为其他项目共享服务,不随 PlantAI 部署流程管理**;PlantAI-rag-new 一律走裸进程(§5.0/5.0b);遗留库(26333/18788)现网无引用,清理需用户单独命令。
|
||||
|
||||
@@ -18,7 +18,11 @@ route(text, has_attachments, has_active_artifact) -> "qa" | "docgen" | "edit" |
|
||||
|
||||
### 3. docgen(写文档意图)
|
||||
条件:text 含任一 WRITE_VERB 且含任一 DOC_TYPE
|
||||
- WRITE_VERB = [写, 撰写, 拟, 起草, 编制, 生成, 编写, 拟定, 草拟, 制定]
|
||||
- WRITE_VERB = [写, 撰写, 拟, 起草, 编制, 生成, 编写, 拟定, 草拟, 制定, 总结, 概括, 归纳](r4 #52 新增后三词)
|
||||
- 动词自指保护(r4 #52 新增):规则 3 判定时,若 text 命中的 DOC_TYPE 恰好等于命中的 WRITE_VERB 本身
|
||||
(如「总结一下植物光合作用的过程」——「总结」既是动词又是文档类型词),不算写文档意图,
|
||||
继续向下判定。「写一篇总结」(动词=写,类型=总结)与「总结这篇论文」(动词=总结,类型=论文)
|
||||
不受影响。
|
||||
- DOC_TYPE = [论文, 报告, 通知, 通告, 公告, 请示, 批复, 函, 申请书, 申请, 申报书, 标书, 合同, 协议, 方案, 计划, 总结, 汇报, 讲话稿, 致辞, 证明, 纪要, 简报, 材料, 说明书, 指南, 文档, 申报材料, 工作总结, 调研报告, 可行性报告, 项目书, 策划书, 演讲稿, 新闻稿, 文案, 简历, 制度, 规定, 办法, 细则, 章程, 声明, 邀请函, 感谢信, 慰问信, 介绍信, 推荐信, 倡议书, 述职报告, 开题报告, 结题报告, 实验报告, 分析报告, 会议纪要]
|
||||
- 明确排除(不属于 DOC_TYPE,防误判):介绍, 内容, 代码, 诗, 作业, 问题, 建议, 想法
|
||||
|
||||
@@ -28,7 +32,7 @@ route(text, has_attachments, has_active_artifact) -> "qa" | "docgen" | "edit" |
|
||||
|
||||
### 5. 其余一律 "qa"(漏判安全原则:不命中绝不进 docgen)
|
||||
|
||||
## 用例表(30 条,前后端实现与测试共用同一张表)
|
||||
## 用例表(34 条,前后端实现与测试共用同一张表)
|
||||
|
||||
| # | 输入 text | 附件 | 活动文档 | 期望 |
|
||||
|---|---|---|---|---|
|
||||
@@ -62,6 +66,10 @@ route(text, has_attachments, has_active_artifact) -> "qa" | "docgen" | "edit" |
|
||||
| 28 | 帮我写个会议通知模板 | 否 | 是 | docgen |
|
||||
| 29 | 给这个通知加个结尾 | 否 | 是 | edit |
|
||||
| 30 | 增加一段关于风险的描述 | 否 | 是 | edit |
|
||||
| 31 | 帮我总结论文 | 是 | 否 | docgen |
|
||||
| 32 | 总结一下植物光合作用的过程 | 否 | 否 | qa |
|
||||
| 33 | 概括一下这个通知 | 是 | 是 | docgen |
|
||||
| 34 | 归纳这篇文章 | 是 | 否 | docgen |
|
||||
|
||||
## 一致性要求
|
||||
- dev1 的 sdk/app/services/docgen/intent.py 与 dev3 的 ui/src/intent.js 必须按本表逐条通过
|
||||
|
||||
@@ -983,7 +983,11 @@ async def export_artifact(
|
||||
html, _rendered_css = render_html(
|
||||
markdown, title=title, key=key, letterhead=letterhead
|
||||
)
|
||||
data = await docx_from_html_document(html, title)
|
||||
# 缺陷 #47a(2026-09-11):css 一并传给 DOCX 转换,
|
||||
# 红头/标题样式与预览/PDF 同源;frozen_css 优先(同 PDF 路径)
|
||||
data = await docx_from_html_document(
|
||||
html, title, css=frozen_css or _rendered_css
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"docgen: HTML→DOCX 转换失败(%s),回退 md2docx 旧路径", exc
|
||||
|
||||
@@ -108,7 +108,7 @@ def _reset_mocks(monkeypatch):
|
||||
async def rebuild_ok():
|
||||
return {"rebuilt": True}
|
||||
|
||||
async def docx_html_ok(html: str, title: str):
|
||||
async def docx_html_ok(html: str, title: str, css: str = ""):
|
||||
return b"DOCX-HTML-BYTES"
|
||||
|
||||
def render_ok(markdown, *, title, key="document", letterhead=""):
|
||||
|
||||
@@ -31,13 +31,17 @@ async def pdf_from_markdown(markdown: str, title: str) -> bytes:
|
||||
return await anyio.to_thread.run_sync(_pdf_bytes, markdown, title)
|
||||
|
||||
|
||||
async def docx_from_html_document(html: str, title: str) -> bytes:
|
||||
"""HTML → docx(缺陷 #47a,2026-09-10):与 converter 其它函数同签名,
|
||||
html2docx 是 CPU 同步转换,to_thread 避免阻塞事件循环。"""
|
||||
async def docx_from_html_document(html: str, title: str, css: str = "") -> bytes:
|
||||
"""HTML → docx(缺陷 #47a,2026-09-10/11):与 converter 其它函数同签名,
|
||||
html2docx 是 CPU 同步转换,to_thread 避免阻塞事件循环。
|
||||
|
||||
css:render_html 返回的模板 CSS(缺陷 #47a 同源解析,2026-09-11)。
|
||||
为空时 html2docx 只解析 html 内嵌 <style> 并回退常量(历史行为不变)。
|
||||
"""
|
||||
from .html2docx import docx_from_html_document as _sync
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
out = _sync(html, title=title, out_path=Path(tmp) / "document.docx")
|
||||
out = _sync(html, title=title, out_path=Path(tmp) / "document.docx", css=css)
|
||||
return out.read_bytes()
|
||||
|
||||
|
||||
|
||||
@@ -35,12 +35,16 @@ from docx.oxml import OxmlElement
|
||||
from docx.oxml.ns import qn
|
||||
from docx.shared import Inches, Pt, RGBColor
|
||||
|
||||
from . import html_css # 缺陷 #47a:模板 CSS 解析(预览同源)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 字体:正文宋体 / 标题黑体(与 md2docx 一致;Windows 中文环境标准)
|
||||
_FONT_BODY = "宋体"
|
||||
_FONT_HEADING = "黑体"
|
||||
|
||||
# 缺陷 #47a(2026-09-11):红头/标题样式从 render_html 产物(<style> 内嵌
|
||||
# CSS)解析,与预览/PDF 同源;以下仅为解析失败时的回退常量(保留 r3 行为)。
|
||||
_HEADING_SIZES = {1: 22, 2: 16, 3: 14}
|
||||
_LETTERHEAD_SIZE = 22.0
|
||||
_IMAGE_MAX_WIDTH_IN = 6.0 # A4 正文可用宽度约 6 英寸
|
||||
@@ -60,6 +64,9 @@ _LEAF_BLOCKS = {
|
||||
# 遍历时整体跳过的标签
|
||||
_SKIP_TAGS = {"script", "style", "meta", "link", "title", "head"}
|
||||
|
||||
# 当前文档解析出的标题字号(docx_from_html_document 设置;单线程转换,无并发问题)
|
||||
_ACTIVE_HEADING_SIZES: dict[int, float] = dict(_HEADING_SIZES)
|
||||
|
||||
|
||||
def _set_run_font(run, font_name: str, size_pt: float | None = None,
|
||||
bold: bool | None = None, italic: bool | None = None,
|
||||
@@ -192,7 +199,7 @@ def _add_rich_block(doc: Document, node: Tag, list_level: int = 0) -> None:
|
||||
if level:
|
||||
para = doc.add_heading("", level=level)
|
||||
run = para.add_run(node.get_text(" ", strip=True))
|
||||
_set_run_font(run, _FONT_HEADING, _HEADING_SIZES[level], bold=True)
|
||||
_set_run_font(run, _FONT_HEADING, _ACTIVE_HEADING_SIZES.get(level, _HEADING_SIZES[level]), bold=True)
|
||||
para.paragraph_format.space_before = Pt(10)
|
||||
return
|
||||
|
||||
@@ -334,16 +341,24 @@ def _add_image(doc: Document, img: Tag) -> None:
|
||||
_set_run_font(run, _FONT_BODY, 10.5)
|
||||
|
||||
|
||||
def docx_from_html_document(html: str, title: str = "", out_path: str | Path | None = None) -> Path:
|
||||
def docx_from_html_document(html: str, title: str = "", out_path: str | Path | None = None, *, css: str = "") -> Path:
|
||||
"""HTML → docx。返回写入路径(out_path 为 None 时按 title 命名到当前目录)。
|
||||
|
||||
- 红头:.doc-letterhead 红字(RGBColor(255,0,0))居中 22pt 黑体
|
||||
- h1-h3:黑体 22/16/14pt 加粗
|
||||
- 红头:.doc-letterhead 颜色/字号从 render_html 产物解析(css 参数 +
|
||||
html 内嵌 <style>),解析失败回退历史常量(FF0000/22pt)
|
||||
- h1-h3:黑体,字号同样解析优先、回退 22/16/14pt
|
||||
- 表格 Table Grid + 表头灰底;figure/img 内嵌(失败落 alt)
|
||||
- 空 HTML 产出空文档(不抛)
|
||||
"""
|
||||
doc = Document()
|
||||
soup = BeautifulSoup(html or "", "html.parser")
|
||||
# 缺陷 #47a:红头/标题样式同源解析(render_html 产物 -> DOCX 与预览一致)
|
||||
styles = html_css.parse_css_styles(html)
|
||||
if css:
|
||||
styles = html_css.merge_css_text(styles, css)
|
||||
letterhead_color, letterhead_size = html_css.letterhead_style(styles)
|
||||
global _ACTIVE_HEADING_SIZES
|
||||
_ACTIVE_HEADING_SIZES = html_css.heading_sizes(styles)
|
||||
|
||||
# 标题:<title> 优先(render_html 已回填 {{TITLE}})
|
||||
if not title:
|
||||
@@ -361,8 +376,8 @@ def docx_from_html_document(html: str, title: str = "", out_path: str | Path | N
|
||||
para = doc.add_paragraph()
|
||||
para.alignment = WD_ALIGN_PARAGRAPH.CENTER
|
||||
run = para.add_run(letterhead)
|
||||
_set_run_font(run, _FONT_HEADING, _LETTERHEAD_SIZE, bold=True,
|
||||
color=RGBColor(255, 0, 0))
|
||||
_set_run_font(run, _FONT_HEADING, letterhead_size, bold=True,
|
||||
color=RGBColor(*letterhead_color))
|
||||
letterhead_el.decompose()
|
||||
|
||||
# 主内容:main 优先;递归下钻包裹容器(doc-body 等),保文档序
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
"""services/docgen/html_css.py —— 自家 render_html 产物的 CSS 解析(缺陷 #47a,2026-09-11)。
|
||||
|
||||
render_html 返回 (html, css):html 是模板 wrapper(css 不内联,
|
||||
css 由调用方单独持有——PDF 路径与预览拼装时注入)。html2docx 的输入
|
||||
原本只有 html,缺陷 #47a 后调用方(docgen 路由)把 css 一并传入
|
||||
(converter.docx_from_html_document(html, title, css)),本模块负责:
|
||||
- 解析 html 内嵌 <style> 块(预览注入后的产物形态);
|
||||
- 合并调用方传入的模板 CSS 文本(后出现者优先,与调用方拼装语义一致);
|
||||
- 解析不到声明时回退历史硬编码常量(红头 FF0000 22pt、h1/h2/h3 22/16/14pt),
|
||||
保证无样式 HTML / 解析失败的行为与 r3 一致。
|
||||
|
||||
解析只覆盖 CSS 安全子集(块流/字体/颜色,见 templates 各 css 文件头注释)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---- 历史硬编码常量(解析不出声明时的最终回退,行为与 r3 相同)----
|
||||
_LEGACY_LETTERHEAD_COLOR = (255, 0, 0) # FF0000
|
||||
_LEGACY_LETTERHEAD_SIZE_PT = 22.0
|
||||
_LEGACY_HEADING_SIZES_PT = {1: 22.0, 2: 16.0, 3: 14.0}
|
||||
|
||||
_ColorRGB = tuple[int, int, int]
|
||||
|
||||
# rgb(r, g, b) / rgba(...) / #rgb / #rrggbb(容忍空白)
|
||||
_RGB_RE = re.compile(
|
||||
r"rgba?\(\s*(\d{1,3})\s*,\s*(\d{1,3})\s*,\s*(\d{1,3})", re.IGNORECASE
|
||||
)
|
||||
_HEX6_RE = re.compile(r"#([0-9a-fA-F]{6})\b")
|
||||
_HEX3_RE = re.compile(r"#([0-9a-fA-F]{3})\b")
|
||||
|
||||
# 1.2e3 / 12.5 / 12 后接 px|pt|em|rem(匹配声明值中的尺寸)
|
||||
_SIZE_RE = re.compile(
|
||||
r"(\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)\s*(px|pt|em|rem)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _declarations(style: Tag) -> list[dict[str, str]]:
|
||||
"""解析 <style> 文本为声明块列表:{selector, prop: value...}。
|
||||
|
||||
组合选择器("h3, h4, h5, h6")按逗号拆分为独立条目,
|
||||
让 h3/h4… 各自可被单独查取(模板 CSS 只含简单元素/类选择器)。
|
||||
"""
|
||||
out: list[dict[str, str]] = []
|
||||
for match in re.finditer(r"([^{}]+)\{([^{}]*)\}", style.get_text() or ""):
|
||||
selectors = [sel.strip() for sel in match.group(1).split(",") if sel.strip()]
|
||||
if not selectors:
|
||||
continue
|
||||
block: dict[str, str] = {}
|
||||
for decl in re.findall(r"([\w-]+)\s*:\s*([^;]+);", match.group(2)):
|
||||
block[decl[0].strip().lower()] = decl[1].strip()
|
||||
if not block:
|
||||
continue
|
||||
for selector in selectors:
|
||||
out.append({"selector": selector, "block": dict(block)})
|
||||
return out
|
||||
|
||||
|
||||
def parse_rgb(value: str) -> _ColorRGB | None:
|
||||
"""CSS 颜色 → (r,g,b);支持 #rrggbb/#rgb/rgb()/rgba(),其余返回 None。"""
|
||||
if not value:
|
||||
return None
|
||||
m = _HEX6_RE.search(value)
|
||||
if m:
|
||||
h = m.group(1)
|
||||
return (int(h[0:2], 16), int(h[2:4], 16), int(h[4:6], 16))
|
||||
m = _HEX3_RE.search(value)
|
||||
if m:
|
||||
h = m.group(1)
|
||||
return (int(h[0], 16) * 17, int(h[1], 16) * 17, int(h[2], 16) * 17)
|
||||
m = _RGB_RE.search(value)
|
||||
if m:
|
||||
r, g, b = (max(0, min(255, int(c))) for c in m.groups())
|
||||
return (r, g, b)
|
||||
return None
|
||||
|
||||
|
||||
def parse_size_pt(value: str) -> float | None:
|
||||
"""CSS 尺寸 → pt(px 按 96dpi:1px = 0.75pt);非尺寸返回 None。"""
|
||||
if not value:
|
||||
return None
|
||||
m = _SIZE_RE.search(value)
|
||||
if not m:
|
||||
return None
|
||||
number = float(m.group(1))
|
||||
unit = m.group(2).lower()
|
||||
if unit == "px":
|
||||
return number * 0.75
|
||||
if unit in ("pt", "em", "rem"):
|
||||
return number
|
||||
return None
|
||||
|
||||
|
||||
def parse_css_styles(html: str) -> dict[str, dict[str, str]]:
|
||||
"""HTML 内嵌 <style> 块 → {selector: {prop: value}}。
|
||||
|
||||
多个 <style> 按出现顺序合并,后出现者优先(与浏览器级联一致)。
|
||||
无 <style> 时返回空 dict(调用方回退常量)。
|
||||
"""
|
||||
merged: dict[str, dict[str, str]] = {}
|
||||
if not html:
|
||||
return merged
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
for style in soup.find_all("style"):
|
||||
for decl in _declarations(style):
|
||||
block = merged.setdefault(decl["selector"], {})
|
||||
block.update(decl["block"])
|
||||
return merged
|
||||
|
||||
|
||||
def merge_css_text(
|
||||
styles: dict[str, dict[str, str]], css: str
|
||||
) -> dict[str, dict[str, str]]:
|
||||
"""把调用方持有的模板 CSS 文本(render_html 返回的 css)并入解析结果。
|
||||
|
||||
css 文本通常不含 <style> 标签,这里包一层再走同一解析器;
|
||||
后出现者优先(模板 CSS 覆盖 html 内嵌声明,与调用方拼装语义一致)。
|
||||
"""
|
||||
if not css:
|
||||
return styles
|
||||
merged = dict(styles)
|
||||
wrapped = BeautifulSoup(f"<style>{css}</style>", "html.parser")
|
||||
for decl in _declarations(wrapped.style):
|
||||
block = merged.setdefault(decl["selector"], {})
|
||||
block.update(decl["block"])
|
||||
return merged
|
||||
|
||||
|
||||
def letterhead_style(styles: dict[str, dict[str, str]]) -> tuple[_ColorRGB, float]:
|
||||
"""红头样式:(rgb 颜色, 字号 pt)。解析不到时回退历史常量。"""
|
||||
block = styles.get(".doc-letterhead") or {}
|
||||
color = parse_rgb(block.get("color", "")) or _LEGACY_LETTERHEAD_COLOR
|
||||
size = parse_size_pt(block.get("font-size", ""))
|
||||
if size is None:
|
||||
size = _LEGACY_LETTERHEAD_SIZE_PT
|
||||
return color, size
|
||||
|
||||
|
||||
def heading_sizes(styles: dict[str, dict[str, str]]) -> dict[int, float]:
|
||||
"""h1-h3 字号(pt):解析 h1.doc-title、h2、h3 声明;缺失回落历史常量。"""
|
||||
h1 = styles.get("h1.doc-title") or styles.get("h1") or {}
|
||||
h2 = styles.get("h2") or {}
|
||||
h3 = styles.get("h3") or {}
|
||||
parsed = {
|
||||
1: parse_size_pt(h1.get("font-size", "")),
|
||||
2: parse_size_pt(h2.get("font-size", "")),
|
||||
3: parse_size_pt(h3.get("font-size", "")),
|
||||
}
|
||||
return {
|
||||
level: parsed[level] if parsed[level] is not None else _LEGACY_HEADING_SIZES_PT[level]
|
||||
for level in (1, 2, 3)
|
||||
}
|
||||
@@ -20,7 +20,8 @@ FORMAT_WORD = ["pdf", "docx", "doc", "word"] # 英文不区分大小写(conta
|
||||
EDIT_WORD = ["改", "修改", "调整", "删", "删除", "加一段", "补充", "润色",
|
||||
"缩写", "扩写", "重写", "改写", "优化", "加个", "增加一段"]
|
||||
|
||||
WRITE_VERB = ["写", "撰写", "拟", "起草", "编制", "生成", "编写", "拟定", "草拟", "制定"]
|
||||
WRITE_VERB = ["写", "撰写", "拟", "起草", "编制", "生成", "编写", "拟定", "草拟", "制定",
|
||||
"总结", "概括", "归纳"] # r4 #52(2026-09-11)
|
||||
|
||||
DOC_TYPE = [
|
||||
"论文", "报告", "通知", "通告", "公告", "请示", "批复", "函", "申请书",
|
||||
@@ -71,8 +72,16 @@ def route(text: str, has_attachments: bool, has_active_artifact: bool) -> str:
|
||||
return "edit"
|
||||
|
||||
# 3. docgen:写类动词 + 文档类型词
|
||||
if _contains_any(text, WRITE_VERB) and _has_doc_type(text):
|
||||
return "docgen"
|
||||
# r4 #52(2026-09-11)动词自指保护:当唯一命中的 DOC_TYPE 恰好等于
|
||||
# 命中的动词本身(如「总结一下植物光合作用的过程」——「总结」既是动词
|
||||
# 又是文档类型词),不视为写文档意图(继续向下判定)。「写一篇总结」
|
||||
# (动词=写,类型=总结)与「总结这篇论文」(动词=总结,类型=论文)
|
||||
# 不受影响。规则详见 docs/team/cases_intent.md 用例 31-34。
|
||||
_matched_verbs = [w for w in WRITE_VERB if w in text]
|
||||
_matched_types = [w for w in DOC_TYPE if w in text]
|
||||
if _matched_verbs and _matched_types and _has_doc_type(text):
|
||||
if set(_matched_types) != set(_matched_verbs):
|
||||
return "docgen"
|
||||
|
||||
# 4. docgen:附件信号(拖入文件 + 写类动词)
|
||||
if has_attachments and _contains_any(text, WRITE_VERB):
|
||||
|
||||
@@ -48,7 +48,7 @@ PROXY_PATH = "/api/figures/proxy?pmcid={pmcid}&url={url}"
|
||||
|
||||
_cache: dict[str, dict[str, Any]] = {} # pmcid -> {ts, figures, license}
|
||||
_local_index: dict[str, list[dict]] | None = None # pmcid数字 -> 本地图片清单
|
||||
EXTERNAL_TIMEOUT = 5.0
|
||||
EXTERNAL_TIMEOUT = 15.0 # 176 实测 EBI 2-8s 波动,5s 必超时(2026-09-12 S2 部署质检)
|
||||
CACHE_TTL = 3600.0
|
||||
|
||||
|
||||
@@ -160,7 +160,7 @@ def _job_pmcid(job_dir: Path, pmcid_to_paper: dict[str, str]) -> str | None:
|
||||
# PMC 外网解析
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def _epmc_figures(client: httpx.AsyncClient, pmcid: str) -> tuple[list[dict], str]:
|
||||
async def _epmc_figures_once(client: httpx.AsyncClient, pmcid: str) -> tuple[list[dict], str]:
|
||||
"""返回 (figures, license)。任何失败返回 ([], "")。"""
|
||||
try:
|
||||
r = await client.get(EPMC_FULLTEXT.format(pmcid=pmcid), timeout=EXTERNAL_TIMEOUT)
|
||||
@@ -227,6 +227,17 @@ async def _epmc_figures(client: httpx.AsyncClient, pmcid: str) -> tuple[list[dic
|
||||
return [], ""
|
||||
|
||||
|
||||
async def _epmc_figures(client: httpx.AsyncClient, pmcid: str) -> tuple[list[dict], str]:
|
||||
"""PMC 解析重试包装:2026-09-12 部署质检发现 176 出网到 EBI 冷连接
|
||||
首次 TLS 握手 10s+ 超时、重连即秒回(实测 1.6-2.2s),失败重试一次。"""
|
||||
for attempt in (1, 2):
|
||||
figs, license_ = await _epmc_figures_once(client, pmcid)
|
||||
if figs or attempt == 2:
|
||||
return figs, license_
|
||||
logger.info("PMC 首次解析失败(%s)——重试一次(176 冷连接实测)", pmcid)
|
||||
return [], ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 对外 API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -20,6 +20,16 @@ def pick(obj: dict, *names: str, default=None):
|
||||
# ---------------------------------------------------------------------------
|
||||
# §8 Source 归一化(缺失字段键必须存在,值为 null)
|
||||
# ---------------------------------------------------------------------------
|
||||
# title 回填优先级(缺陷 #27,2026-09-11):title/paper_title 缺失时,
|
||||
# 依次从以下字段取可读文本(论文标题优先于实体名),全部缺失才保持 null。
|
||||
# 排除 content/evidence/text/snippet 正文字段——正文片段不是标题。
|
||||
_TITLE_FALLBACK_TOPLEVEL = (
|
||||
"publication_title", "article_title", "journal_title", "name",
|
||||
)
|
||||
_TITLE_FALLBACK_METADATA = (
|
||||
"paper_title", "publication_title", "article_title", "journal_title",
|
||||
"source_entity", "name", "source",
|
||||
)
|
||||
|
||||
def normalize_source(item: dict[str, Any] | None) -> dict[str, Any]:
|
||||
"""单个 source item → API.md §8 形态(所有键存在,缺失为 null)。
|
||||
@@ -41,6 +51,23 @@ def normalize_source(item: dict[str, Any] | None) -> dict[str, Any]:
|
||||
|
||||
source = pick(item, "source") # 断言主体实体名(上游语义)
|
||||
title = pick(item, "title", "paper_title")
|
||||
# 缺陷 #27:title 为空时按固定优先级回填——
|
||||
# 顶层标题类字段 → metadata 标题类字段 → 顶层 source(断言主体实体名)。
|
||||
# 正文字段绝不参与(正文片段不是标题)。
|
||||
if not title:
|
||||
for _key in _TITLE_FALLBACK_TOPLEVEL:
|
||||
_candidate = pick(item, _key)
|
||||
if _candidate:
|
||||
title = _candidate
|
||||
break
|
||||
if not title and isinstance(pick(item, "metadata"), dict):
|
||||
for _key in _TITLE_FALLBACK_METADATA:
|
||||
_candidate = pick(item["metadata"], _key)
|
||||
if _candidate:
|
||||
title = _candidate
|
||||
break
|
||||
if not title:
|
||||
title = source
|
||||
|
||||
doi = pick(item, "doi")
|
||||
url = pick(item, "url", "doi_url")
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
BFF 直连 DeepSeek 生成 300 字内带 [资料n] 引用的 markdown 短答:
|
||||
- ask_private(question, *, top_k=8, min_score=0.30) -> dict
|
||||
检索 chunk 索引 → 空则 {hit:false, mode:"empty", answer:"", items:[]}(不调 LLM)
|
||||
→ 同 doc 按 score 序去重(每 doc ≤3 块)再取 top_k
|
||||
→ 先按 doc_id 聚合(doc 级 max score 排序取 top 文档)再取各 doc 内
|
||||
最相关块(每 doc ≤3,缺陷 #52c)
|
||||
→ 上下文拼 `[资料n]《name》(相关度 x.xx)\nsnippet`(12k 预算截断)
|
||||
→ complete_text(system 约束, temperature=0.2, max_tokens=800, timeout=40)
|
||||
→ {hit:true, mode:"vector", answer, items:[...带 ref], elapsed_ms}
|
||||
@@ -65,24 +66,73 @@ def _safe_float(value: Any) -> bool:
|
||||
|
||||
|
||||
def _pick_items(items: list[dict[str, Any]], top_k: int) -> list[dict[str, Any]]:
|
||||
"""同 doc 按 score 序去重(每 doc ≤ PRIVATE_QA_PER_DOC_MAX 块),再取 top_k。
|
||||
"""先按 doc_id 聚合再取块(缺陷 #52c):单一文档不再占满全部名额。
|
||||
|
||||
输入已按 score 降序(vstore faiss 顺序);同 doc 靠前块优先。
|
||||
原实现按输入 score 序逐条取(每 doc ≤ PRIVATE_QA_PER_DOC_MAX 块)直到
|
||||
top_k——重复上传的作者页块(得分几乎相同)会占满 top-8、正文块进不了
|
||||
上下文。新语义:
|
||||
1. 按 doc_id 分组,doc 级得分 = 组内 max score;
|
||||
2. doc 级得分降序取前 top_k 个文档;
|
||||
3. 每选中文档取组内得分最高的 ≤ PRIVATE_QA_PER_DOC_MAX 个块,
|
||||
且组内近重复块(内容指纹相同,重复上传残留产物)只取 1 条——
|
||||
否则「同内容 200 条作者页块」仍会把该 doc 的 3 个名额占满、
|
||||
正文块永远进不了上下文;
|
||||
4. 全部选中块按 score 降序稳定排序、截断到 top_k 返回(_build_context
|
||||
的 [资料n] 编号与 items 顺序不变,回归语义保留)。
|
||||
"""
|
||||
picked: list[dict[str, Any]] = []
|
||||
per_doc: dict[str, int] = {}
|
||||
groups: dict[str, list[dict[str, Any]]] = {}
|
||||
doc_score: dict[str, float] = {}
|
||||
doc_order: list[str] = []
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
doc_id = str(item.get("doc_id") or "")
|
||||
used = per_doc.get(doc_id, 0)
|
||||
if used >= PRIVATE_QA_PER_DOC_MAX:
|
||||
continue
|
||||
per_doc[doc_id] = used + 1
|
||||
picked.append(dict(item))
|
||||
if len(picked) >= top_k:
|
||||
break
|
||||
return picked
|
||||
if doc_id not in doc_order:
|
||||
doc_order.append(doc_id)
|
||||
groups.setdefault(doc_id, []).append(item)
|
||||
try:
|
||||
score = float(item.get("score"))
|
||||
except (TypeError, ValueError):
|
||||
score = 0.0
|
||||
if score > doc_score.get(doc_id, float("-inf")):
|
||||
doc_score[doc_id] = score
|
||||
# 先聚合后排序:得分相同时按输入首现顺序(稳定)
|
||||
top_docs = sorted(doc_order, key=lambda d: doc_score.get(d, 0.0), reverse=True)[:top_k]
|
||||
for doc_id in top_docs:
|
||||
group = sorted(groups.get(doc_id, []), key=_score_of, reverse=True)
|
||||
taken_fingerprints: set[str] = set()
|
||||
for item in group:
|
||||
fingerprint = _chunk_fingerprint(item)
|
||||
if fingerprint in taken_fingerprints:
|
||||
continue # 组内近重复块(重复上传残留)不重复占名额
|
||||
taken_fingerprints.add(fingerprint)
|
||||
picked.append(dict(item))
|
||||
if len(taken_fingerprints) >= PRIVATE_QA_PER_DOC_MAX:
|
||||
break
|
||||
picked.sort(key=_score_of, reverse=True)
|
||||
return picked[:top_k]
|
||||
|
||||
|
||||
def _chunk_fingerprint(item: dict[str, Any]) -> str:
|
||||
"""组内近重复指纹:归一化(去空白)后的 snippet 前 48 字符。
|
||||
|
||||
重复上传残留的块内容逐字相同(同内容多次入库切块结果一致);
|
||||
正常相邻块(重叠 1 段)不会整段相同,不受影响。
|
||||
"""
|
||||
import re
|
||||
|
||||
snippet = str(item.get("snippet") or "")
|
||||
normalized = re.sub(r"\s+", "", snippet)
|
||||
return normalized[:48]
|
||||
|
||||
|
||||
def _score_of(item: dict[str, Any]) -> float:
|
||||
"""item 得分(无/非法 → 0.0)。"""
|
||||
try:
|
||||
return float(item.get("score"))
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
|
||||
def _build_context(picked: list[dict[str, Any]]) -> tuple[str, list[dict[str, Any]]]:
|
||||
@@ -119,7 +169,11 @@ async def ask_private(
|
||||
异常(DocgenUnavailable / Exception / LLMError)原样上抛。
|
||||
"""
|
||||
started = time.perf_counter()
|
||||
payload = await search_chunks(question, top_k=top_k, min_score=min_score, timeout=12.0)
|
||||
# 缺陷 #52c:先聚合再取块需要更宽的候选池——只取 top_k 条时重复文档
|
||||
# 的块会把池占满、聚合阶段无块可选。取 top_k×4(vstore 上限 20 clamp),
|
||||
# 最终 _pick_items 仍收敛到 top_k 条,响应量与 [资料n] 编号语义不变。
|
||||
pool_k = min(max(top_k * 4, top_k), 20)
|
||||
payload = await search_chunks(question, top_k=pool_k, min_score=min_score, timeout=12.0)
|
||||
items = [i for i in payload.get("items") or [] if isinstance(i, dict)]
|
||||
if not items:
|
||||
return {"hit": False, "mode": "empty", "answer": "", "items": []}
|
||||
|
||||
@@ -127,6 +127,96 @@ def test_bold_italic_code_runs(tmp_path):
|
||||
assert code_found
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 缺陷 #47a(2026-09-11):红头/标题样式从 render_html 产物解析(css 参数)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
HTML_OFFICIAL_STYLED = """<!DOCTYPE html><html><head><meta charset="utf-8">
|
||||
<title>红头公文</title></head><body>
|
||||
<div class="doc-cover"><div class="doc-letterhead">某某单位文件</div></div>
|
||||
<main><div class="doc-body">
|
||||
<h1 class="doc-title">关于加强安全生产的通知</h1>
|
||||
<h2>一、总体要求</h2>
|
||||
<h3>1.1 具体要求</h3>
|
||||
<p>正文内容。</p>
|
||||
</div></main>
|
||||
</body></html>"""
|
||||
|
||||
OFFICIAL_CSS = """
|
||||
.doc-letterhead { color: #c00000; font-size: 30px; font-weight: bold; }
|
||||
h1.doc-title { font-size: 22px; }
|
||||
h2 { font-size: 16px; }
|
||||
h3, h4, h5, h6 { font-size: 15px; }
|
||||
"""
|
||||
|
||||
|
||||
def test_letterhead_style_from_css_param(tmp_path):
|
||||
"""红头样式从模板 CSS 解析:#c00000 系颜色、30px→22.5pt 字号。"""
|
||||
out = docx_from_html_document(
|
||||
HTML_OFFICIAL_STYLED, title="红头公文",
|
||||
out_path=tmp_path / "a.docx", css=OFFICIAL_CSS,
|
||||
)
|
||||
doc = Document(str(out))
|
||||
head = doc.paragraphs[0]
|
||||
assert "某某单位文件" in head.text
|
||||
assert head.alignment.name == "CENTER"
|
||||
run = head.runs[0]
|
||||
assert str(_first_run_color(run)) == "C00000"
|
||||
assert run.font.size is not None and abs(run.font.size.pt - 22.5) < 0.01
|
||||
assert run.font.bold is True
|
||||
|
||||
|
||||
def test_heading_sizes_from_css_param(tmp_path):
|
||||
"""标题字号从模板 CSS 解析:h1 22px→16.5pt、h2 16px→12pt、h3 15px→11.25pt。"""
|
||||
out = docx_from_html_document(
|
||||
HTML_OFFICIAL_STYLED, title="红头公文",
|
||||
out_path=tmp_path / "b.docx", css=OFFICIAL_CSS,
|
||||
)
|
||||
doc = Document(str(out))
|
||||
sizes = {}
|
||||
for p in doc.paragraphs:
|
||||
if p.style.name.startswith("Heading") and p.runs:
|
||||
sizes[p.style.name] = p.runs[0].font.size.pt
|
||||
# DOCX 字号按半点(half-point)量化:11.25pt 落 22 半点 = 11.0pt
|
||||
assert abs(sizes.get("Heading 1") - 16.5) < 0.01
|
||||
assert abs(sizes.get("Heading 2") - 12.0) < 0.01
|
||||
assert abs(sizes.get("Heading 3") - 11.25) <= 0.26
|
||||
|
||||
|
||||
def test_no_style_falls_back_to_legacy_constants(tmp_path):
|
||||
"""无样式 HTML(不传 css)→ 回退 r3 常量(FF0000/22pt 红头,22/16/14pt 标题)。"""
|
||||
out = docx_from_html_document(HTML_SAMPLE, title="测试公文",
|
||||
out_path=tmp_path / "c.docx")
|
||||
doc = Document(str(out))
|
||||
head = doc.paragraphs[0]
|
||||
run = head.runs[0]
|
||||
assert str(_first_run_color(run)) == "FF0000"
|
||||
assert run.font.size.pt == 22.0
|
||||
sizes = {}
|
||||
for p in doc.paragraphs:
|
||||
if p.style.name.startswith("Heading") and p.runs:
|
||||
sizes[p.style.name] = p.runs[0].font.size.pt
|
||||
assert sizes.get("Heading 1") == 22.0
|
||||
assert sizes.get("Heading 2") == 16.0
|
||||
assert sizes.get("Heading 3") == 14.0
|
||||
|
||||
|
||||
def test_report_template_grey_letterhead_not_red(tmp_path):
|
||||
"""非红头模板(report:灰 #333、26px)不误渲染红头。"""
|
||||
report_css = """
|
||||
.doc-letterhead { color: #333333; font-size: 26px; font-weight: bold; }
|
||||
h1.doc-title { font-size: 22px; }
|
||||
"""
|
||||
out = docx_from_html_document(
|
||||
HTML_OFFICIAL_STYLED, title="报告",
|
||||
out_path=tmp_path / "d.docx", css=report_css,
|
||||
)
|
||||
doc = Document(str(out))
|
||||
run = doc.paragraphs[0].runs[0]
|
||||
assert str(_first_run_color(run)) == "333333"
|
||||
assert abs(run.font.size.pt - 19.5) < 0.01 # 26px * 0.75
|
||||
|
||||
|
||||
# r3-47-01 复现样例:真实 render_html 产物形态——
|
||||
# <main><div class="doc-body">…</div></main> 包裹层 + 嵌套 section
|
||||
HTML_WRAPPED = """<!DOCTYPE html><html><head><meta charset="utf-8">
|
||||
|
||||
@@ -47,6 +47,11 @@ CASES = [
|
||||
("帮我写个会议通知模板", False, True, "docgen"),
|
||||
("给这个通知加个结尾", False, True, "edit"),
|
||||
("增加一段关于风险的描述", False, True, "edit"),
|
||||
# r4 #52(2026-09-11)新增:总结/概括/归纳 三动词 + 动词自指保护
|
||||
("帮我总结论文", True, False, "docgen"),
|
||||
("总结一下植物光合作用的过程", False, False, "qa"),
|
||||
("概括一下这个通知", True, True, "docgen"),
|
||||
("归纳这篇文章", True, False, "docgen"),
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -84,6 +84,52 @@ def test_normalize_source_only_doi_derives_url():
|
||||
assert src["metadata"] == {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# §8 title 回填(缺陷 #27,2026-09-11)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_normalize_source_title_fill_from_source_entity():
|
||||
"""title 为空 -> 回填 source(断言主体实体名),正文片段绝不参与。"""
|
||||
src = normalize_source({
|
||||
"source": "OsSPL14",
|
||||
"title": "",
|
||||
"evidence": "## 正文片段开头(不是标题)",
|
||||
"paper_id": "P1",
|
||||
})
|
||||
assert src["title"] == "OsSPL14"
|
||||
# title 即 source 回填:两者相等,source_entity 不落 metadata(不重复)
|
||||
assert src["metadata"].get("source_entity") is None
|
||||
assert src["paper_id"] == "P1"
|
||||
|
||||
|
||||
def test_normalize_source_title_fill_from_metadata():
|
||||
"""title/paper_title 均空 -> metadata 标题类字段回填。"""
|
||||
src = normalize_source({
|
||||
"source": "entity",
|
||||
"metadata": {"paper_title": "水稻抗病基因研究进展", "source_entity": "entity"},
|
||||
})
|
||||
assert src["title"] == "水稻抗病基因研究进展"
|
||||
|
||||
|
||||
def test_normalize_source_title_not_overridden():
|
||||
"""title 非空 -> 不回填(既有语义不变)。"""
|
||||
src = normalize_source({
|
||||
"title": "Real Title",
|
||||
"source": "Entity",
|
||||
"paper_title": "Other",
|
||||
"metadata": {"paper_title": "Meta"},
|
||||
})
|
||||
assert src["title"] == "Real Title"
|
||||
assert src["metadata"]["source_entity"] == "Entity"
|
||||
|
||||
|
||||
def test_normalize_source_title_all_empty_stays_none():
|
||||
"""全空 -> 保持 null(前端有 fallback)。"""
|
||||
src = normalize_source({"content": "snippet only", "paper_id": "P1"})
|
||||
assert src["title"] is None
|
||||
assert src["snippet"] == "snippet only"
|
||||
|
||||
|
||||
def test_normalize_source_alias_chains():
|
||||
src = normalize_source({
|
||||
"content": "snippet text",
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
"""test_privateqa_pick.py —— _pick_items 先按 doc_id 聚合再取块(缺陷 #52c,验收 §6 第 3 条)。
|
||||
|
||||
运行:cd sdk && /home/xzq/5090/teamWorkspace/zh/plantai-rag-new-venv/bin/python -m pytest app/services/test_privateqa_pick.py -v
|
||||
(venv = plantai-rag-new-venv;sdk 的 app/ 布局与 pyproject testpaths 一致)
|
||||
|
||||
构造合成数据(不真跑 FAISS):
|
||||
- 4 个 doc_id 各若干「作者页块」(高分、分接近)+ 1 条低分「正文块」。
|
||||
- 旧实现会把高分的作者页块占满 top-8、正文块进不了上下文;
|
||||
新实现按 doc 级 max score 取 top 文档 → 各 doc 只占 ≤3 块,
|
||||
正文块(本组内最高低分块)能进入 picked。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from app.services.privateqa import (
|
||||
PRIVATE_QA_PER_DOC_MAX,
|
||||
_build_context,
|
||||
_pick_items,
|
||||
)
|
||||
|
||||
|
||||
def make_items(doc_specs):
|
||||
"""doc_specs: [(doc_id, [(score, snippet), ...]), ...] → item 列表(输入序打散,
|
||||
模拟 vstore 全局 score 序)。"""
|
||||
items = []
|
||||
for doc_id, chunks in doc_specs:
|
||||
for score, snippet in chunks:
|
||||
items.append({"doc_id": doc_id, "score": score, "snippet": snippet, "doc_name": f"资料{doc_id}"})
|
||||
return sorted(items, key=lambda it: -it["score"])
|
||||
|
||||
|
||||
def test_dup_docs_no_longer_monopolize_topk():
|
||||
"""#52c 主场景:4 个 doc_id 各 200 条近同分作者页块(同 doc 内内容相同,
|
||||
模拟重复上传残留——同一页面被反复切块/复传)+ 正文块 →
|
||||
选出的 items 覆盖多个 doc 且含正文块。"""
|
||||
specs = []
|
||||
for i in range(4):
|
||||
author = [(0.94 - i * 0.001, f"作者页块 {i}") for _ in range(200)]
|
||||
body = [(0.30, f"正文块 {i} 方法与结论")]
|
||||
specs.append((f"doc-{i}", author + body))
|
||||
items = make_items(specs)
|
||||
picked = _pick_items(items, top_k=8)
|
||||
|
||||
doc_ids = [it["doc_id"] for it in picked]
|
||||
assert len(set(doc_ids)) >= 2, f"重复文档仍占满名额:{doc_ids}"
|
||||
for doc_id in set(doc_ids):
|
||||
assert doc_ids.count(doc_id) <= PRIVATE_QA_PER_DOC_MAX
|
||||
snippets = [it["snippet"] for it in picked]
|
||||
assert any("正文块" in s for s in snippets), "正文块未进入上下文"
|
||||
# 总分 ≤ top_k;块按分数降序
|
||||
assert len(picked) <= 8
|
||||
scores = [it["score"] for it in picked]
|
||||
assert scores == sorted(scores, reverse=True)
|
||||
|
||||
|
||||
def test_normal_case_semantics_preserved():
|
||||
"""回归:正常场景 top_k/每 doc 上限语义不变。"""
|
||||
items = [
|
||||
{"doc_id": "A", "score": 0.95, "snippet": "a1"},
|
||||
{"doc_id": "A", "score": 0.90, "snippet": "a2"},
|
||||
{"doc_id": "B", "score": 0.85, "snippet": "b1"},
|
||||
{"doc_id": "C", "score": 0.80, "snippet": "c1"},
|
||||
{"doc_id": "D", "score": 0.75, "snippet": "d1"},
|
||||
]
|
||||
picked = _pick_items(items, top_k=5)
|
||||
assert len(picked) == 5
|
||||
for doc_id in {it["doc_id"] for it in picked}:
|
||||
assert sum(1 for it in picked if it["doc_id"] == doc_id) <= PRIVATE_QA_PER_DOC_MAX
|
||||
# 每 doc 内按分数降序取 ≤3 块
|
||||
a = [it for it in picked if it["doc_id"] == "A"]
|
||||
assert [it["snippet"] for it in a] == ["a1", "a2"]
|
||||
|
||||
|
||||
def test_single_doc_caps_at_per_doc_max():
|
||||
"""回归:单一 doc 大量块时仍 ≤ per-doc 上限。"""
|
||||
items = [{"doc_id": "X", "score": 0.9 - i * 0.001, "snippet": f"x{i}"} for i in range(20)]
|
||||
picked = _pick_items(items, top_k=8)
|
||||
assert 1 <= len(picked) <= PRIVATE_QA_PER_DOC_MAX
|
||||
|
||||
|
||||
def test_ties_are_stable():
|
||||
"""回归:同分并列时文档选择稳定(按输入首现顺序),结果确定。"""
|
||||
items = make_items(
|
||||
[
|
||||
("A", [(0.9, "a1"), (0.9, "a2")]),
|
||||
("B", [(0.9, "b1")]),
|
||||
("C", [(0.9, "c1")]),
|
||||
]
|
||||
)
|
||||
p1 = _pick_items(items, top_k=4)
|
||||
p2 = _pick_items(items, top_k=4)
|
||||
assert [it["doc_id"] for it in p1] == [it["doc_id"] for it in p2]
|
||||
assert len(p1) == 4
|
||||
|
||||
|
||||
def test_malformed_items_ignored():
|
||||
"""回归:非 dict / 缺 doc_id 条目不炸。"""
|
||||
picked = _pick_items(
|
||||
[None, "bad", {"score": 0.9, "snippet": "s"}, {"doc_id": "Z", "score": 0.5, "snippet": "z"}],
|
||||
top_k=4,
|
||||
)
|
||||
assert picked
|
||||
|
||||
|
||||
def test_build_context_integration():
|
||||
"""回归:_build_context 12k 预算截断 + [资料n] 引用编号与 items 顺序一致。"""
|
||||
picked = _pick_items(
|
||||
[
|
||||
{"doc_id": "A", "score": 0.95, "snippet": "正文一"},
|
||||
{"doc_id": "A", "score": 0.90, "snippet": "正文二"},
|
||||
{"doc_id": "B", "score": 0.80, "snippet": "正文三"},
|
||||
],
|
||||
top_k=3,
|
||||
)
|
||||
context, out_items = _build_context(picked)
|
||||
assert "[资料1]" in context
|
||||
assert out_items[0]["ref"] == 1
|
||||
assert [it["snippet"] for it in out_items] == [it["snippet"] for it in picked]
|
||||
|
||||
|
||||
async def test_ask_private_fetches_wider_pool(monkeypatch):
|
||||
"""#52c:ask_private 以 top_k×4 向 vstore 取候选池,再收敛到 top_k。"""
|
||||
import app.services.privateqa as pq
|
||||
|
||||
calls = {}
|
||||
|
||||
async def fake_search(query, *, top_k, min_score, timeout):
|
||||
calls["requested_top_k"] = top_k
|
||||
items = []
|
||||
for doc_i in range(4):
|
||||
for j in range(200):
|
||||
items.append(
|
||||
{
|
||||
"doc_id": f"doc-{doc_i}",
|
||||
"score": 0.94 - doc_i * 0.001,
|
||||
# 同 doc 内作者页块内容相同(重复上传残留的真实形态)
|
||||
"snippet": f"作者页块 {doc_i}",
|
||||
"doc_name": f"资料{doc_i}",
|
||||
}
|
||||
)
|
||||
items.append(
|
||||
{
|
||||
"doc_id": f"doc-{doc_i}",
|
||||
"score": 0.30,
|
||||
"snippet": f"正文块 {doc_i} 方法与结论",
|
||||
"doc_name": f"资料{doc_i}",
|
||||
}
|
||||
)
|
||||
return {"ok": True, "mode": "vector", "items": items}
|
||||
|
||||
async def fake_llm(system, user, **kwargs):
|
||||
return "方法与结论见[资料1]。"
|
||||
|
||||
monkeypatch.setattr(pq, "search_chunks", fake_search)
|
||||
monkeypatch.setattr(pq, "complete_text", fake_llm)
|
||||
|
||||
result = await pq.ask_private("总结这篇论文的主要方法和结论", top_k=8)
|
||||
assert calls["requested_top_k"] == 20, "候选池应取 min(top_k×4, vstore 上限 20)"
|
||||
assert result["hit"] is True
|
||||
assert result["mode"] == "vector"
|
||||
doc_ids = [it["doc_id"] for it in result["items"]]
|
||||
assert len(set(doc_ids)) >= 2
|
||||
assert any("正文块" in (it["snippet"] or "") for it in result["items"])
|
||||
assert len(result["items"]) <= 8
|
||||
assert all(it["ref"] >= 1 for it in result["items"])
|
||||
|
||||
|
||||
async def test_ask_private_pool_capped_at_20(monkeypatch):
|
||||
"""#52c:候选池 = min(top_k×4, 20)(vstore CHUNK_TOP_K_MAX clamp)。"""
|
||||
import app.services.privateqa as pq
|
||||
|
||||
calls = {}
|
||||
|
||||
async def fake_search(query, *, top_k, min_score, timeout):
|
||||
calls["requested_top_k"] = top_k
|
||||
return {"ok": True, "mode": "vector", "items": []}
|
||||
|
||||
monkeypatch.setattr(pq, "search_chunks", fake_search)
|
||||
await pq.ask_private("问题", top_k=8)
|
||||
assert calls["requested_top_k"] == 20
|
||||
await pq.ask_private("问题", top_k=5)
|
||||
assert calls["requested_top_k"] == 20 # 5×4=20
|
||||
await pq.ask_private("问题", top_k=3)
|
||||
assert calls["requested_top_k"] == 12 # 3×4=12(低于 20)
|
||||
@@ -16,6 +16,10 @@ const query = ref('');
|
||||
const results = ref([]);
|
||||
const searching = ref(false);
|
||||
const searchError = ref(false);
|
||||
// 缺陷 #32:搜索成功但空结果(200 + results:[],非超时)的明确提示——
|
||||
// 与 kgDown(服务不可用/超时)互斥:请求成功且空 results 置 true,
|
||||
// 下一次搜索发起时复位。
|
||||
const noResult = ref(false);
|
||||
const detailNode = ref(null);
|
||||
const graphData = ref(null); // 当前渲染的 {nodes, edges}(答案图谱或邻域结果)
|
||||
const graphSource = ref(null); // 'answer' | 'neighborhood'
|
||||
@@ -80,9 +84,11 @@ async function onSearch() {
|
||||
}
|
||||
searching.value = true;
|
||||
searchError.value = false;
|
||||
noResult.value = false; // 新一次搜索复位空结果提示
|
||||
try {
|
||||
const data = await searchEntities(q, 20);
|
||||
results.value = data?.results || [];
|
||||
noResult.value = !results.value.length; // 成功但空:明确提示(#32,非超时)
|
||||
} catch {
|
||||
searchError.value = true;
|
||||
} finally {
|
||||
@@ -197,6 +203,8 @@ const kgDown = computed(() => !bench.kgReady && bench.kgStatus != null);
|
||||
</button>
|
||||
</div>
|
||||
<div v-if="searchError" class="offline-text">{{ T.kgDown }}</div>
|
||||
<!-- 缺陷 #32:成功但空结果提示(kgNoResult,与 kgDown/kgEmpty 三态区分互斥) -->
|
||||
<div v-else-if="noResult && !searching" class="no-result-note">{{ T.kgNoResult }}</div>
|
||||
<div v-if="results.length" class="search-results">
|
||||
<div v-for="r in results" :key="r.id" class="result-item" @click="onSelectResult(r)">
|
||||
<span class="rs-name">{{ r.name }}</span>
|
||||
@@ -234,6 +242,8 @@ const kgDown = computed(() => !bench.kgReady && bench.kgStatus != null);
|
||||
}
|
||||
.offline-dot { width: 7px; height: 7px; border-radius: 50%; background: var(--pa-error); flex: none; }
|
||||
.offline-text { font-size: 12px; color: var(--pa-error); }
|
||||
/* 缺陷 #32:空结果提示——非错误态(200 空 results 属正常),用中性副文案色 */
|
||||
.no-result-note { font-size: 12px; color: var(--pa-ink-subtle); }
|
||||
.degrade-note { font-size: 11.5px; color: var(--pa-ink-subtle); }
|
||||
.node-list {
|
||||
display: flex;
|
||||
|
||||
@@ -47,6 +47,18 @@ const badgeTitle = computed(() => {
|
||||
return raw;
|
||||
});
|
||||
|
||||
// 缺陷 #27:title 渲染空值 fallback——后端未回填 title 时绝不渲染空文本
|
||||
// (空文本会让条目不可读、视觉上「串行」),也绝不让正文片段被当成标题。
|
||||
// 显示优先级:非空 title > paper_id > id > i18n 占位「(无标题)」。
|
||||
const displayTitle = computed(() => {
|
||||
const s = props.source || {};
|
||||
const t = s.title;
|
||||
if (t != null && String(t).trim() !== '') return String(t);
|
||||
const raw = badgeTitle.value;
|
||||
if (raw) return raw;
|
||||
return T.sourceNoTitle;
|
||||
});
|
||||
|
||||
const metaParts = computed(() => {
|
||||
const s = props.source || {};
|
||||
const parts = [];
|
||||
@@ -63,7 +75,7 @@ const metaParts = computed(() => {
|
||||
<span class="ft-ico idx" :title="badgeTitle">{{ badgeId }}</span>
|
||||
<div class="src-main">
|
||||
<div class="src-line1">
|
||||
<span class="ft-name" :title="source.title || ''">{{ source.title }}</span>
|
||||
<span class="ft-name" :title="displayTitle">{{ displayTitle }}</span>
|
||||
</div>
|
||||
<div class="ft-size src-meta">{{ metaParts.join(' · ') }}</div>
|
||||
<div v-if="source.snippet" class="src-snippet" :class="{ clamp: !expanded }" @click="expanded = !expanded">
|
||||
|
||||
@@ -11,11 +11,14 @@ import ChainGroup from './ChainGroup.vue';
|
||||
const bench = useBenchStore();
|
||||
const chains = ref({}); // chain_id → open
|
||||
|
||||
// 会话级聚合按需拉取(仅当用户切到「本会话全部」且尚无缓存时)
|
||||
// 会话级聚合按需拉取(仅当用户切到「本会话全部」且尚无缓存时);
|
||||
// 缺陷 #27:新回答完成后 chat.js 终态置 sourcesStale → 自动 force 重拉
|
||||
//(无需手动切 TAB/刷新页面),拉取成功后标记由 store 内部清除。
|
||||
watch(
|
||||
() => [bench.sourceScope, bench.currentAssistantMessage],
|
||||
() => [bench.sourceScope, bench.currentAssistantMessage, bench.sourcesStale],
|
||||
([scope]) => {
|
||||
if (scope === 'session' && !bench.sessionSources && bench._activeSessionId()) {
|
||||
if (scope === 'session' && bench._activeSessionId()
|
||||
&& (!bench.sessionSources || bench.sourcesStale)) {
|
||||
bench.loadSessionSources().catch(() => {});
|
||||
}
|
||||
},
|
||||
|
||||
@@ -62,21 +62,26 @@ function backToSelect() {
|
||||
async function confirmCreate() {
|
||||
const cand = selectedCandidate.value;
|
||||
if (!cand || confirming.value) return;
|
||||
if (props.block.jobId) return; // 已创建过(防重复创建;FR-BIO 任务幂等语义)
|
||||
confirming.value = true;
|
||||
error.value = '';
|
||||
try {
|
||||
const sessionId = chat.sessionId || '';
|
||||
const detail = await bioflow.createJobFromPlan({
|
||||
sessionId,
|
||||
// r4 修复:不再 emit('updated')(该事件全链无人监听,任务创建后 UI 静默)——
|
||||
// 改走 chat store 一站式 confirmBioflowPath:创建 → 挂进度块 → 订阅任务事件
|
||||
// 桥接推进 → 落库。aMsg 从 chat 消息流定位(当前消息 = 含本 path 块的 assistant 消息)。
|
||||
const aMsg = (chat.messages || []).find(
|
||||
(m) => m && m.role === 'assistant' && Array.isArray(m.blocks) && m.blocks.includes(props.block),
|
||||
);
|
||||
const detail = await chat.confirmBioflowPath({
|
||||
title: (chat.messages || []).find((m) => m.role === 'user')?.text?.slice(0, 50) || '生信任务',
|
||||
stages: cand.stages,
|
||||
params: {},
|
||||
autoContinue: false,
|
||||
archiveId: reuseArchive.value || null,
|
||||
aMsg: aMsg || null,
|
||||
});
|
||||
// 进度块由任务事件流/详情回填;此处标记 path 块已确认
|
||||
props.block.jobId = detail.id || null;
|
||||
emit('updated');
|
||||
// 进度块由 confirmBioflowPath 挂载;此处标记 path 块已确认
|
||||
props.block.jobId = (detail && detail.id) || null;
|
||||
} catch (err) {
|
||||
error.value = err?.message || BF.pathStageCreating;
|
||||
emit('flow-error', err);
|
||||
|
||||
@@ -134,6 +134,11 @@ export const T = {
|
||||
extractUploadFail: '上传失败:',
|
||||
kgEmpty: '本次检索未产生图谱关系,可使用下方搜索浏览知识库',
|
||||
kgSearchPlaceholder: '搜索实体名,如 OsSPL14',
|
||||
// 缺陷 #32:搜索成功但无结果(200 + results:[])的明确提示——与 kgEmpty(初始空态)、
|
||||
// kgDown(服务不可用/超时)三态区分、触发条件互斥
|
||||
kgNoResult: '未找到相关实体',
|
||||
// 缺陷 #27:文献来源条目 title 为空时的占位(绝不渲染空文本或正文片段)
|
||||
sourceNoTitle: '(无标题)',
|
||||
chainBadge: '跳',
|
||||
sessionEmptyTitle: '海古纪 PlantAI 植物科研大模型',
|
||||
sessionEmptySub: '基于文献知识库的基因家族问答 —— 每一次提问都有证据锚定',
|
||||
|
||||
+39
-3
@@ -3,7 +3,9 @@
|
||||
// 判定顺序(优先级从高到低,命中即返回;漏判安全:不命中一律 qa):
|
||||
// 1. export:hasActiveArtifact 且含 EXPORT_WORD 且含 FORMAT_WORD
|
||||
// 2. edit: hasActiveArtifact 且含 EDIT_WORD
|
||||
// 3. docgen:含 WRITE_VERB 且含 DOC_TYPE(排除词优先,防误判)
|
||||
// 3. docgen:含 WRITE_VERB 且含 DOC_TYPE(排除词优先,防误判;
|
||||
// 缺陷 #52a 细化:纯总结类动词句的「总结」自身不作 DOC_TYPE 证据,
|
||||
// 须命中其他文档类型词如「论文」才进 docgen)
|
||||
// 4. docgen:hasAttachments 且含 WRITE_VERB(附件兜底信号)
|
||||
// 5. qa
|
||||
|
||||
@@ -16,7 +18,12 @@ const FORMAT_WORD = ['pdf', 'docx', 'doc', 'word'];
|
||||
const EDIT_WORD = ['改', '修改', '调整', '删', '删除', '加一段', '补充', '润色', '缩写', '扩写', '重写', '改写', '优化', '加个', '增加一段'];
|
||||
|
||||
// —— 写文档意图词典 ——
|
||||
const WRITE_VERB = ['写', '撰写', '拟', '起草', '编制', '生成', '编写', '拟定', '草拟', '制定'];
|
||||
// 缺陷 #52a:补「总结/概括/归纳」——「帮我总结论文」+附件走 docgen(规则 4 附件兜底)。
|
||||
// 纯总结类动词句(无其他写动词、无附件)须命中「论文」等其他文档类型词才进
|
||||
// docgen(规则 3 细化,详见 WRITE_VERB_SUMMARIZE_EXCLUDE 注释)——
|
||||
// 无附件「总结一下植物光合作用的过程」仍走 QA(漏判安全)。
|
||||
// 后端同词典:sdk/app/services/docgen/intent.py 的 WRITE_VERB(由 Dev-B1 同步)。
|
||||
const WRITE_VERB = ['写', '撰写', '拟', '起草', '编制', '生成', '编写', '拟定', '草拟', '制定', '总结', '概括', '归纳'];
|
||||
const DOC_TYPE = [
|
||||
'论文', '报告', '通知', '通告', '公告', '请示', '批复', '函', '申请书', '申请',
|
||||
'申报书', '标书', '合同', '协议', '方案', '计划', '总结', '汇报', '讲话稿', '致辞',
|
||||
@@ -28,6 +35,15 @@ const DOC_TYPE = [
|
||||
];
|
||||
// 明确排除(不属于 DOC_TYPE,防误判):介绍、内容、代码、诗、作业、问题、建议、想法
|
||||
const DOC_TYPE_EXCLUDE = ['介绍', '内容', '代码', '诗', '作业', '问题', '建议', '想法'];
|
||||
// 缺陷 #52a:「总结/概括/归纳」入 WRITE_VERB 后,「总结」本身会命中 DOC_TYPE
|
||||
// (如「总结一下植物光合作用的过程」无附件也被规则 3 拉进 docgen)。
|
||||
// 防护语义(TL 裁决):句子的写动词仅由总结类新动词构成时,「总结」自身
|
||||
// 不作 DOC_TYPE 证据——必须有其他文档类型词(如「总结这篇论文」的「论文」)
|
||||
// 才进 docgen;带附件场景由规则 4 兜底,不受影响。
|
||||
// 「写一篇总结」「写一份工作总结」等既有写类动词 + 总结类文档词路径不受影响。
|
||||
const WRITE_VERB_SUMMARIZE_EXCLUDE = ['总结', '概括', '归纳'];
|
||||
const WRITE_VERB_NON_SUMMARIZE = WRITE_VERB.filter((w) => !WRITE_VERB_SUMMARIZE_EXCLUDE.includes(w));
|
||||
const DOC_TYPE_NON_SUMMARIZE = DOC_TYPE.filter((w) => !WRITE_VERB_SUMMARIZE_EXCLUDE.includes(w));
|
||||
|
||||
/** 文本是否含词典中任一词(子串判定;英文不区分大小写——中文不受影响,与后端一致) */
|
||||
export function hasAnyWord(text, words) {
|
||||
@@ -43,6 +59,21 @@ export function hasAnyExcludeWord(text) {
|
||||
return hasAnyWord(text, DOC_TYPE_EXCLUDE);
|
||||
}
|
||||
|
||||
/** 缺陷 #52a:文本含总结类新动词 → 规则 3 需额外校验 */
|
||||
export function hasAnySummarizeVerb(text) {
|
||||
return hasAnyWord(text, WRITE_VERB_SUMMARIZE_EXCLUDE);
|
||||
}
|
||||
|
||||
/** 文本含除总结类新动词外的其他写类动词(写/撰写/生成/制定…) */
|
||||
export function hasNonSummarizeWriteVerb(text) {
|
||||
return hasAnyWord(text, WRITE_VERB_NON_SUMMARIZE);
|
||||
}
|
||||
|
||||
/** 文本命中非总结类的文档类型词(「总结」自身不作证据) */
|
||||
export function hasNonSummarizeDocType(text) {
|
||||
return hasAnyWord(text, DOC_TYPE_NON_SUMMARIZE);
|
||||
}
|
||||
|
||||
/** 导出文档类型词词典(任务书约定的导出类型词;判定走 EXPORT_WORD+FORMAT_WORD,此处导出供组件复用) */
|
||||
export const EXPORT_DOC_TYPE_WORDS = [
|
||||
'论文', '报告', '通知', '通告', '公告', '请示', '批复', '函', '申请书', '申报书',
|
||||
@@ -64,7 +95,12 @@ export function route(text, hasAttachments, hasActiveArtifact) {
|
||||
// 2. 编辑意图(需活动文档)
|
||||
if (hasActiveArtifact && hasAnyWord(s, EDIT_WORD)) return 'edit';
|
||||
// 3. 写文档意图:写类动词 + 文档类型词(排除词防误判优先)
|
||||
if (hasAnyWord(s, WRITE_VERB) && !hasAnyExcludeWord(s) && hasAnyWord(s, DOC_TYPE)) return 'docgen';
|
||||
// 缺陷 #52a:纯总结类动词句(无其他写动词)须命中非总结类文档类型词
|
||||
// (如「论文」)才进 docgen——「总结一下植物光合作用的过程」仍走 QA
|
||||
if (hasAnyWord(s, WRITE_VERB) && !hasAnyExcludeWord(s) && hasAnyWord(s, DOC_TYPE)) {
|
||||
const summarizeOnly = hasAnySummarizeVerb(s) && !hasNonSummarizeWriteVerb(s);
|
||||
if (!summarizeOnly || hasNonSummarizeDocType(s)) return 'docgen';
|
||||
}
|
||||
// 4. 附件兜底:拖入文件 + 写类动词,即使无文档类型词也视为生成意图
|
||||
if (hasAttachments && hasAnyWord(s, WRITE_VERB)) return 'docgen';
|
||||
// 5. 漏判安全:一律 qa
|
||||
|
||||
+19
-2
@@ -78,6 +78,10 @@ export const useBenchStore = defineStore('bench', {
|
||||
kgReady: false, // Neo4j 是否可用(false → 离线降级)
|
||||
kgError: false,
|
||||
kgNodeLimit: 50, // 移动端由 KgPane 下调为 30
|
||||
// ---- 会话来源缓存失效标记(缺陷 #27:新回答完成后强制重拉)----
|
||||
// 由 chat.js 终态(onComplete/_runSearch 完成)置 true;loadSessionSources
|
||||
// 拉取成功后置 false。SourcesPane 监听此标记自动 force 重拉。
|
||||
sourcesStale: false,
|
||||
// ---- 统一状态机 ----
|
||||
refreshSeq: 0, // 每轮「置空 → 加载」自增,用于丢弃过期响应
|
||||
needRefresh: {}, // pane → true:下次激活时重新装配(离线重试等场景)
|
||||
@@ -174,6 +178,11 @@ export const useBenchStore = defineStore('bench', {
|
||||
setSourceScope(scope) {
|
||||
this.sourceScope = scope;
|
||||
},
|
||||
/** 缺陷 #27:一轮问答终态时由 chat.js 调用——标记会话来源缓存失效,
|
||||
* 促使 SourcesPane 自动 force 重拉(不在流式过程中逐帧触发)。 */
|
||||
markSourcesStale() {
|
||||
this.sourcesStale = true;
|
||||
},
|
||||
// FR-CHAT-015:引用 chip 点击 → 切到来源 tab 并高亮对应条目 2s(PM 集成补丁,B2 依赖)
|
||||
// 缺陷 #27 修复:参数由 index 改为 key(paper_id/id)——「本次回答」域 items 与
|
||||
// chip 的来源块同序,key 匹配即可精确定位,不再依赖行序号(会话聚合域 id 错位
|
||||
@@ -211,7 +220,10 @@ export const useBenchStore = defineStore('bench', {
|
||||
* degraded/离线时保留已有数据并标记错误(失败必须可见,见 PRD §2.2)。
|
||||
*/
|
||||
async loadSessionSources({ force = false } = {}) {
|
||||
if (!force && this.sessionSources) return this.sessionSources;
|
||||
// 缺陷 #27:终态失效标记(chat.js 完成一轮后置 sourcesStale=true)——
|
||||
// 与 force 一样需要绕过缓存短路;成功落库后清除失效标记
|
||||
const mustReload = force || this.sourcesStale;
|
||||
if (!mustReload && this.sessionSources) return this.sessionSources;
|
||||
const sid = this._activeSessionId();
|
||||
if (!sid) return null;
|
||||
const seq = ++this.refreshSeq;
|
||||
@@ -219,9 +231,13 @@ export const useBenchStore = defineStore('bench', {
|
||||
this.sourcesError = false;
|
||||
try {
|
||||
const data = await httpGet(`/api/sessions/${sid}/sources`);
|
||||
if (seq === this.refreshSeq) this.sessionSources = data;
|
||||
if (seq === this.refreshSeq) {
|
||||
this.sessionSources = data;
|
||||
this.sourcesStale = false;
|
||||
}
|
||||
return data;
|
||||
} catch (err) {
|
||||
// 失败保留失效标记:下次 scope/pane 激活仍会重试
|
||||
if (seq === this.refreshSeq) this.sourcesError = true;
|
||||
throw err;
|
||||
} finally {
|
||||
@@ -351,6 +367,7 @@ export const useBenchStore = defineStore('bench', {
|
||||
resetSessionData() {
|
||||
this.localStageEvents = [];
|
||||
this.sessionSources = null;
|
||||
this.sourcesStale = false;
|
||||
this.sourcesLoading = false;
|
||||
this.sourcesError = false;
|
||||
this.jobDetail = {};
|
||||
|
||||
@@ -273,7 +273,7 @@ export const useBioflowStore = defineStore('bioflow', {
|
||||
// 事件订阅(SSE 断线重连回放;EventSource 契约允许测试注入 mock)
|
||||
// ---------------------------------------------------------------
|
||||
/** 打开 /jobs/{id}/events 订阅(首帧快照整体刷新 + 回放 + 实时;终态服务端关闭) */
|
||||
connectJobEvents(jobId, sessionId, { sourceFactory, onOpen } = {}) {
|
||||
connectJobEvents(jobId, sessionId, { sourceFactory, onOpen, onEvent } = {}) {
|
||||
const key = sessionId || 'default';
|
||||
this.disconnectJobEvents(key);
|
||||
const lastSeq = this.currentJob.lastSeq || 0;
|
||||
@@ -295,6 +295,10 @@ export const useBioflowStore = defineStore('bioflow', {
|
||||
this.applyJobEvent({ type, data, seq });
|
||||
if (typeof seq === 'number') rec.lastSeq = Math.max(rec.lastSeq, seq);
|
||||
}
|
||||
// 事件桥接(r4 修复:路径卡确认创建后消息进度块与 store 同源推进)
|
||||
if (typeof onEvent === 'function') {
|
||||
try { onEvent(type, data, seq); } catch { /* 桥接异常不阻断事件应用 */ }
|
||||
}
|
||||
};
|
||||
es.addEventListener('snapshot', (e) => {
|
||||
try { apply('snapshot', JSON.parse(e.data || '{}'), 0); } catch { /* 坏帧忽略 */ }
|
||||
|
||||
+102
-1
@@ -12,7 +12,7 @@
|
||||
// (streamCtx),切走后流在后台继续(消息数组 sessionMsgs 持续更新),
|
||||
// 切回时直接拿实时数组;stop/重试只作用于当前会话的流。
|
||||
// 原 FR-SESSION-004 的「切会话即中断」行为废止,见 CHANGELOG。
|
||||
import { defineStore } from 'pinia';
|
||||
import { defineStore, getActivePinia } from 'pinia';
|
||||
import { createMessage, updateMessage } from '../api/sessions.js';
|
||||
import { chatStream, search } from '../api/qa.js';
|
||||
import { useSessionsStore } from './sessions.js';
|
||||
@@ -572,6 +572,32 @@ export const useChatStore = defineStore('chat', {
|
||||
this.lastResult = null;
|
||||
this.scrollSeq++;
|
||||
hydrateBench();
|
||||
// r4 修复:恢复生信任务事件订阅(路径卡确认创建的任务;刷新后进度卡继续流转。
|
||||
// 触发条件:assistant 消息含 bioflow_progress 块且 jobId 非空、任务未终态。
|
||||
// 订阅的 onEvent 桥接把每帧事件 patch 进该消息的进度块(与 confirmBioflowPath 同款)。
|
||||
try {
|
||||
const bioflowStore = useBioflowStore();
|
||||
const BIOFLOW_TERMINAL = new Set(['done', 'paused', 'archived', 'failed', 'error']);
|
||||
for (const m of this.messages) {
|
||||
if (!m || m.role !== 'assistant' || !Array.isArray(m.blocks)) continue;
|
||||
const blk = m.blocks.find((b) => b && b.type === 'bioflow_progress');
|
||||
if (!blk || !blk.jobId || BIOFLOW_TERMINAL.has(blk.jobStatus)) continue;
|
||||
const jobId = blk.jobId;
|
||||
bioflowStore.connectJobEvents(jobId, this.sessionId, {
|
||||
onEvent: (type, data) => {
|
||||
const frame = patchBioflowEvent(m, { type, data });
|
||||
if (frame) this.scrollSeq++;
|
||||
if (type === 'stage_await_confirm' && data) { patchBioflowForm(m, data); this.scrollSeq++; }
|
||||
else if (type === 'stage_await_files' && data) { patchBioflowFile(m, data); this.scrollSeq++; }
|
||||
else if (type === 'job_done' && data) {
|
||||
const ab = patchBioflowArchiveSuggest(m, data.job_id || jobId);
|
||||
if (ab) ab.jobId = data.job_id || jobId;
|
||||
this.scrollSeq++;
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
} catch { /* 生信未就绪容错 */ }
|
||||
// 缺陷 #52:恢复 docgen 任务轮询(hydrate 后 aMsg 已是响应式代理)
|
||||
if (this._docgenResumeQueue && this._docgenResumeQueue.length) {
|
||||
const queue = this._docgenResumeQueue;
|
||||
@@ -1360,6 +1386,64 @@ export const useChatStore = defineStore('chat', {
|
||||
return null;
|
||||
},
|
||||
|
||||
/** 路径卡「确认创建任务」(r4 修复:修复前 FlowPathCard.confirmCreate 自调
|
||||
* createJobFromPlan 后 emit('updated') 无人监听——任务 201 创建成功但 UI 静默:
|
||||
* 无进度卡、无任务事件订阅、不落库(刷新即丢)。此处把
|
||||
* 「创建 → 挂进度块 → 订阅任务事件(桥接推进消息块)→ 落库」收拢成一站式。
|
||||
* 返回 bioflow store 的 job detail(与 createJobFromPlan 同形,卡片可继续用)。 */
|
||||
async confirmBioflowPath({ title, stages, params, autoContinue, archiveId, aMsg, userMessageId }) {
|
||||
const bioflow = useBioflowStore();
|
||||
const detail = await bioflow.createJobFromPlan({
|
||||
sessionId: this.sessionId || '',
|
||||
title: title || '生信任务',
|
||||
stages,
|
||||
params: params || {},
|
||||
autoContinue: !!autoContinue,
|
||||
archiveId: archiveId || null,
|
||||
});
|
||||
const jobId = (detail && detail.id) || null;
|
||||
if (!jobId || !aMsg) return detail;
|
||||
// 1. 路径卡落定:path 块打 jobId + selected 固化(复用刷新后仍可确认态)
|
||||
const pathBlk = (aMsg.blocks || []).find((b) => b && b.type === 'bioflow_path');
|
||||
if (pathBlk) {
|
||||
pathBlk.jobId = jobId;
|
||||
if (typeof pathBlk.selected !== 'number') pathBlk.selected = 0;
|
||||
}
|
||||
// 2. 进度块:以 job_created 事件形态挂(stage 状态 waiting,jobStatus created)
|
||||
patchBioflowEvent(aMsg, { type: 'job_created', data: { job_id: jobId, auto_continue: !!autoContinue, stages: detail.stages || [] } });
|
||||
// 3. 订阅任务事件并把每帧桥接进消息进度块(与 sendBioflow 流内 openJobEvents 同源)
|
||||
try {
|
||||
bioflow.connectJobEvents(jobId, this.sessionId || '', {
|
||||
onEvent: (type, data, seq) => {
|
||||
const frame = patchBioflowEvent(aMsg, { type, data });
|
||||
if (frame) this.scrollSeq++;
|
||||
if (type === 'stage_await_confirm' && data) {
|
||||
patchBioflowForm(aMsg, data);
|
||||
this.scrollSeq++;
|
||||
} else if (type === 'stage_await_files' && data) {
|
||||
patchBioflowFile(aMsg, data);
|
||||
this.scrollSeq++;
|
||||
} else if (type === 'job_done' && data) {
|
||||
const blk = patchBioflowArchiveSuggest(aMsg, data.job_id || jobId);
|
||||
if (blk) blk.jobId = data.job_id || jobId;
|
||||
bioflow.registerReport({
|
||||
jobId: data.job_id || jobId,
|
||||
sessionId: this.sessionId || '',
|
||||
title: bioflow.currentJob.title || '生信分析报告',
|
||||
reportPath: data.report_path || '',
|
||||
figures: Array.isArray(data.figures) ? data.figures : [],
|
||||
}).catch(() => {});
|
||||
this.scrollSeq++;
|
||||
}
|
||||
},
|
||||
});
|
||||
} catch { /* EventSource 不可用:任务事件仍由快照兜底 */ }
|
||||
// 4. 落库(刷新后进度卡不丢;jobId 随块持久化)
|
||||
await this._persistFinal(aMsg, this.sessionId, 'streaming', { raw: aMsg.raw }).catch(() => {});
|
||||
this.scrollSeq++;
|
||||
return detail;
|
||||
},
|
||||
|
||||
/** 上传文件到生信任务(XHR 进度;FR-BIO-006)。返回登记结果。 */
|
||||
async uploadBioflowFileToJob(jobId, file, { onProgress } = {}) {
|
||||
const { uploadBioflowFile } = await import('../api/bioflow.js');
|
||||
@@ -1448,6 +1532,19 @@ export const useChatStore = defineStore('chat', {
|
||||
}
|
||||
},
|
||||
|
||||
/** 缺陷 #27:终态后标记会话来源聚合缓存失效(bench.sourcesStale = true)。
|
||||
* SourcesPane 监听到标记后自动 force 重拉——缓存只在选中会话时有意义,
|
||||
* 终态才触发一次,流式过程中不反复重拉。
|
||||
* 同步实现:经 getActivePinia 取 bench store(与 bench.js 读 chat 的方式对称,
|
||||
* 无动态 import 的微任务窗口——发送后立即断言/挂载渲染都能看到标记)。 */
|
||||
_markSourcesStaleSafe(sessionId) {
|
||||
try {
|
||||
if (sessionId && this.sessionId !== sessionId) return; // 跨会话(切走后后台完成):不打扰当前视图
|
||||
const bench = getActivePinia()?._s.get('bench');
|
||||
bench?.markSourcesStale();
|
||||
} catch { /* bench 未就绪(测试/降级):静默 */ }
|
||||
},
|
||||
|
||||
async _runStream(q, modeCfg, aMsg, sessionId, userId, ctx) {
|
||||
const controller = new AbortController();
|
||||
ctx.controller = controller;
|
||||
@@ -1543,6 +1640,8 @@ export const useChatStore = defineStore('chat', {
|
||||
// 落库失败不掩盖成功事实:本地消息保持 done,仅记录
|
||||
console.warn('persistFinal failed after complete:', persistErr);
|
||||
}
|
||||
// 缺陷 #27:回答完成后会话来源聚合缓存失效(列表自动刷新)
|
||||
this._markSourcesStaleSafe(sessionId);
|
||||
this._finishCtx(ctx, {
|
||||
ok: true,
|
||||
payload,
|
||||
@@ -1675,6 +1774,8 @@ export const useChatStore = defineStore('chat', {
|
||||
const dur = ((Date.now() - startAt) / 1000).toFixed(1) + 's';
|
||||
this._flushPrivate(aMsg); // FR-PRIV-005:blocks 赋值后 flush 私有 pending
|
||||
await this._persistFinal(aMsg, sessionId, 'done', { raw: aMsg.raw });
|
||||
// 缺陷 #27:检索模式回答完成后会话来源聚合缓存失效(列表自动刷新)
|
||||
this._markSourcesStaleSafe(sessionId);
|
||||
this._finishCtx(ctx, {
|
||||
ok: true,
|
||||
payload,
|
||||
|
||||
+37
-5
@@ -75,8 +75,16 @@ export const useDocgenStore = defineStore('docgen', {
|
||||
if (!id) return;
|
||||
const prevId = this.activeArtifact ? (this.activeArtifact.artifact_id || this.activeArtifact.id) : null;
|
||||
const sameArtifact = prevId === id;
|
||||
if (sameArtifact && !force && this.md) return; // 同 id 且有本地内容:复用不重复拉取
|
||||
if (sameArtifact && !force && this.md) {
|
||||
// 同 id 且有本地内容:复用不重复拉取;但资料库视图下重开同一文档
|
||||
// 必须复位视图(缺陷 #47b 自含复位语义)
|
||||
this.viewMode = 'doc';
|
||||
return;
|
||||
}
|
||||
this.activeArtifact = { ...artifact, artifact_id: id };
|
||||
// 缺陷 #47b:打开/切换文档永远回到文档视图——close() 后活动文档已清空,
|
||||
// 重开必须自含地把视图复位;否则关闭→重开后停在空态/资料库视图「看不到文档」。
|
||||
this.viewMode = 'doc';
|
||||
this.loadError = null;
|
||||
this.saveError = null;
|
||||
this.editError = null;
|
||||
@@ -90,6 +98,8 @@ export const useDocgenStore = defineStore('docgen', {
|
||||
this.letterhead = '';
|
||||
this.exportError = {};
|
||||
this.exporting = {};
|
||||
this._contentSeq = (this._contentSeq || 0) + 1; // 拉取归属序号整体换代:上一次打开的全部迟到响应作废
|
||||
this._htmlSeq = (this._htmlSeq || 0) + 1;
|
||||
}
|
||||
if (!this.md) await this.fetchContent(id);
|
||||
},
|
||||
@@ -117,17 +127,21 @@ export const useDocgenStore = defineStore('docgen', {
|
||||
} finally {
|
||||
if (seq === this._contentSeq) this.loading = false;
|
||||
}
|
||||
if (seq === this._contentSeq) await this.fetchHtml(id);
|
||||
// 缺陷 #47b:html 拉取同样以内容序号守卫——文档 A 关闭后其迟到的
|
||||
// html 响应曾覆盖重开文档 B 的内容(空壳残留);归属不匹配即丢弃
|
||||
if (seq === this._contentSeq) await this.fetchHtml(id, seq);
|
||||
},
|
||||
|
||||
/** 拉取模版 HTML(预览与导出同源);失败静默(预览回退客户端 md 渲染)
|
||||
* 2026-09-04 问题5:序号防竞态——迟到的旧 html 响应不再覆盖新文档内容。 */
|
||||
async fetchHtml(id) {
|
||||
* 2026-09-04 问题5:序号防竞态——迟到的旧 html 响应不再覆盖新文档内容。
|
||||
* 缺陷 #47b:contentSeq 一并纳入守卫——文档已关闭/切换后迟到响应作废。 */
|
||||
async fetchHtml(id, contentSeq) {
|
||||
const seq = (this._htmlSeq || 0) + 1;
|
||||
this._htmlSeq = seq;
|
||||
try {
|
||||
const data = await getArtifactHtml(id);
|
||||
if (seq !== this._htmlSeq) return;
|
||||
if (contentSeq != null && contentSeq !== this._contentSeq) return;
|
||||
if (data && data.ok) {
|
||||
this.html = String(data.html || '');
|
||||
this.css = String(data.css || '');
|
||||
@@ -137,6 +151,7 @@ export const useDocgenStore = defineStore('docgen', {
|
||||
}
|
||||
} catch {
|
||||
if (seq !== this._htmlSeq) return;
|
||||
if (contentSeq != null && contentSeq !== this._contentSeq) return;
|
||||
this.html = '';
|
||||
this.css = '';
|
||||
this.renderNonce++; // 值可能不变(此前已空):bump 序号强制预览走 md 回退渲染
|
||||
@@ -158,10 +173,21 @@ export const useDocgenStore = defineStore('docgen', {
|
||||
const id = artifact.artifact_id || artifact.id;
|
||||
if (!id) return;
|
||||
this.activeArtifact = { ...artifact, artifact_id: id };
|
||||
// 缺陷 #47b:生成接管同样自含复位文档视图(与 open 同一状态机);
|
||||
// 拉取归属序号换代——上一文档的全部迟到响应作废
|
||||
this.viewMode = 'doc';
|
||||
this.loadError = null;
|
||||
this.editError = null;
|
||||
this.saveError = null;
|
||||
this.editing = false;
|
||||
this.html = '';
|
||||
this.css = '';
|
||||
this.templateKey = '';
|
||||
this.letterhead = '';
|
||||
this.exportError = {};
|
||||
this.exporting = {};
|
||||
this._contentSeq = (this._contentSeq || 0) + 1;
|
||||
this._htmlSeq = (this._htmlSeq || 0) + 1;
|
||||
this.md = String(initialMd ?? '');
|
||||
if (!this.md) await this.fetchContent(id);
|
||||
},
|
||||
@@ -275,7 +301,10 @@ export const useDocgenStore = defineStore('docgen', {
|
||||
},
|
||||
|
||||
/** 关闭活动文档(切走 doc TAB 不清空;DocPane「关闭文档」入口)
|
||||
* 缺陷 #55:关闭时复位视图模式(回到资料库空态区视图) */
|
||||
* 缺陷 #55:关闭时复位视图模式(回到资料库空态区视图)。
|
||||
* 缺陷 #47b:关闭必须作废全部在途拉取——close() 只清结果状态,
|
||||
* 若之前 open/adopt 的 fetchHtml 仍在途,其迟到响应会把已清空的
|
||||
* html/css/templateKey 重新写回(宿主 v-if 已销毁,回来看不到文档)。 */
|
||||
close() {
|
||||
this.viewMode = 'doc';
|
||||
this.activeArtifact = null;
|
||||
@@ -290,6 +319,9 @@ export const useDocgenStore = defineStore('docgen', {
|
||||
this.saveError = null;
|
||||
this.editError = null;
|
||||
this.exportError = {};
|
||||
// 在途序号作废:迟到的 content/html 响应不得再写任何状态
|
||||
this._contentSeq = (this._contentSeq || 0) + 1;
|
||||
this._htmlSeq = (this._htmlSeq || 0) + 1;
|
||||
},
|
||||
|
||||
/** 会话切换时清空文档域状态(会话级产物不跨会话) */
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
// tests/r3debug/bioflowPathConfirm.test.js —— 生信路径卡确认创建后 UI 断链修复
|
||||
// 缺陷:FlowPathCard.confirmCreate 创建任务成功后 emit('updated') 无人监听,
|
||||
// UI 无进度卡/无事件订阅/不落库(刷新即丢)。修复后走 store 一站式 confirmBioflowPath。
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
import { setActivePinia, createPinia } from 'pinia';
|
||||
|
||||
vi.mock('../../src/api/sessions.js', () => ({
|
||||
createMessage: vi.fn(async () => ({ id: 'm' + Math.random().toString(36).slice(2, 6) })),
|
||||
updateMessage: vi.fn(async () => ({ updated_at: new Date().toISOString() })),
|
||||
}));
|
||||
vi.mock('../../src/api/qa.js', () => ({
|
||||
chatStream: vi.fn(async () => {}),
|
||||
search: vi.fn(async () => ({ answer: '' })),
|
||||
}));
|
||||
vi.mock('../../src/api/private.js', () => ({
|
||||
askPrivate: vi.fn(async () => ({ hit: false })),
|
||||
}));
|
||||
vi.mock('../../src/api/docgen.js', () => ({
|
||||
generate: vi.fn(), getTask: vi.fn(),
|
||||
getArtifactContent: vi.fn(async () => ({ markdown: '' })),
|
||||
getArtifactHtml: vi.fn(async () => ({ ok: true, html: '', css: '' })),
|
||||
saveArtifactContent: vi.fn(async () => ({})), editArtifact: vi.fn(async () => ({})),
|
||||
exportArtifact: vi.fn(async () => ({})), uploadLibrary: vi.fn(),
|
||||
listLibrary: vi.fn(async () => ({ items: [] })), removeLibrary: vi.fn(async () => ({ ok: true })),
|
||||
listTasks: vi.fn(async () => ({ tasks: [] })),
|
||||
}));
|
||||
|
||||
const connectJobEventsSpy = vi.fn();
|
||||
const createJobFromPlanSpy = vi.fn();
|
||||
vi.mock('../../src/stores/bioflow.js', async () => {
|
||||
const actual = await vi.importActual('../../src/stores/bioflow.js');
|
||||
const { useBioflowStore: useReal } = actual;
|
||||
return {
|
||||
...actual,
|
||||
useBioflowStore: () => {
|
||||
const st = useReal();
|
||||
st.connectJobEvents = connectJobEventsSpy;
|
||||
st.createJobFromPlan = createJobFromPlanSpy;
|
||||
return st;
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
import { useChatStore } from '../../src/stores/chat.js';
|
||||
import { useSessionsStore } from '../../src/stores/sessions.js';
|
||||
import { useBioflowStore } from '../../src/stores/bioflow.js';
|
||||
import { updateMessage } from '../../src/api/sessions.js';
|
||||
|
||||
function freshStores() {
|
||||
const pinia = createPinia();
|
||||
setActivePinia(pinia);
|
||||
const chat = useChatStore();
|
||||
const sessions = useSessionsStore();
|
||||
chat.sessionId = 's1';
|
||||
sessions.activeId = 's1';
|
||||
sessions.items = [{ id: 's1', title: 't', status: 'done', created_at: '', updated_at: '' }];
|
||||
return { chat, sessions };
|
||||
}
|
||||
|
||||
describe('confirmBioflowPath 一站式任务创建', () => {
|
||||
beforeEach(() => {
|
||||
connectJobEventsSpy.mockReset();
|
||||
createJobFromPlanSpy.mockReset();
|
||||
vi.mocked(updateMessage).mockClear();
|
||||
});
|
||||
|
||||
it('创建成功 → 挂 job_created 进度块 + 订阅任务事件 + 落库', async () => {
|
||||
const { chat } = freshStores();
|
||||
createJobFromPlanSpy.mockResolvedValue({
|
||||
id: 'job-1', title: '生信任务', status: 'created',
|
||||
stages: [
|
||||
{ stage: 'genome_prepare', status: 'waiting' },
|
||||
{ stage: 'identify', status: 'waiting' },
|
||||
],
|
||||
});
|
||||
const aMsg = {
|
||||
id: 'am1', role: 'assistant', status: 'done',
|
||||
blocks: [
|
||||
{ type: 'bioflow_path', status: 'done', candidates: [{ stages: ['genome_prepare', 'identify'] }], selected: 0 },
|
||||
],
|
||||
raw: {},
|
||||
};
|
||||
chat.messages = [aMsg];
|
||||
|
||||
const detail = await chat.confirmBioflowPath({
|
||||
title: '生信任务',
|
||||
stages: ['genome_prepare', 'identify'],
|
||||
aMsg,
|
||||
});
|
||||
|
||||
// 1. 返回 detail 与创建入参一致
|
||||
expect(detail.id).toBe('job-1');
|
||||
// 2. path 块落定 jobId + selected 固化
|
||||
const pathBlk = aMsg.blocks.find((b) => b.type === 'bioflow_path');
|
||||
expect(pathBlk.jobId).toBe('job-1');
|
||||
expect(pathBlk.selected).toBe(0);
|
||||
// 3. 进度块挂上且 jobStatus=created、阶段 waiting
|
||||
const progBlk = aMsg.blocks.find((b) => b.type === 'bioflow_progress');
|
||||
expect(progBlk).toBeTruthy();
|
||||
expect(progBlk.jobId).toBe('job-1');
|
||||
expect(progBlk.jobStatus).toBe('created');
|
||||
expect(progBlk.stages.length).toBe(2);
|
||||
// 4. 订阅了任务事件流(带 onEvent 桥接)
|
||||
expect(connectJobEventsSpy).toHaveBeenCalledTimes(1);
|
||||
expect(connectJobEventsSpy.mock.calls[0][0]).toBe('job-1');
|
||||
expect(typeof connectJobEventsSpy.mock.calls[0][2].onEvent).toBe('function');
|
||||
// 5. 落库(updateMessage 调用)
|
||||
expect(updateMessage).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('onEvent 桥接把任务事件 patch 进消息进度块(阶段推进)', async () => {
|
||||
const { chat } = freshStores();
|
||||
createJobFromPlanSpy.mockResolvedValue({
|
||||
id: 'job-2', title: '生信任务', status: 'created', stages: [{ stage: 'identify', status: 'waiting' }],
|
||||
});
|
||||
const aMsg = {
|
||||
id: 'am2', role: 'assistant', status: 'done',
|
||||
blocks: [{ type: 'bioflow_path', candidates: [{ stages: ['identify'] }], selected: 0 }],
|
||||
raw: {},
|
||||
};
|
||||
chat.messages = [aMsg];
|
||||
await chat.confirmBioflowPath({ title: 't', stages: ['identify'], aMsg });
|
||||
|
||||
const onEvent = connectJobEventsSpy.mock.calls[0][2].onEvent;
|
||||
onEvent('stage_started', { job_id: 'job-2', stage: 'identify' });
|
||||
onEvent('stage_done', { job_id: 'job-2', stage: 'identify' });
|
||||
|
||||
const progBlk = aMsg.blocks.find((b) => b.type === 'bioflow_progress');
|
||||
const st = progBlk.stages.find((s) => s.stage === 'identify');
|
||||
expect(st.status).toBe('done'); // 终态不回退语义保持
|
||||
});
|
||||
|
||||
it('job_done 事件 → 挂存档提示块', async () => {
|
||||
const { chat } = freshStores();
|
||||
createJobFromPlanSpy.mockResolvedValue({
|
||||
id: 'job-3', title: '生信任务', status: 'created', stages: [{ stage: 'identify', status: 'waiting' }],
|
||||
});
|
||||
const aMsg = {
|
||||
id: 'am3', role: 'assistant', status: 'done',
|
||||
blocks: [{ type: 'bioflow_path', candidates: [{ stages: ['identify'] }], selected: 0 }],
|
||||
raw: {},
|
||||
};
|
||||
chat.messages = [aMsg];
|
||||
await chat.confirmBioflowPath({ title: 't', stages: ['identify'], aMsg });
|
||||
|
||||
connectJobEventsSpy.mock.calls[0][2].onEvent('job_done', { job_id: 'job-3', report_path: '/r.html', figures: [] });
|
||||
|
||||
const archBlk = aMsg.blocks.find((b) => b.type === 'bioflow_archive');
|
||||
expect(archBlk).toBeTruthy();
|
||||
expect(archBlk.jobId).toBe('job-3');
|
||||
});
|
||||
|
||||
it('创建失败不挂块、不落库', async () => {
|
||||
const { chat } = freshStores();
|
||||
createJobFromPlanSpy.mockRejectedValue(new Error('创建失败'));
|
||||
const aMsg = {
|
||||
id: 'am4', role: 'assistant', status: 'done',
|
||||
blocks: [{ type: 'bioflow_path', candidates: [{ stages: ['identify'] }], selected: 0 }],
|
||||
raw: {},
|
||||
};
|
||||
chat.messages = [aMsg];
|
||||
await expect(
|
||||
chat.confirmBioflowPath({ title: 't', stages: ['identify'], aMsg }),
|
||||
).rejects.toThrow('创建失败');
|
||||
expect(aMsg.blocks.some((b) => b.type === 'bioflow_progress')).toBe(false);
|
||||
expect(updateMessage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('hydrateDetail 恢复订阅:含未终态进度块的消息重连事件流', async () => {
|
||||
const { chat, sessions } = freshStores();
|
||||
const progressBlk = { type: 'bioflow_progress', jobId: 'job-9', jobStatus: 'running', stages: [] };
|
||||
sessions.fetchDetail = vi.fn(async () => ({
|
||||
messages: [
|
||||
{ id: 'u1', role: 'user', text: 'q' },
|
||||
{ id: 'a1', role: 'assistant', kind: 'qa', status: 'done', blocks: [progressBlk], raw: {} },
|
||||
],
|
||||
}));
|
||||
await chat.loadSession('s1');
|
||||
|
||||
expect(connectJobEventsSpy).toHaveBeenCalledWith('job-9', 's1', expect.anything());
|
||||
// 终态不重连
|
||||
connectJobEventsSpy.mockClear();
|
||||
sessions.fetchDetail = vi.fn(async () => ({
|
||||
messages: [
|
||||
{ id: 'a2', role: 'assistant', kind: 'qa', status: 'done', blocks: [{ type: 'bioflow_progress', jobId: 'job-10', jobStatus: 'done', stages: [] }], raw: {} },
|
||||
],
|
||||
}));
|
||||
await chat.loadSession('s1');
|
||||
expect(connectJobEventsSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,468 @@
|
||||
// @vitest-environment jsdom
|
||||
// tests/r4debug/r4fixes.test.js —— r4 前端四项缺陷单测(Dev-F 归属)
|
||||
// #27 消息级来源 title 空值 fallback + 新回答完成后会话来源强制重拉
|
||||
// #32 知识图谱搜索成功但空结果的明确提示(与 kgDown/kgEmpty 三态互斥)
|
||||
// #47b DocPane 视图状态机:关闭→重开 / 资料库↔文档往返 / 会话切换
|
||||
// #52a 意图路由 WRITE_VERB 补「总结/概括/归纳」(带附件进 docgen,普通问答不误伤)
|
||||
//
|
||||
// 运行方式(worktree 内):cd ui && npx vitest run --config tests/r3debug/vitest.config.js tests/r4debug/r4fixes.test.js
|
||||
// (r3debug/vitest.config.js 的 include 覆盖 tests/r3debug/**,显式指定文件路径即可跑本文件)
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
import { mount, flushPromises } from '@vue/test-utils';
|
||||
import { setActivePinia, createPinia, getActivePinia } from 'pinia';
|
||||
|
||||
// jsdom 无 scrollTo:polyfill(SourceRow 高亮滚动 / 组件挂载用)
|
||||
if (typeof window !== 'undefined' && typeof window.Element !== 'undefined' && !window.Element.prototype.scrollTo) {
|
||||
window.Element.prototype.scrollTo = vi.fn();
|
||||
}
|
||||
|
||||
// ---- mock 依赖链(与 r3debug/docgenPoll.test.js 同款)----
|
||||
vi.mock('../../src/api/sessions.js', () => ({
|
||||
createMessage: vi.fn(async () => ({ id: 'm' + Math.random().toString(36).slice(2, 6) })),
|
||||
updateMessage: vi.fn(async () => ({ updated_at: new Date().toISOString() })),
|
||||
}));
|
||||
vi.mock('../../src/api/qa.js', () => ({
|
||||
chatStream: vi.fn(async () => {}),
|
||||
search: vi.fn(async () => ({ answer: '' })),
|
||||
}));
|
||||
vi.mock('../../src/api/private.js', () => ({
|
||||
askPrivate: vi.fn(async () => ({ hit: false })),
|
||||
}));
|
||||
vi.mock('../../src/api/docgen.js', () => ({
|
||||
generate: vi.fn(),
|
||||
getTask: vi.fn(),
|
||||
getArtifactContent: vi.fn(async () => ({ markdown: '' })),
|
||||
getArtifactHtml: vi.fn(async () => ({ ok: true, html: '', css: '' })),
|
||||
saveArtifactContent: vi.fn(async () => ({})),
|
||||
editArtifact: vi.fn(async () => ({})),
|
||||
exportArtifact: vi.fn(async () => ({})),
|
||||
uploadLibrary: vi.fn(),
|
||||
listLibrary: vi.fn(async () => ({ items: [] })),
|
||||
removeLibrary: vi.fn(async () => ({ ok: true })),
|
||||
listTasks: vi.fn(async () => ({ tasks: [] })),
|
||||
}));
|
||||
vi.mock('../../src/api/kg.js', () => ({
|
||||
searchEntities: vi.fn(),
|
||||
getNeighborhood: vi.fn(),
|
||||
kgStatus: vi.fn(async () => ({ ok: true, active: true })),
|
||||
}));
|
||||
|
||||
// ---- 被测模块 ----
|
||||
import { useChatStore } from '../../src/stores/chat.js';
|
||||
import { useSessionsStore } from '../../src/stores/sessions.js';
|
||||
import { useBenchStore, __setHttpGet } from '../../src/stores/bench.js';
|
||||
import { useDocgenStore } from '../../src/stores/docgen.js';
|
||||
import SourceRow from '../../src/components/bench/SourceRow.vue';
|
||||
import KgPane from '../../src/components/bench/KgPane.vue';
|
||||
import { route } from '../../src/intent.js';
|
||||
|
||||
function freshPinia() {
|
||||
const pinia = createPinia();
|
||||
setActivePinia(pinia);
|
||||
return pinia;
|
||||
}
|
||||
|
||||
/** 预置会话(跳过 create 网络路径) */
|
||||
function seedSession() {
|
||||
const chat = useChatStore();
|
||||
const sessions = useSessionsStore();
|
||||
chat.sessionId = 's1';
|
||||
sessions.activeId = 's1';
|
||||
sessions.items = [{ id: 's1', title: 't', status: 'done', created_at: '', updated_at: '' }];
|
||||
return { chat, sessions };
|
||||
}
|
||||
|
||||
describe('#27 SourceRow title 空值 fallback(消息级来源条目)', () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
function mountRow(source) {
|
||||
return mount(SourceRow, {
|
||||
props: { source, index: 0, count: null },
|
||||
global: { stubs: { teleport: true } },
|
||||
});
|
||||
}
|
||||
|
||||
it('title 为空 → 显示 paper_id(绝不渲染空文本或正文片段)', () => {
|
||||
const w = mountRow({ paper_id: 'PMC12345678', title: '', snippet: 'RESEARCH ARTICLE 正文片段…', score: 0.21 });
|
||||
const name = w.find('.ft-name');
|
||||
expect(name.text()).toBe('PMC12345678');
|
||||
// tooltip 与显示文本一致(完整 paper_id)
|
||||
expect(name.attributes('title')).toBe('PMC12345678');
|
||||
// 正文片段绝不进入标题行
|
||||
expect(name.text()).not.toContain('RESEARCH ARTICLE');
|
||||
});
|
||||
|
||||
it('title 与 paper_id 都空 → 显示 id', () => {
|
||||
const w = mountRow({ id: 'src-9', title: null, snippet: 'x' });
|
||||
expect(w.find('.ft-name').text()).toBe('src-9');
|
||||
});
|
||||
|
||||
it('title/paper_id/id 全空 → 显示 i18n 占位「(无标题)」', () => {
|
||||
const w = mountRow({ title: ' ', snippet: 'x' });
|
||||
expect(w.find('.ft-name').text()).toBe('(无标题)');
|
||||
});
|
||||
|
||||
it('正常 title 显示不受影响(r3 契约不回归)', () => {
|
||||
const w = mountRow({ paper_id: 'P1', title: '水稻抗病基因研究进展', score: 0.5 });
|
||||
expect(w.find('.ft-name').text()).toBe('水稻抗病基因研究进展');
|
||||
// 徽标真实 paper_id 契约不回归
|
||||
expect(w.find('.ft-ico.idx').text()).toBe('P1');
|
||||
});
|
||||
});
|
||||
|
||||
describe('#27 新回答完成后会话来源强制重拉(sourcesStale 链路)', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
__setHttpGet(null);
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
vi.restoreAllMocks();
|
||||
__setHttpGet(null);
|
||||
});
|
||||
|
||||
it('有缓存且无失效标记 → 缓存短路不重拉', async () => {
|
||||
freshPinia();
|
||||
seedSession();
|
||||
const bench = useBenchStore();
|
||||
bench.sessionSources = { sources: [{ paper_id: 'P1', count: 1 }] };
|
||||
const http = vi.fn(async () => ({ sources: [] }));
|
||||
__setHttpGet(http);
|
||||
const out = await bench.loadSessionSources();
|
||||
expect(out.sources[0].paper_id).toBe('P1');
|
||||
expect(http).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('chat 检索模式完成 → sourcesStale 置真 → 自动绕过缓存重拉,成功后标记清除', async () => {
|
||||
freshPinia();
|
||||
const { chat } = seedSession();
|
||||
const bench = useBenchStore();
|
||||
bench.sessionSources = { sources: [{ paper_id: 'P1', count: 1 }] };
|
||||
const http = vi.fn(async () => ({ sources: [{ paper_id: 'P2', count: 2 }] }));
|
||||
__setHttpGet(http);
|
||||
|
||||
const { search } = await import('../../src/api/qa.js');
|
||||
vi.mocked(search).mockResolvedValue({ answer: 'ok', sources: [{ paper_id: 'P2', title: 't' }], degraded: false });
|
||||
await chat.send('问题', { mode: 'search' });
|
||||
// 终态后失效标记已置
|
||||
expect(bench.sourcesStale).toBe(true);
|
||||
|
||||
const out = await bench.loadSessionSources();
|
||||
expect(http).toHaveBeenCalledWith('/api/sessions/s1/sources', undefined);
|
||||
expect(out.sources[0].paper_id).toBe('P2');
|
||||
expect(bench.sourcesStale).toBe(false); // 成功后清除,不会反复重拉
|
||||
});
|
||||
|
||||
it('QA 流 onComplete 终态 → sourcesStale 置真(终态才一次,不在流式中触发)', async () => {
|
||||
freshPinia();
|
||||
const { chat } = seedSession();
|
||||
const bench = useBenchStore();
|
||||
|
||||
const { chatStream } = await import('../../src/api/qa.js');
|
||||
let captured = null;
|
||||
vi.mocked(chatStream).mockImplementation(async (_body, handlers) => {
|
||||
captured = handlers;
|
||||
handlers.onProgress({ payload: { phase: 'retrieval', label: 'r' } });
|
||||
handlers.onProgress({ payload: { phase: 'generation', label: 'g' } });
|
||||
handlers.onComplete({ payload: { answer: '答案', sources: [] } });
|
||||
});
|
||||
const p = chat.send('问题', { mode: 'ask' });
|
||||
await p;
|
||||
expect(captured).not.toBeNull();
|
||||
// 完成前(流式过程中)不触发失效:终态后才一次
|
||||
expect(chat.messages.find((m) => m.role === 'assistant')?.status).toBe('done');
|
||||
expect(bench.sourcesStale).toBe(true);
|
||||
});
|
||||
|
||||
it('跨会话后台完成不打扰当前视图(sessionId 不匹配时失效标记不置真)', async () => {
|
||||
freshPinia();
|
||||
const { chat } = seedSession();
|
||||
const bench = useBenchStore();
|
||||
// 当前视图在 s2;后台流属于 s1
|
||||
chat.sessionId = 's2';
|
||||
chat._markSourcesStaleSafe('s1');
|
||||
expect(bench.sourcesStale).toBe(false);
|
||||
chat._markSourcesStaleSafe('s2');
|
||||
expect(bench.sourcesStale).toBe(true);
|
||||
});
|
||||
|
||||
it('resetSessionData 清理 sourcesStale(会话切换不残留失效标记)', () => {
|
||||
freshPinia();
|
||||
const bench = useBenchStore();
|
||||
bench.sourcesStale = true;
|
||||
bench.resetSessionData();
|
||||
expect(bench.sourcesStale).toBe(false);
|
||||
expect(bench.sessionSources).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('#32 知识图谱搜索空结果提示(kgNoResult)', () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
function mountKg() {
|
||||
return mount(KgPane, {
|
||||
global: { plugins: [getActivePinia()], stubs: { teleport: true, KgGraphView: true, KgEntityDetail: true } },
|
||||
});
|
||||
}
|
||||
|
||||
it('搜索成功但 results 空 → 显示「未找到相关实体」,搜索按钮恢复可用', async () => {
|
||||
freshPinia();
|
||||
const bench = useBenchStore();
|
||||
bench.kgReady = true;
|
||||
bench.kgStatus = { ok: true };
|
||||
|
||||
const { searchEntities } = await import('../../src/api/kg.js');
|
||||
vi.mocked(searchEntities).mockResolvedValue({ results: [] });
|
||||
|
||||
const w = mountKg();
|
||||
await w.find('.search-box').setValue('水稻');
|
||||
await w.find('.search-btn').trigger('click');
|
||||
await flushPromises();
|
||||
|
||||
expect(w.text()).toContain('未找到相关实体');
|
||||
// 与 kgDown(服务不可用)互斥:不出现离线文案
|
||||
expect(w.text()).not.toContain('知识图谱服务离线');
|
||||
// 搜索按钮恢复可用(空结果不锁按钮)
|
||||
const btn = w.find('.search-btn');
|
||||
expect(btn.attributes('disabled')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('搜索失败(kgDown)不出现空结果提示(触发条件互斥)', async () => {
|
||||
freshPinia();
|
||||
const bench = useBenchStore();
|
||||
bench.kgReady = true;
|
||||
bench.kgStatus = { ok: true };
|
||||
|
||||
const { searchEntities } = await import('../../src/api/kg.js');
|
||||
vi.mocked(searchEntities).mockRejectedValue(new Error('down'));
|
||||
|
||||
const w = mountKg();
|
||||
await w.find('.search-box').setValue('水稻');
|
||||
await w.find('.search-btn').trigger('click');
|
||||
await flushPromises();
|
||||
|
||||
expect(w.text()).toContain('知识图谱服务离线');
|
||||
expect(w.text()).not.toContain('未找到相关实体');
|
||||
});
|
||||
|
||||
it('正常命中路径不回归:arginine 命中渲染结果列表、无「未找到」提示', async () => {
|
||||
freshPinia();
|
||||
const bench = useBenchStore();
|
||||
bench.kgReady = true;
|
||||
bench.kgStatus = { ok: true };
|
||||
|
||||
const { searchEntities } = await import('../../src/api/kg.js');
|
||||
vi.mocked(searchEntities).mockResolvedValue({
|
||||
results: [{ id: 'e1', name: 'arginine', type: 'Compound', support_count: 42 }],
|
||||
});
|
||||
|
||||
const w = mountKg();
|
||||
await w.find('.search-box').setValue('arginine');
|
||||
await w.find('.search-btn').trigger('click');
|
||||
await flushPromises();
|
||||
|
||||
expect(w.find('.result-item').exists()).toBe(true);
|
||||
expect(w.find('.rs-name').text()).toBe('arginine');
|
||||
expect(w.text()).not.toContain('未找到相关实体');
|
||||
});
|
||||
|
||||
it('初始空态 kgEmpty 与空结果提示不同文案(三态区分)', () => {
|
||||
freshPinia();
|
||||
const bench = useBenchStore();
|
||||
bench.kgReady = true;
|
||||
const w = mountKg();
|
||||
expect(w.find('.empty-note').text()).toContain('本次检索未产生图谱关系');
|
||||
expect(w.find('.empty-note').text()).not.toContain('未找到相关实体');
|
||||
});
|
||||
});
|
||||
|
||||
describe('#47b DocPane 视图状态机(close→重开 / 资料库往返 / 会话切换)', () => {
|
||||
beforeEach(() => {
|
||||
freshPinia();
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('打开文档 A → 关闭 → 重开:viewMode 复位为 doc、内容完整渲染', async () => {
|
||||
const docgen = useDocgenStore();
|
||||
await docgen.open({ artifact_id: 'a1', title: 'A', template_name: 't', status: 'done' });
|
||||
expect(docgen.viewMode).toBe('doc');
|
||||
expect(docgen.activeArtifact?.artifact_id).toBe('a1');
|
||||
|
||||
docgen.close();
|
||||
expect(docgen.viewMode).toBe('doc');
|
||||
expect(docgen.activeArtifact).toBeNull();
|
||||
expect(docgen.md).toBe('');
|
||||
expect(docgen.html).toBe('');
|
||||
|
||||
await docgen.open({ artifact_id: 'a1', title: 'A', template_name: 't', status: 'done' });
|
||||
expect(docgen.viewMode).toBe('doc');
|
||||
expect(docgen.activeArtifact?.artifact_id).toBe('a1');
|
||||
expect(docgen.loading).toBe(false);
|
||||
expect(docgen.loadError).toBeNull();
|
||||
});
|
||||
|
||||
it('反复关闭 3 次:状态干净、无残留(html/css/templateKey/letterhead 全空)', async () => {
|
||||
const docgen = useDocgenStore();
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await docgen.open({ artifact_id: 'a1', title: 'A', status: 'done' });
|
||||
docgen.close();
|
||||
expect(docgen.activeArtifact).toBeNull();
|
||||
expect(docgen.md).toBe('');
|
||||
expect(docgen.html).toBe('');
|
||||
expect(docgen.css).toBe('');
|
||||
expect(docgen.templateKey).toBe('');
|
||||
expect(docgen.letterhead).toBe('');
|
||||
expect(docgen.viewMode).toBe('doc');
|
||||
expect(docgen.loading).toBe(false);
|
||||
expect(docgen.loadError).toBeNull();
|
||||
}
|
||||
});
|
||||
|
||||
it('资料库↔文档往返:资料库→返回文档→再资料库,viewMode 切换无残留', async () => {
|
||||
const docgen = useDocgenStore();
|
||||
const { getArtifactContent } = await import('../../src/api/docgen.js');
|
||||
vi.mocked(getArtifactContent).mockResolvedValue({ markdown: '# 文档 A', title: 'A' });
|
||||
await docgen.open({ artifact_id: 'a1', title: 'A', status: 'done' });
|
||||
|
||||
for (let i = 0; i < 3; i++) {
|
||||
docgen.viewMode = 'library';
|
||||
expect(docgen.viewMode).toBe('library');
|
||||
docgen.viewMode = 'doc';
|
||||
expect(docgen.viewMode).toBe('doc');
|
||||
expect(docgen.activeArtifact?.artifact_id).toBe('a1');
|
||||
expect(docgen.md).toBe('# 文档 A');
|
||||
}
|
||||
// 资料库视图下重开同一文档 → open 自含复位回 doc
|
||||
docgen.viewMode = 'library';
|
||||
await docgen.open({ artifact_id: 'a1', title: 'A', status: 'done' });
|
||||
expect(docgen.viewMode).toBe('doc');
|
||||
});
|
||||
|
||||
it('文档 A 关闭后迟到的 html 响应作废:不得复活 A 的状态(#47b 根因)', async () => {
|
||||
const docgen = useDocgenStore();
|
||||
const { getArtifactHtml } = await import('../../src/api/docgen.js');
|
||||
|
||||
// A 的 html 响应悬挂 10s:模拟打开过程中用户点击「关闭文档」(缺陷场景)
|
||||
vi.mocked(getArtifactHtml).mockImplementation(async (id) => {
|
||||
await new Promise((r) => setTimeout(r, 10000));
|
||||
return { ok: true, html: `<h1>${id}</h1>`, css: 'body{}', template_key: 'official', letterhead: '' };
|
||||
});
|
||||
|
||||
const openA = docgen.open({ artifact_id: 'a1', title: 'A', status: 'done' });
|
||||
await vi.advanceTimersByTimeAsync(0); // content 微任务排干,html 已发出(悬挂中)
|
||||
// 打开未完成时关闭:close 必须作废在途拉取
|
||||
docgen.close();
|
||||
expect(docgen.activeArtifact).toBeNull();
|
||||
|
||||
// 放行悬挂响应
|
||||
await vi.advanceTimersByTimeAsync(10000);
|
||||
await flushPromises();
|
||||
await openA.catch(() => {});
|
||||
|
||||
// 关闭后迟到响应被序号守卫作废:A 的 html/css/templateKey 不复活
|
||||
expect(docgen.html).toBe('');
|
||||
expect(docgen.css).toBe('');
|
||||
expect(docgen.templateKey).toBe('');
|
||||
expect(docgen.activeArtifact).toBeNull();
|
||||
|
||||
// 重开文档 B:完整渲染不受 A 残留影响
|
||||
vi.mocked(getArtifactHtml).mockResolvedValue({ ok: true, html: '<h1>B</h1>', css: 'b{}', template_key: 'report', letterhead: '' });
|
||||
await docgen.open({ artifact_id: 'a2', title: 'B', status: 'done' });
|
||||
expect(docgen.activeArtifact?.artifact_id).toBe('a2');
|
||||
expect(docgen.html).toBe('<h1>B</h1>');
|
||||
expect(docgen.viewMode).toBe('doc');
|
||||
});
|
||||
|
||||
it('adoptArtifact 自含复位视图并清空旧内容(生成接管)', async () => {
|
||||
const docgen = useDocgenStore();
|
||||
docgen.viewMode = 'library';
|
||||
docgen.html = '<h1>old</h1>';
|
||||
docgen.css = 'old{}';
|
||||
docgen.templateKey = 'old';
|
||||
docgen.letterhead = 'old';
|
||||
await docgen.adoptArtifact({ artifact_id: 'a1', title: 'A', status: 'done' });
|
||||
expect(docgen.viewMode).toBe('doc');
|
||||
expect(docgen.html).toBe('');
|
||||
expect(docgen.css).toBe('');
|
||||
expect(docgen.templateKey).toBe('');
|
||||
expect(docgen.letterhead).toBe('');
|
||||
expect(docgen.activeArtifact?.artifact_id).toBe('a1');
|
||||
});
|
||||
|
||||
it('会话切换 resetSession 清空文档域:A 不串挂到 B', async () => {
|
||||
const docgen = useDocgenStore();
|
||||
await docgen.open({ artifact_id: 'a1', title: 'A', status: 'done' });
|
||||
expect(docgen.hasActiveArtifact).toBe(true);
|
||||
|
||||
docgen.resetSession();
|
||||
expect(docgen.activeArtifact).toBeNull();
|
||||
expect(docgen.md).toBe('');
|
||||
expect(docgen.html).toBe('');
|
||||
expect(docgen.viewMode).toBe('doc');
|
||||
expect(docgen.templateCandidates).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('#52a 意图路由 WRITE_VERB 补总结/概括/归纳', () => {
|
||||
it('「帮我总结论文」+ 附件 → docgen(规则 4 附件兜底)', () => {
|
||||
expect(route('帮我总结论文', true, false)).toBe('docgen');
|
||||
});
|
||||
|
||||
it('「帮我概括这篇文章」+ 附件 → docgen', () => {
|
||||
expect(route('帮我概括这篇文章', true, false)).toBe('docgen');
|
||||
});
|
||||
|
||||
it('「帮我归纳资料」+ 附件 → docgen', () => {
|
||||
expect(route('帮我归纳资料', true, false)).toBe('docgen');
|
||||
});
|
||||
|
||||
it('无附件「总结一下植物光合作用的过程」→ qa(无其他 DOC_TYPE 证据)', () => {
|
||||
expect(route('总结一下植物光合作用的过程', false, false)).toBe('qa');
|
||||
});
|
||||
|
||||
it('无附件「概括一下 OsSPL14 的功能」→ qa(无其他 DOC_TYPE 证据)', () => {
|
||||
expect(route('概括一下 OsSPL14 的功能', false, false)).toBe('qa');
|
||||
});
|
||||
|
||||
it('无附件「总结这篇论文」→ docgen(TL 裁决:DOC_TYPE 含「论文」,接受该行为变化)', () => {
|
||||
expect(route('总结这篇论文', false, false)).toBe('docgen');
|
||||
});
|
||||
|
||||
it('无附件「帮我概括这篇文章」→ docgen(DOC_TYPE 含「文章」类材料词之外,按 DOC_TYPE 命中)', () => {
|
||||
// 「文章」不在 DOC_TYPE:按漏判安全应走 qa(附规则 4 附件兜底覆盖带附件场景)
|
||||
expect(route('帮我概括这篇文章', false, false)).toBe('qa');
|
||||
expect(route('帮我概括这篇文章', true, false)).toBe('docgen');
|
||||
});
|
||||
|
||||
it('普通问答不误伤:你好 / 介绍自己 → qa', () => {
|
||||
expect(route('你好,介绍一下你自己', false, false)).toBe('qa');
|
||||
expect(route('你是谁', true, false)).toBe('qa');
|
||||
});
|
||||
|
||||
it('既有写类动词回归:写/撰写 + 论文 → docgen;无附件无 DOC_TYPE 写类仍 qa', () => {
|
||||
expect(route('帮我写一份水稻研究报告', false, false)).toBe('docgen');
|
||||
expect(route('撰写论文', false, false)).toBe('docgen');
|
||||
expect(route('写一首诗', false, false)).toBe('qa'); // DOC_TYPE_EXCLUDE 优先
|
||||
});
|
||||
|
||||
it('「写一篇总结」仍进 docgen(DOC_TYPE 含「总结」,行为不变)', () => {
|
||||
expect(route('写一篇总结', false, false)).toBe('docgen');
|
||||
});
|
||||
|
||||
it('既有写类动词 + 总结类文档词不回归:「写一份工作总结」→ docgen', () => {
|
||||
expect(route('写一份工作总结', false, false)).toBe('docgen');
|
||||
});
|
||||
|
||||
it('规则 4 附件兜底:总结类动词带附件无 DOC_TYPE 也进 docgen', () => {
|
||||
expect(route('总结一下这份资料', true, false)).toBe('docgen');
|
||||
expect(route('归纳一下内容', true, false)).toBe('docgen');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,123 @@
|
||||
"""test_vault_dedup.py —— 内容 SHA256 去重(缺陷 #52b,验收 §6 第 2 条 + 回归 G3)。
|
||||
|
||||
运行:cd vstore && /home/xzq/5090/teamWorkspace/zh/plantai-rag-new-venv/bin/python -m pytest tests/test_vault_dedup.py -v
|
||||
(或 vstore/pyproject.toml testpaths 下全量 pytest;venv = plantai-rag-new-venv)
|
||||
|
||||
G3 断言:
|
||||
- 同内容 add 两次 → 库中该内容仅 1 个 doc_id、chunk 总数不翻倍;
|
||||
- 不同内容 add → 2 个 doc_id 互不覆盖;
|
||||
- 同名不同内容 → 按内容哈希不误判(保留 2 个 doc_id)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
|
||||
import pytest
|
||||
|
||||
from vstore.vault import Vault
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def vault(tmp_path, embed_fn, monkeypatch):
|
||||
from vstore.settings import VstorePaths, set_paths
|
||||
|
||||
set_paths(VstorePaths(root=str(tmp_path), vault_path="", index_path="", chunk_path=""))
|
||||
# 桩掉 chunk 批量嵌入(vault._chunked_embed 动态查找),不触网真 Ollama
|
||||
dim = 1024
|
||||
monkeypatch.setattr(
|
||||
"vstore.embed.embed_batch_chunked",
|
||||
lambda texts, *a, **k: [[1.0 / dim] * dim for _ in texts],
|
||||
)
|
||||
return Vault(tmp_path / "vault", tmp_path / "index")
|
||||
|
||||
|
||||
def _doc_ids_for_content(vault, content: bytes) -> list[str]:
|
||||
"""库中 content_sha == sha256(content) 的 doc 行 id 列表(磁盘事实)。"""
|
||||
sha = hashlib.sha256(content).hexdigest()
|
||||
import json
|
||||
|
||||
rows = []
|
||||
try:
|
||||
data = json.loads(vault.index.json_path.read_text(encoding="utf-8"))
|
||||
items = data.get("items", []) if isinstance(data, dict) else []
|
||||
except (ValueError, OSError):
|
||||
items = []
|
||||
for item in items:
|
||||
if isinstance(item, dict) and str(item.get("content_sha") or "") == sha:
|
||||
rows.append(item)
|
||||
return [r.get("id") for r in rows]
|
||||
|
||||
|
||||
def _make_pdf(text: str) -> bytes:
|
||||
"""真实最小 PDF 字节(pymupdf 可解析、含目标文本)。"""
|
||||
import pymupdf
|
||||
|
||||
doc = pymupdf.open()
|
||||
page = doc.new_page()
|
||||
page.insert_text((72, 72), text)
|
||||
return doc.tobytes()
|
||||
|
||||
|
||||
def test_same_content_added_twice_keeps_one_doc_id(vault):
|
||||
"""#52b:同内容 add 两次 → 1 个 doc_id、chunk 数不翻倍、旧密文被清。"""
|
||||
content = _make_pdf("同一份论文内容。")
|
||||
first = vault.add(content, "paper.pdf", {"name": "论文", "category": "资料"})
|
||||
assert first["content_sha"] == hashlib.sha256(content).hexdigest()
|
||||
chunks_after_first = vault.chunks.count()
|
||||
assert chunks_after_first >= 1
|
||||
|
||||
second = vault.add(content, "paper.pdf", {"name": "论文", "category": "资料"})
|
||||
|
||||
ids = _doc_ids_for_content(vault, content)
|
||||
assert ids == [second["document_id"]], f"库中该内容 doc_id 应为仅新 doc:{ids}"
|
||||
assert second["document_id"] != first["document_id"]
|
||||
# 旧密文被清理(覆盖语义不留两份)
|
||||
assert not (vault.vault_dir / f"{first['document_id']}.enc").exists()
|
||||
assert (vault.vault_dir / f"{second['document_id']}.enc").exists()
|
||||
# chunk 总数不翻倍(旧 doc_id 的 chunk 已清理)
|
||||
assert vault.chunks.count() == chunks_after_first
|
||||
# chunk 行全部属于新 doc_id(旧 doc_id 不再出现在检索结果)
|
||||
assert {r["doc_id"] for r in vault.chunks.rows()} == {second["document_id"]}
|
||||
|
||||
|
||||
def test_duplicate_reuses_latest_meta(vault):
|
||||
"""#52b:覆盖语义——重复上传 meta 以本次为准(created_at 刷新、name 覆盖)。"""
|
||||
content = "内容甲。".encode("utf-8")
|
||||
first = vault.add(content, "a.txt", {"name": "旧名"})
|
||||
second = vault.add(content, "a.txt", {"name": "新名"})
|
||||
rows = vault.list_rows()
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["name"] == "新名"
|
||||
assert rows[0]["id"] == second["document_id"]
|
||||
assert first["document_id"] not in [r["id"] for r in rows]
|
||||
|
||||
|
||||
def test_different_content_keeps_two_doc_ids(vault):
|
||||
"""G3-1:不同内容上传 → 2 个 doc_id 互不覆盖。"""
|
||||
a = vault.add("内容甲。".encode("utf-8"), "a.txt", {"name": "甲"})
|
||||
b = vault.add("内容乙。".encode("utf-8"), "b.txt", {"name": "乙"})
|
||||
assert a["document_id"] != b["document_id"]
|
||||
assert len(vault.list_rows()) == 2
|
||||
assert set(_doc_ids_for_content(vault, "内容甲。".encode("utf-8"))) == {a["document_id"]}
|
||||
assert set(_doc_ids_for_content(vault, "内容乙。".encode("utf-8"))) == {b["document_id"]}
|
||||
|
||||
|
||||
def test_same_name_different_content_not_deduped(vault):
|
||||
"""G3-2:同名不同内容 → 按 SHA256 不误判(保留 2 个 doc_id)。"""
|
||||
a = vault.add(_make_pdf("内容甲。"), "same.pdf", {"name": "同名"})
|
||||
b = vault.add(_make_pdf("内容乙。"), "same.pdf", {"name": "同名"})
|
||||
assert a["document_id"] != b["document_id"]
|
||||
assert len(vault.list_rows()) == 2
|
||||
assert vault.chunks.count() >= 2
|
||||
|
||||
|
||||
def test_duplicate_dedup_persists_across_reload(vault, tmp_path, monkeypatch):
|
||||
"""#52b:去重依据(content_sha)落盘持久——重启后(新实例 load)仍不判重。"""
|
||||
content = "持久化内容。".encode("utf-8")
|
||||
vault.add(content, "a.txt", {"name": "甲"})
|
||||
vault2 = Vault(tmp_path / "vault", tmp_path / "index")
|
||||
vault2.index.load()
|
||||
vault2.chunks.load()
|
||||
item = vault2.add(content, "a.txt", {"name": "甲"})
|
||||
assert _doc_ids_for_content(vault2, content) == [item["document_id"]]
|
||||
assert vault2.chunks.count() >= 1
|
||||
+89
-11
@@ -130,22 +130,22 @@ class Vault:
|
||||
|
||||
# ---- 增删改查 ----
|
||||
|
||||
def add(
|
||||
def _add_file(
|
||||
self,
|
||||
doc_id: str,
|
||||
file_bytes: bytes,
|
||||
filename: str,
|
||||
meta: dict[str, Any],
|
||||
as_template: bool = False,
|
||||
name: str,
|
||||
category: str,
|
||||
tags: str,
|
||||
description: str,
|
||||
as_template: bool,
|
||||
) -> dict[str, Any]:
|
||||
"""入库一条。原文只在函数内流转,末尾丢弃引用。"""
|
||||
doc_id = str(uuid.uuid4())
|
||||
name = str(meta.get("name") or filename or "未命名")
|
||||
category = str(meta.get("category") or "")
|
||||
tags = str(meta.get("tags") or "")
|
||||
description = str(meta.get("description") or "")
|
||||
"""入库一条(覆盖语义已由调用方保证:同内容 doc 已被移除)。
|
||||
|
||||
if not file_bytes:
|
||||
raise VaultError("上传文件为空")
|
||||
与旧 add() 唯一差异是 doc_id 由调用方传入;函数内失败照常抛 VaultError,
|
||||
由调用方决定失败回滚语义。
|
||||
"""
|
||||
try:
|
||||
extracted = extract(file_bytes, filename)
|
||||
except ParsingError as exc:
|
||||
@@ -162,6 +162,7 @@ class Vault:
|
||||
"description": description,
|
||||
"as_template": bool(as_template),
|
||||
"size": len(file_bytes),
|
||||
"content_sha": hashlib.sha256(bytes(file_bytes)).hexdigest(),
|
||||
"created_at": _now_iso(),
|
||||
"embed_chunks": 0,
|
||||
}
|
||||
@@ -252,6 +253,7 @@ class Vault:
|
||||
"description": description,
|
||||
"as_template": bool(as_template),
|
||||
"enc_path": str(enc_path),
|
||||
"content_sha": doc_meta["content_sha"],
|
||||
"embed_sha": embed_sha,
|
||||
"embed_chunks": embed_chunks,
|
||||
# FR-PRIV-002:chunk 级索引结果(失败不阻断 doc 级)
|
||||
@@ -260,6 +262,75 @@ class Vault:
|
||||
"chunk_error": chunk_error,
|
||||
}
|
||||
|
||||
def _find_content_duplicate(self, content_sha: str) -> dict[str, Any] | None:
|
||||
"""库中同内容哈希的 doc 行(无则 None)。
|
||||
|
||||
内容哈希存于 index.json 每行的 content_sha 字段(#52b 新增);历史行
|
||||
无该字段(字段缺失/不一致时绝不判重)。直接读 index.json 而非内存
|
||||
self.index._rows——覆盖语义要求与磁盘一致(同内容密文先删后建)。
|
||||
"""
|
||||
try:
|
||||
data = json.loads(self.index.json_path.read_text(encoding="utf-8"))
|
||||
items = data.get("items", []) if isinstance(data, dict) else []
|
||||
except (ValueError, OSError):
|
||||
return None
|
||||
for item in items:
|
||||
if isinstance(item, dict) and str(item.get("content_sha") or "") == content_sha:
|
||||
return item
|
||||
return None
|
||||
|
||||
def add(
|
||||
self,
|
||||
file_bytes: bytes,
|
||||
filename: str,
|
||||
meta: dict[str, Any],
|
||||
as_template: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""入库一条(内容 SHA256 去重,缺陷 #52b)。原文只在函数内流转,末尾丢弃引用。
|
||||
|
||||
去重语义(验收 §6 第 2 条 + 回归 G3):
|
||||
- 内容哈希 = SHA256(原文件字节)。同内容重复上传 → 移除旧 doc 的密文/
|
||||
检索行/chunk 行 → 落一条新 doc;该内容在库中始终仅 1 个 doc_id,
|
||||
chunk 总数不翻倍。
|
||||
- 旧行清理先于新 doc 落盘;新 doc 建库失败时旧 doc 已删(覆盖语义,
|
||||
失败由调用方重传即可,绝无同内容双份)。
|
||||
- 新 doc 的 meta 以本次上传为准(name/category/tags/description 覆盖,
|
||||
created_at 刷新);as_template 以本次为准。
|
||||
"""
|
||||
doc_id = str(uuid.uuid4())
|
||||
name = str(meta.get("name") or filename or "未命名")
|
||||
category = str(meta.get("category") or "")
|
||||
tags = str(meta.get("tags") or "")
|
||||
description = str(meta.get("description") or "")
|
||||
|
||||
if not file_bytes:
|
||||
raise VaultError("上传文件为空")
|
||||
|
||||
# ---- 内容去重(#52b):同内容哈希 → 先清旧 doc 再落新 doc ----
|
||||
content_sha = hashlib.sha256(bytes(file_bytes)).hexdigest()
|
||||
dup = self._find_content_duplicate(content_sha)
|
||||
if dup:
|
||||
old_doc_id = str(dup.get("id") or "")
|
||||
if old_doc_id and old_doc_id != doc_id:
|
||||
try:
|
||||
self.index.remove_item(old_doc_id)
|
||||
except Exception as exc:
|
||||
logger.warning("vault: 去重移除旧检索行失败(%s):%s", old_doc_id, exc)
|
||||
old_enc = self.vault_dir / f"{old_doc_id}.enc"
|
||||
if old_enc.exists():
|
||||
try:
|
||||
secure_delete(old_enc)
|
||||
except OSError as exc:
|
||||
logger.warning("vault: 去重移除旧密文失败(%s):%s", old_enc, exc)
|
||||
try:
|
||||
self.chunks.remove_document(old_doc_id)
|
||||
except Exception as exc:
|
||||
logger.warning("vault: 去重移除旧 chunk 行失败(%s):%s", old_doc_id, exc)
|
||||
|
||||
return self._add_file(
|
||||
doc_id, file_bytes, filename, name, category, tags, description, as_template
|
||||
)
|
||||
|
||||
def get_decrypted(self, doc_id: str) -> dict[str, Any] | None:
|
||||
"""取解密后的文档结构(meta + extracted)。不存在返回 None。"""
|
||||
enc_path = self.vault_dir / f"{doc_id}.enc"
|
||||
@@ -362,6 +433,13 @@ class Vault:
|
||||
row = dict(meta)
|
||||
# keywords 只用 meta,不含正文派生的 title(明文纪律)
|
||||
row["keywords"] = _meta_to_keywords(meta)
|
||||
# content_sha(#52b):历史密文 meta 若缺失则以密文内容补算(与入库时一致),
|
||||
# 保证重建后去重依据不丢
|
||||
if not row.get("content_sha"):
|
||||
try:
|
||||
row["content_sha"] = hashlib.sha256(load_encrypted(enc)).hexdigest()
|
||||
except (CryptoError, VaultError, OSError):
|
||||
row["content_sha"] = ""
|
||||
items.append((row, vectors[0]))
|
||||
try:
|
||||
count = self.index.rebuild(items)
|
||||
|
||||
Reference in New Issue
Block a user