diff --git a/.gitignore b/.gitignore index b246617..8d3182d 100644 --- a/.gitignore +++ b/.gitignore @@ -54,6 +54,12 @@ review_service/data/reformat_jobs/ review_service/tools/tectonic review_service/data/tectonic-cache/ review_service/data/format_cache/ +review_service/data/compile_jobs/ +review_service/data/compile_inspect/ +review_service/prewarm.log # 部署方案文档（含主机凭据，不入库） docs/DEPLOY_PAPER_176.md + +# 一次性诊断脚本（不入库） +scripts/debug_compile.py diff --git a/review_service/app/config.py b/review_service/app/config.py index 6485685..8b57c5b 100644 --- a/review_service/app/config.py +++ b/review_service/app/config.py @@ -62,6 +62,23 @@ class Settings(BaseSettings): compile_queue_limit: int = 5 # 等待槽位上限（>5 → QUEUE_FULL 429） compile_retention_days: int = 7 # 编译产物保留天数（后台清扫） + # ---- LaTeX 项目 zip 编译（P4：两段式 inspect + run）---- + max_zip_bytes: int = 50 * 1024 * 1024 # 上传 zip 上限 50MB + max_extract_bytes: int = 200 * 1024 * 1024 # 单项目解压总大小硬顶 200MB（实写字节计数） + max_zip_entries: int = 2000 # 条目数上限（预扫第一件事） + max_zip_entry_bytes: int = 50 * 1024 * 1024 # 单条目解压上限 50MB + max_zip_ratio: float = 100.0 # 单条目压缩比 file_size/compress_size 上限 + max_zip_path_depth: int = 16 # 路径深度上限（深层嵌套拒绝） + max_zip_path_len: int = 200 # 单条目路径长度上限 + compile_project_timeout_s: float = 120.0 # 项目 zip 编译墙钟上限（单文件仍 compile_timeout_s） + inspect_dir: str = "data/compile_inspect" # zip 解压暂存（1 小时 TTL，相对 review_service/） + inspect_ttl_s: int = 3600 # inspect 暂存过期秒数 + inspect_sweep_interval_s: float = 300.0 # inspect 后台清扫周期（lifespan 定时任务） + compile_jobs_dir: str = "data/compile_jobs" # 项目编译产物（pdf/log/meta，7 天 sweep） + failed_project_ttl_s: int = 24 * 3600 # 编译失败保留 project/ 现场秒数（诊断用） + keep_failed_projects: bool = False # 失败 project/ 永久保留（调试开关； + # 亦读裸环境变量 KEEP_FAILED_PROJECTS=1） + @property def _service_root(self) -> "Path": # review_service/app/config.py → review_service/（相对本文件定位，不依赖 cwd） @@ -91,10 +108,21 @@ class Settings(BaseSettings): return p if p.is_absolute() else self._service_root / p @property - def tectonic_cache_abs(self) -> "Path": + def tectonic_cache_abs(self) -> Path: p = Path(self.tectonic_cache_dir) return p if p.is_absolute() else self._service_root / p + @property + def inspect_dir_abs(self) -> Path: + p = Path(self.inspect_dir) + return p if p.is_absolute() else self._service_root / p + + @property + def compile_jobs_dir_abs(self) -> Path: + p = Path(self.compile_jobs_dir) + return p if p.is_absolute() else self._service_root / p + + @property def vstore_base(self) -> str: return self.vstore_base_url.rstrip("/") diff --git a/review_service/app/main.py b/review_service/app/main.py index 6d3b2c6..c378384 100644 --- a/review_service/app/main.py +++ b/review_service/app/main.py @@ -61,6 +61,33 @@ def create_app() -> FastAPI: sweep_expired() except Exception: # noqa: BLE001 logger.warning("compile sweep 启动清扫失败（忽略）", exc_info=True) + # P4：inspect 暂存（1h TTL）+ 项目编译产物（7 天）启动清扫 + try: + from .services.latex_compile import ( + sweep_compile_jobs_expired, + sweep_inspect_expired, + ) + sweep_inspect_expired() + sweep_compile_jobs_expired() + except Exception: # noqa: BLE001 + logger.warning("P4 sweep 启动清扫失败（忽略）", exc_info=True) + # P4：inspect 暂存后台定时清扫（默认 300s；补充 1——上传不点编译不堆积） + sweep_task: asyncio.Task[None] | None = None + try: + from .services.latex_compile import sweep_inspect_expired as _sweep_insp + + async def _inspect_sweep_loop() -> None: + interval = get_settings().inspect_sweep_interval_s + while True: + await asyncio.sleep(interval) + try: + await asyncio.to_thread(_sweep_insp) + except Exception: # noqa: BLE001 + logger.warning("inspect 后台清扫失败（忽略）", exc_info=True) + + sweep_task = asyncio.create_task(_inspect_sweep_loop()) + except Exception: # noqa: BLE001 + logger.warning("inspect 后台清扫任务启动失败（忽略）", exc_info=True) # journal_directory 自动 seed：空表才导（xlsx 解析约 1-3s，不堵 event loop； # 路径沿用 db.path 注入链——生产=settings.db_path_abs，测试=tmp） if settings.seed_on_startup: @@ -88,6 +115,14 @@ def create_app() -> FastAPI: try: yield finally: + if sweep_task is not None: + sweep_task.cancel() + try: + await sweep_task + except asyncio.CancelledError: + pass + except Exception: # noqa: BLE001 + pass await close_db() app.router.lifespan_context = _lifespan diff --git a/review_service/app/routers/compile.py b/review_service/app/routers/compile.py index 86d61ea..12d1ae5 100644 --- a/review_service/app/routers/compile.py +++ b/review_service/app/routers/compile.py @@ -132,6 +132,107 @@ async def compile_tex( ) +# ================================================================ P4：项目 zip（两段式） + +def _p4_error(code: str, message: str, detail: str = "", + compile_id: str | None = None, + log_url: str | None = None) -> CompileResponse: + """P4 错误信封（补充 3：失败也带 compile_id + log_url，前端可下载完整 log）。""" + err: dict[str, Any] = {"code": code, "message": message, "degraded": False} + if detail: + err["detail"] = detail + return CompileResponse( + ok=False, compile_id=compile_id, log_url=log_url, error=err) + + +def _project_pdf_url(cid: str) -> str: + return f"/api/review/compile/zip/{cid}/pdf" + + +def _project_log_url(cid: str) -> str: + return f"/api/review/compile/zip/{cid}/log" + + +@router.post("/compile/inspect", response_model=CompileResponse) +async def inspect_zip_project( + file: UploadFile = File(..., description="LaTeX 项目 zip（含 main.tex/.cls/图片/.bib）"), +) -> CompileResponse: + """两段式之①：安全解压 + 主文件识别 → job_id（1 小时 TTL，run 前不占编译队列）。 + + 暂用 request.body() 整体缓冲（≤50MB）；Phase 2 改流式上传 + """ + settings = get_settings() + filename = file.filename or "project.zip" + if Path(filename).name != filename or ".." in filename: + return _p4_error("VALIDATION", "非法文件名") + if not filename.lower().endswith(".zip"): + return _p4_error("VALIDATION", "仅支持 .zip 文件") + # 暂用 request.body() 整体缓冲（≤50MB）；Phase 2 改流式上传 + content = await file.read() + if not content: + return _p4_error("VALIDATION", "上传文件为空") + if len(content) > settings.max_zip_bytes: + return _p4_error( + "VALIDATION", + f"zip 超过 {settings.max_zip_bytes // (1024 * 1024)}MB 上限") + try: + meta = await lc.inspect_zip(content) + except lc.InspectError as exc: + return _p4_error(exc.code, exc.message) + return CompileResponse(ok=True, meta=meta) + + +class RunZipRequest(BaseModel): + job_id: str + + +@router.post("/compile/run", response_model=CompileResponse) +async def run_zip_compile(payload: RunZipRequest) -> CompileResponse: + """两段式之②：原子消费 inspect（rename 防并发双跑）→ tectonic 编译 → pdf/log。 + + 队列/工具失败发生在消费**之前** → inspect 保留可重试； + 编译失败（COMPILE_FAILED/TIMEOUT）信封仍带 compile_id + log_url（补充 3）。 + """ + jid = (payload.job_id or "").strip() + if not _CID_RE.fullmatch(jid): + return _p4_error( + "VALIDATION", "job_id 格式非法（应为 12 位 hex）") + try: + result = await lc.compile_project_job(jid) + except lc.InspectError as exc: + return _p4_error(exc.code, exc.message) + except lc.ToolUnavailableError as exc: + return _p4_error("TOOL_UNAVAILABLE", str(exc)) + except lc.QueueFullError as exc: + # 队列满发生在 consume_inspect 之前 → job 未消费，前端可直接重试 + return JSONResponse( + status_code=429, + content={ + "ok": False, + "compile_id": None, "pdf_url": None, "log_url": None, "meta": None, + "error": {"code": "QUEUE_FULL", "message": str(exc), + "degraded": False}, + }, + ) + except lc.CompileError as exc: + # 补充 3：失败也带 compile_id + log_url（cid 由 compile_project_job 附加） + cid = getattr(exc, "compile_id", None) + return _p4_error( + exc.code, str(exc), detail=exc.log_tail, + compile_id=cid, + log_url=_project_log_url(cid) if cid else None) + return CompileResponse( + ok=True, + compile_id=result["compile_id"], + pdf_url=_project_pdf_url(result["compile_id"]), + log_url=_project_log_url(result["compile_id"]), + meta=json.loads((lc.compile_job_dir(result["compile_id"]) / "meta.json") + .read_text("utf-8")) + if (lc.compile_job_dir(result["compile_id"]) / "meta.json").is_file() + else {"compile_id": result["compile_id"], "seconds": result["seconds"]}, + ) + + def _cid_dir(cid: str) -> Path | None: if not _CID_RE.fullmatch(cid or ""): return None @@ -139,6 +240,63 @@ def _cid_dir(cid: str) -> Path | None: return d if d.is_dir() else None +def _zip_cid_dir(cid: str) -> Path | None: + """P4 项目产物目录 data/compile_jobs/{cid}/（独立于 P3a reformat_dir 树）。""" + if not _CID_RE.fullmatch(cid or ""): + return None + d = lc.compile_job_dir(cid) + return d if d.is_dir() else None + + +@router.get("/compile/zip/{compile_id}", response_model=CompileResponse) +async def get_zip_compile(compile_id: str) -> CompileResponse: + cdir = _zip_cid_dir(compile_id) + meta_path = cdir / "meta.json" if cdir else None + if meta_path is None or not meta_path.is_file(): + return _p4_error("NOT_FOUND", "编译任务不存在或已过期，请重新编译") + try: + meta = json.loads(meta_path.read_text("utf-8")) + except (OSError, ValueError): + return _p4_error("NOT_FOUND", "编译产物损坏，请重新编译") + return CompileResponse( + ok=True, + compile_id=compile_id, + pdf_url=_project_pdf_url(compile_id), + log_url=_project_log_url(compile_id), + meta=meta, + ) + + +@router.get("/compile/zip/{compile_id}/pdf", response_model=None) +async def download_zip_pdf(compile_id: str) -> FileResponse | JSONResponse: + cdir = _zip_cid_dir(compile_id) + pdf = cdir / "output.pdf" if cdir else None + if pdf is None or not pdf.is_file(): + return JSONResponse( + status_code=200, + content={"ok": False, "error": { + "code": "NOT_FOUND", "message": "PDF 不存在或已过期", + "degraded": False}}, + ) + return FileResponse(pdf, media_type="application/pdf", + filename=f"{compile_id}.pdf") + + +@router.get("/compile/zip/{compile_id}/log", response_model=None) +async def download_zip_log(compile_id: str) -> PlainTextResponse | JSONResponse: + """补充 3：成功/失败都可下载 output.log。""" + cdir = _zip_cid_dir(compile_id) + log = cdir / "output.log" if cdir else None + if log is None or not log.is_file(): + return JSONResponse( + status_code=200, + content={"ok": False, "error": { + "code": "NOT_FOUND", "message": "日志不存在或已过期", + "degraded": False}}, + ) + return PlainTextResponse(log.read_text("utf-8", errors="replace")) + + @router.get("/compile/{compile_id}", response_model=CompileResponse) async def get_compile(compile_id: str) -> CompileResponse: cdir = _cid_dir(compile_id) diff --git a/review_service/app/services/latex_compile.py b/review_service/app/services/latex_compile.py index b396769..a4e5b6d 100644 --- a/review_service/app/services/latex_compile.py +++ b/review_service/app/services/latex_compile.py @@ -1,4 +1,4 @@ -"""services/latex_compile.py —— tectonic 编译 .tex → PDF（P3a）。 +"""services/latex_compile.py —— tectonic 编译 .tex → PDF（P3a + P4 项目 zip）。 隔离与安全（对齐设计清单）： 1. 一编译一目录：{reformat_dir}/{anchor_id}/compile/（input.tex/output.pdf/output.log/meta.json） @@ -9,6 +9,13 @@ 6. 产物 PDF > max_compile_pdf_bytes → 删文件 + OutputTooLargeError 7. 启动/每次编译前机会性清扫 retention_days 天前的 job 目录（只删 12 位 hex 锚点目录） +P4 项目 zip（两段式 inspect + run，与 P3a 共享单槽队列/清扫/错误类）： + - inspect_zip：安全解压（zip_safety）→ 主文件识别 → compile_inspect/{job_id}/（1h TTL） + - compile_project_job：原子 rename 消费 inspect → compile_jobs/{cid}/ → tectonic → + finally 删 project 树（只留 output.pdf/output.log/meta.json） + - 失败也保证 output.log 落盘（补充 3）；classify_compile_error 给出用户友好文案 + - P3a 单文件路径（prepare_input/compile_tex/_run_once）零改动 + tectonic 二进制缺失 → ToolUnavailableError（路由层回安装指引）。 """ from __future__ import annotations @@ -21,6 +28,8 @@ import re import shutil import signal import time +import uuid +from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -40,6 +49,9 @@ _pending = 0 class CompileError(Exception): """编译失败（含 log 尾）。code ∈ {COMPILE_FAILED, TIMEOUT, OUTPUT_TOO_LARGE}。""" + # compile_id：P4 项目编译失败时由 compile_project_job 附加（P3a 单文件为 None）， + # 路由层据此在失败信封里回 compile_id + log_url（补充 3）。 + compile_id: str | None = None def __init__(self, code: str, message: str, log_tail: str = "") -> None: super().__init__(message) @@ -234,3 +246,507 @@ def sweep_expired() -> int: logger.info("compile sweep: 清理 %d 个过期任务目录（>%d 天）", removed, settings.compile_retention_days) return removed + + +# ================================================================ P4：项目 zip（inspect + run） +# +# 与 P3a 共用：_lock/_pending 单槽队列、_ensure_tool、CompileError/QueueFullError/ +# ToolUnavailableError、_kill_group、_log_tail、12-hex 目录与 mtime sweep 约定。 +# P3a 函数（prepare_input/compile_tex/_run_once）零改动。 + + +class InspectError(Exception): + """inspect 业务失败。code ∈ {VALIDATION, ZIP_UNSAFE, ZIP_BOMB, ZIP_INVALID}。""" + + def __init__(self, code: str, message: str) -> None: + super().__init__(message) + self.code = code + self.message = message + + +_DOCCLASS_RE = re.compile(r"(? Path: + """data/compile_inspect/（1 小时 TTL 暂存根，存在性由调用方按需 mkdir）。""" + return get_settings().inspect_dir_abs + + +def compile_job_dir(cid: str) -> Path: + """data/compile_jobs/{cid}/（产物根；不走 reformat_dir，P4 独立清扫）。""" + return get_settings().compile_jobs_dir_abs / cid + + +def _strip_tex_comments(text: str) -> str: + """粗剥注释：行内 % 截断（\\% 误伤可接受，MVP 设计决策）。""" + return "\n".join(ln.split("%", 1)[0] for ln in text.splitlines()) + + +def _read_tex(path: Path) -> str: + """utf-8 → gb18030 → latin-1 降级读（中文模板常见 GBK）。""" + raw = path.read_bytes() + for enc in ("utf-8", "gb18030", "latin-1"): + try: + return raw.decode(enc) + except UnicodeDecodeError: + continue + return raw.decode("utf-8", errors="replace") + + +def find_main_tex(project_dir: Path) -> tuple[str | None, list[str], int]: + """主文件识别（补充 2）→ (main_rel | None, 含 documentclass 的相对路径清单, .tex 总数)。 + + 场景：唯一候选 → main；0 个 .tex / 0 候选 / ≥2 候选 → None + 清单（调用方报错）。 + 返回的相对路径用 posix 分隔符（与 zip 条目名一致）。 + """ + tex_files = sorted( + p for p in project_dir.rglob("*") + if p.is_file() and p.suffix.lower() == ".tex" + ) + if not tex_files: + return None, [], 0 + candidates: list[str] = [] + for p in tex_files: + if _DOCCLASS_RE.search(_strip_tex_comments(_read_tex(p))): + candidates.append(p.relative_to(project_dir).as_posix()) + if len(candidates) == 1: + return candidates[0], candidates, len(tex_files) + return None, candidates, len(tex_files) + + +_MISSING_FILE_RE = re.compile( + r"LaTeX Error: File [`']([^`']+)' not found" + r"|I can't find file [`']([^`']+)'" +) +_IMG_INCLUSION_RE = re.compile(r"""image inclusion failed for ["']([^"']+)["']""") + +_EPS_CONVERTED_SUFFIX = "-eps-converted-to.pdf" +_REF_TEXT_EXTS = {".tex", ".cls", ".sty", ".cfg", ".def"} + + +def fallback_eps_to_converted_pdf(project: Path) -> list[str]: + """EPS 兼容回退：XeTeX/xdvipdfmx 无法内嵌 EPS（MDPI 模板 logo 即此例）。 + + 对项目内每个 X.eps，若同目录存在预转换版 X-eps-converted-to.pdf + （epstopdf/MDPI 官方约定命名）： + 1. 显式引用：把 .tex/.cls/.sty/.cfg/.def 中的 `X.eps` 替换为转换版文件名 + （纯字节 ASCII 替换，不触碰文件编码）； + 2. 缺省扩展名引用：graphicx 的 `{.../X}` 解析顺序为 .pdf→…→.eps， + 若无 X.pdf 则把转换版复制为 X.pdf，规避 .eps 命中 + （MDPI 正文模板存在 `\includegraphics{Definitions/logo-mdpi}` 写法）。 + → 被改写的文件相对路径列表（无可用转换版/无引用 → 空）。 + """ + eps_files = [ + p for p in project.rglob("*") + if p.is_file() and p.suffix.lower() == ".eps" + ] + if not eps_files: + return [] + pairs: list[tuple[bytes, bytes]] = [] + for eps in eps_files: + converted = eps.with_name(eps.stem + _EPS_CONVERTED_SUFFIX) + if not converted.is_file(): + continue + pairs.append((eps.name.encode("ascii"), converted.name.encode("ascii"))) + plain_pdf = eps.with_suffix(".pdf") + if not plain_pdf.exists(): + shutil.copyfile(converted, plain_pdf) + logger.info("eps fallback: 已复制 %s → %s（缺省扩展名解析规避 .eps）", + converted.name, plain_pdf.name) + if not pairs: + return [] + changed: list[str] = [] + for f in project.rglob("*"): + if not (f.is_file() and f.suffix.lower() in _REF_TEXT_EXTS): + continue + data = f.read_bytes() + new = data + for old_b, new_b in pairs: + new = new.replace(old_b, new_b) + if new != data: + f.write_bytes(new) + changed.append(f.relative_to(project).as_posix()) + if changed: + logger.info( + "eps fallback: %d 个文件的 .eps 引用已切换为 *%s", + len(changed), _EPS_CONVERTED_SUFFIX) + return changed + + +def classify_compile_error(log_tail: str) -> str: + """COMPILE_FAILED 的日志 → 用户友好文案（Q4/§4.4）。""" + m = _MISSING_FILE_RE.search(log_tail) + if m: + name = next(g for g in m.groups() if g) + ext = Path(name).suffix.lower() + if ext == ".cls": + return f"项目缺少 {name}，请从期刊官网下载模板后重新上传" + if ext in (".sty", ".def", ".clo"): + return f"缺少 LaTeX 宏包 {name}，可能是模板不完整" + if ext in _IMG_LIKE_EXT: + return f"图片 {name} 未在 zip 里，请检查引用路径与大小写" + if ext == ".bib": + return f"缺少文献库 {name}，请确认 .bib 已包含在 zip 中" + return f"编译找不到文件 {name}，请检查 zip 内文件路径" + m = _IMG_INCLUSION_RE.search(log_tail) + if m: + name = m.group(1) + if Path(name).suffix.lower() == ".eps": + stem = Path(name).stem + return ( + f"图片 {name} 无法内嵌（EPS 在当前引擎下不受支持）：" + f"请提供 {stem}{_EPS_CONVERTED_SUFFIX} 预转换 PDF，或改用 PDF/PNG 图片" + ) + return f"图片 {name} 无法内嵌，请检查图片格式与路径（建议使用 PDF/PNG）" + if re.search(r"Emergency stop|Undefined control sequence", log_tail): + return "LaTeX 源码有语法错误，请展开日志定位出错行" + return "编译失败，请查看日志定位 LaTeX 错误" + + +async def inspect_zip(zip_bytes: bytes) -> dict[str, Any]: + """两段式之①：安全解压 + 主文件识别 + 落盘 inspect 暂存（1h TTL）。 + + → {job_id, main, stats{files,tex,images,cls,bib,other,bytes}, skipped[], skipped_count} + 失败抛 InspectError（路由层转信封）；任何失败 → rmtree 暂存目录，磁盘零残留。 + """ + from .zip_safety import ZipSafetyError, safe_extract + + settings = get_settings() + sweep_inspect_expired() # 机会性清扫（同 P3a 模式） + job_id = uuid.uuid4().hex[:12] + jdir = inspect_root() / job_id + project = jdir / "project" + try: + try: + result = await asyncio.to_thread(safe_extract, zip_bytes, project, settings) + except ZipSafetyError as exc: + raise InspectError(exc.code, exc.message) from exc + main, candidates, tex_total = find_main_tex(project) + if tex_total == 0: + raise InspectError("VALIDATION", "zip 中没有 .tex 文件") + if not candidates: + raise InspectError( + "VALIDATION", "未找到含 \\documentclass 的 .tex 文件") + if main is None: + shown = ", ".join(Path(c).name for c in candidates) + raise InspectError( + "VALIDATION", + f"检测到多个主文件（{shown}），请合并为单一入口后重新上传") + meta = { + "job_id": job_id, + "main": main, + "stats": { + "files": result.files, "tex": result.tex, "images": result.images, + "cls": result.cls, "bib": result.bib, "other": result.other, + "bytes": result.bytes, + }, + "skipped": [{"name": s.name, "reason": s.reason} for s in result.skipped], + "skipped_count": result.skipped_count, + "created_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + } + jdir.mkdir(parents=True, exist_ok=True) + (jdir / "inspect.json").write_text( + json.dumps(meta, ensure_ascii=False, indent=2), encoding="utf-8") + return meta + except BaseException: + shutil.rmtree(jdir, ignore_errors=True) + raise + + +def consume_inspect(job_id: str, cid: str) -> tuple[Path, dict[str, Any]]: + """两段式之②前置：**原子 rename 消费** inspect 暂存 → compile_jobs/{cid}/project。 + + 并发正确性（防同一 job 双跑）： + os.rename(src, dst) 在同一文件系统内是原子的，且 src 不存在 → FileNotFoundError。 + 两个并发 run 拿同一 job_id 时，只有一个能成功 rename；后到者拿到 + FileNotFoundError → 路由层转 VALIDATION"已过期/不存在"，**绝不可能双编译**。 + inspect 残壳（inspect.json）在 rename 成功后立即删除，也杜绝二跑读到半状态。 + """ + settings = get_settings() + if not _ANCHOR_ID_RE.fullmatch(job_id or ""): + raise InspectError("VALIDATION", "job_id 格式非法（应为 12 位 hex）") + src_root = inspect_root() / job_id + meta_path = src_root / "inspect.json" + src_project = src_root / "project" + if not (meta_path.is_file() and src_project.is_dir()): + raise InspectError( + "VALIDATION", "上传任务不存在或已过期（1 小时），请重新上传 zip") + try: + meta = json.loads(meta_path.read_text("utf-8")) + except (OSError, ValueError) as exc: + raise InspectError( + "VALIDATION", "上传任务数据损坏，请重新上传 zip") from exc + dst_root = compile_job_dir(cid) + dst_project = dst_root / "project" + dst_root.mkdir(parents=True, exist_ok=True) + try: + os.rename(src_project, dst_project) + except FileNotFoundError as exc: + raise InspectError( + "VALIDATION", "上传任务不存在或已过期（1 小时），请重新上传 zip") from exc + except OSError as exc: # 跨设备等罕见失败 → 保持 inspect 原样可重试 + raise InspectError( + "VALIDATION", "任务消费失败，请重试或重新上传") from exc + shutil.rmtree(src_root, ignore_errors=True) # 消费成功 → 删残壳 + return dst_project, meta + + +async def compile_project_job(job_id: str) -> dict[str, Any]: + """两段式之②：原子消费 inspect → tectonic 编译 → 产物留 pdf/log/meta。 + + 队列/工具检查**先于消费**（inspect 保留 → 失败可直接重试）； + EPS 回退：存在 *-eps-converted-to.pdf 时自动把 .eps 引用切换为转换版； + 成功 → 立即删 project/；失败（COMPILE_FAILED/TIMEOUT）→ **保留 project/ 现场 24h** + 供诊断（sweep_compile_jobs_expired 按 failed_project_ttl_s 清理； + KEEP_FAILED_PROJECTS=1 永久保留）； + 失败必落完整 output.log（含 --print 的 XDV→PDF 阶段输出，补充 3）。 + + → {compile_id, pdf_path, log_path, seconds, pdf_bytes} + """ + global _pending + settings = get_settings() + _ensure_tool() # 快速失败：不消费 inspect、不占队列 + sweep_inspect_expired() + sweep_compile_jobs_expired() + if _pending >= settings.compile_queue_limit: + raise QueueFullError( + f"编译队列已满（{_pending} 个任务排队），请稍后重试") + _pending += 1 # 占位先于消费：QUEUE_FULL 时 inspect 未动 + cid = uuid.uuid4().hex[:12] + project: Path | None = None + meta: dict[str, Any] = {} + try: + project, meta = await asyncio.to_thread(consume_inspect, job_id, cid) + try: + # EPS 回退：xdvipdfmx 无法内嵌 EPS（MDPI 模板 logo 即此例）； + # 存在 *-eps-converted-to.pdf 预转换版时自动切换引用（失败现场保留改写后文件） + await asyncio.to_thread(fallback_eps_to_converted_pdf, project) + async with _lock: + result = await _run_project(cid, project, str(meta.get("main", ""))) + # 成功 → 立即清理 project 树（失败路径在 finally 保留现场） + shutil.rmtree(project, ignore_errors=True) + project = None + return result + except CompileError as exc: + # 补充 3：失败信封也要能下载 log → 异常携带产物 cid（log 落盘于 compile_jobs/{cid}/） + exc.compile_id = cid + raise + finally: + _pending -= 1 + if project is not None: + # 编译失败 → 保留 compile_jobs/{cid}/project/ 现场供诊断； + # touch 目录使 24h 保留窗从失败时刻起算 + try: + os.utime(project, None) + except OSError: + pass + + +async def _run_project(cid: str, project: Path, main_rel: str) -> dict[str, Any]: + """单槽内执行项目编译：cwd=main 所在目录（相对路径解析），outdir=compile_jobs/{cid}。""" + settings = get_settings() + outdir = compile_job_dir(cid) + outdir.mkdir(parents=True, exist_ok=True) + main_path = project / main_rel + if not main_rel or not main_path.is_file(): + raise InspectError("VALIDATION", "主文件缺失，请重新上传 zip") + cwd = main_path.parent + stem = main_path.stem + + env = dict(os.environ) + env["TECTONIC_CACHE_DIR"] = str(settings.tectonic_cache_abs) + # --print：打印引擎完整 chatter（含 XDV→PDF/xdvipdfmx 阶段输出）； + # 不加时 xdvipdfmx 报错只进 tectonic stderr、不落 {stem}.log，导致"LaTeX 成功但 + # 无 PDF"且 log 无错误行（MDPI 模板实测）。stdout/stderr 统一落盘 output.log。 + cmd = [ + str(_tectonic_bin()), "-X", "compile", main_path.name, + "--untrusted", "--keep-logs", "--print", "--outdir", str(outdir), + ] + timeout = settings.compile_project_timeout_s + log_path = outdir / "output.log" + started = time.monotonic() + try: + proc = await asyncio.create_subprocess_exec( + *cmd, cwd=str(cwd), env=env, + stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT, + start_new_session=True, + ) + try: + stdout_bytes, _ = await asyncio.wait_for( + proc.communicate(), timeout=timeout) + except asyncio.TimeoutError: + _kill_group(proc) + try: + await asyncio.wait_for(proc.wait(), timeout=5.0) + except asyncio.TimeoutError: + pass + # 超时后 communicate 已取消，chatter 不可取；.log（--keep-logs）仍完整 + _ensure_project_log(outdir, stem, cwd) + # 追加而非覆盖：保留完整引擎输出（诊断 XDV→PDF 阶段必需） + with log_path.open("a", encoding="utf-8") as fh: + fh.write(f"\n[timeout] 编译超时（{timeout:g}s），进程已终止\n") + raise CompileError( + "TIMEOUT", + f"编译超过 {timeout:g} 秒未完成，已终止" + "（项目首次编译需下载宏包，可能较慢，请重试）", + _log_tail(log_path), + ) from None + except FileNotFoundError as exc: + raise ToolUnavailableError(f"tectonic 不可执行：{exc}") from exc + finally: + seconds = round(time.monotonic() - started, 2) + + _ensure_project_log(outdir, stem, cwd) + + # 产物归一：{stem}.pdf/.log → output.pdf/.log（补充 3：失败也必有 output.log） + produced_pdf = outdir / f"{stem}.pdf" + produced_log = outdir / f"{stem}.log" + if produced_log.is_file(): + shutil.move(str(produced_log), str(log_path)) + if not log_path.is_file(): + log_path.write_text( + "[no log] tectonic 未产出日志（进程异常退出）", encoding="utf-8") + # --print 的进程输出（含 XDV→PDF 阶段）追加到 output.log 全文之后； + # 信封 detail 仍只取尾 200 行，磁盘保留完整 log 供"查看完整 log"下载 + _append_chatter(log_path, stdout_bytes) + + if not produced_pdf.is_file(): + raise CompileError( + "COMPILE_FAILED", + classify_compile_error(_log_tail(log_path)), + _log_tail(log_path), + ) + pdf_bytes = produced_pdf.stat().st_size + if pdf_bytes > settings.max_compile_pdf_bytes: + produced_pdf.unlink(missing_ok=True) + raise CompileError( + "OUTPUT_TOO_LARGE", + f"产出 PDF 超过 {settings.max_compile_pdf_bytes // (1024 * 1024)}MB 上限，已删除", + _log_tail(log_path), + ) + produced_pdf.rename(outdir / "output.pdf") + + meta = { + "compile_id": cid, + "project": True, + "main": main_rel, + "engine": _ENGINE, + "seconds": seconds, + "pdf_bytes": pdf_bytes, + "returncode": 0, + "compiled_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + } + (outdir / "meta.json").write_text( + json.dumps(meta, ensure_ascii=False, indent=2), encoding="utf-8") + logger.info("compile(project): %s main=%s → PDF %d bytes, %.2fs", + cid, main_rel, pdf_bytes, seconds) + return { + "compile_id": cid, + "pdf_path": outdir / "output.pdf", + "log_path": log_path, + "seconds": seconds, + "pdf_bytes": pdf_bytes, + } + + +def _ensure_project_log(outdir: Path, stem: str, cwd: Path) -> None: + """tectonic 落 log 到 outdir 或 cwd → 统一挪到 outdir/output.log；都没有则占位。""" + log_path = outdir / "output.log" + if log_path.is_file(): + return + for cand in (outdir / f"{stem}.log", cwd / f"{stem}.log"): + if cand.is_file(): + shutil.move(str(cand), str(log_path)) + return + + +def _append_chatter(log_path: Path, stdout_bytes: bytes | None) -> None: + """把 tectonic --print 的进程 stdout（含 XDV→PDF/xdvipdfmx 阶段）追加到 output.log。 + + {stem}.log 只有 LaTeX 引擎内容（止于 "Output written on … .xdv"）； + XDV→PDF 转换错误只出现在进程 chatter 里——不落盘则"无 PDF 且 log 无错误行"。 + """ + if not stdout_bytes: + return + chatter = stdout_bytes.decode("utf-8", errors="replace").strip() + if not chatter: + return + with log_path.open("a", encoding="utf-8") as fh: + fh.write("\n\n===== tectonic --print（进程输出，含 XDV→PDF 阶段）=====\n") + fh.write(chatter) + fh.write("\n") + + +def sweep_inspect_expired() -> int: + """删 mtime 超 inspect_ttl_s（默认 1h）的 12-hex inspect 暂存目录。""" + settings = get_settings() + base = settings.inspect_dir_abs + if not base.is_dir(): + return 0 + cutoff = time.time() - settings.inspect_ttl_s + removed = 0 + for child in base.iterdir(): + if not child.is_dir() or not _ANCHOR_ID_RE.fullmatch(child.name): + continue + try: + if child.stat().st_mtime < cutoff: + shutil.rmtree(child, ignore_errors=True) + removed += 1 + except OSError: + continue + if removed: + logger.info("inspect sweep: 清理 %d 个过期 inspect 暂存（>%d 秒）", + removed, settings.inspect_ttl_s) + return removed + + +def _keep_failed_projects() -> bool: + """失败现场是否永久保留（调试）：REVIEW_KEEP_FAILED_PROJECTS=1 或裸 KEEP_FAILED_PROJECTS=1。""" + if get_settings().keep_failed_projects: + return True + return os.environ.get("KEEP_FAILED_PROJECTS", "").strip() == "1" + + +def sweep_compile_jobs_expired() -> int: + """编译产物清扫，两级： + ① 失败现场 project/ 子目录：mtime 超 failed_project_ttl_s（默认 24h）→ 删子目录 + （compile_jobs/{cid} 本体与 output.log/meta 保留，供"查看完整 log"）； + KEEP_FAILED_PROJECTS=1 跳过（永久保留，调试用）。 + ② 整个 compile_jobs/{cid}：mtime 超 compile_retention_days（7 天）→ 整目录删。 + """ + settings = get_settings() + base = settings.compile_jobs_dir_abs + if not base.is_dir(): + return 0 + removed = 0 + keep_failed = _keep_failed_projects() + # ① 失败现场 24h 清理（成功编译的 project/ 已在 compile_project_job 立即删） + if not keep_failed: + proj_cutoff = time.time() - settings.failed_project_ttl_s + for child in base.iterdir(): + if not child.is_dir() or not _ANCHOR_ID_RE.fullmatch(child.name): + continue + proj = child / "project" + if not proj.is_dir(): + continue + try: + if proj.stat().st_mtime < proj_cutoff: + shutil.rmtree(proj, ignore_errors=True) + removed += 1 + except OSError: + continue + # ② 整目录 7 天清理 + cutoff = time.time() - settings.compile_retention_days * 86400 + for child in base.iterdir(): + if not child.is_dir() or not _ANCHOR_ID_RE.fullmatch(child.name): + continue + try: + if child.stat().st_mtime < cutoff: + shutil.rmtree(child, ignore_errors=True) + removed += 1 + except OSError: + continue + if removed: + logger.info("compile_jobs sweep: 清理 %d 个过期项目产物/失败现场", removed) + return removed diff --git a/review_service/app/services/zip_safety.py b/review_service/app/services/zip_safety.py new file mode 100644 index 0000000..0608eee --- /dev/null +++ b/review_service/app/services/zip_safety.py @@ -0,0 +1,288 @@ +r"""services/zip_safety.py —— LaTeX 项目 zip 安全校验与解压（P4，纯函数模块）。 + +安全清单（每条攻击 → 防御位置，对应设计 §2）： + #1 路径穿越 ../../etc/passwd → sanitize_entry_name：拒绝 .. 段/绝对路径/盘符/空段 + （\ 已先归一为 /，Windows 反斜杠变体同规则） + #2 反斜杠变体 ..\\..\\x → 同上（name.replace("\\\\", "/") 后再校验） + #3 符号链接条目 → _is_symlink：external_attr 的 S_IFMT == S_IFLNK → 整包拒绝 + #4 可执行扩展名 .exe/.sh/.so → classify_entry：DENY_EXT → 整包拒绝（点名条目） + #5 Unix 可执行位 0o111 → _has_exec_bit：普通文件带 x 位 → 整包拒绝（目录 0o755 例外） + #6 无害非白名单 .docx/.log/.zip → classify_entry：skip（记 skipped[]；.zip 跳过=天然防嵌套递归） + #7 zip bomb 压缩比 → 预扫 file_size/compress_size > max_zip_ratio → ZIP_BOMB + #8 zip bomb 总量 → 预扫声明累计 > max_extract_bytes → ZIP_BOMB； + 流式写盘按**实际字节**再计数（不信头部声明）超限即中止 + #9 zip bomb 单条目 → 预扫 file_size > max_entry_bytes → ZIP_BOMB；实写计数兜底 + #10 zip bomb 条目数 → 预扫 len(infolist) > max_zip_entries → ZIP_BOMB（先于一切写盘） + #11 深嵌套/超长路径 → 路径深度 > max_path_depth 或总长 > max_path_len → ZIP_UNSAFE + #12 损坏/加密 zip → zipfile.BadZipFile / RuntimeError → ZIP_INVALID + #13 头部谎报小文件 → 实写计数兜底（#8）；CRC 错 → ZIP_INVALID + #14 恶意 .tex shell escape → 不在本模块范围（tectonic --untrusted 防护，P3a 已实证） + #15 解压阻塞事件循环 → 调用方 asyncio.to_thread 包裹；本模块上限 200MB/2000 条封顶 + +设计要点： + - 两段式：①纯预扫（所有结构性检查，全部通过才开始写）→ ②流式写盘（8KB 块 + 实字节计数）。 + 任何异常 → 调用方 rmtree(dest)，磁盘零残留。 + - 绝不 extractall；绝不保留可执行位（写盘一律 0o644）。 + - 目录条目只建不校验扩展名；SKIP_DIRS 整目录跳过（补充 4 提示对应）。 + +纯函数：输入 bytes/路径/Settings，输出 ExtractResult 或抛 ZipSafetyError，无 FastAPI/asyncio 依赖。 +""" +from __future__ import annotations + +import io +import re +import stat +import zipfile +from dataclasses import dataclass, field +from pathlib import Path, PurePosixPath +from typing import Literal + +# 危险类型（deny → 整包拒绝） +DENY_EXT = { + ".exe", ".com", ".bat", ".cmd", ".sh", ".bash", ".ps1", ".vbs", + ".dll", ".so", ".dylib", ".msi", ".jar", ".py", ".rb", ".pl", ".php", ".js", +} +# LaTeX 项目白名单（allow → 解压） +ALLOW_EXT = { + ".tex", ".cls", ".sty", ".bst", ".bib", ".bbl", ".blg", ".def", ".clo", + ".fd", ".cfg", ".ist", ".dtx", ".ins", + ".png", ".jpg", ".jpeg", ".pdf", ".eps", ".svg", + ".txt", ".md", ".csv", ".dat", ".ttf", ".otf", +} +# 无扩展名白名单 +ALLOW_BARE = {"README", "LICENSE", "Makefile", ".gitignore"} +# 整目录跳过（不进 skipped 计数之外的拒绝路径，直接 skip） +SKIP_DIRS = {".git", "node_modules", "__MACOSX"} + + +class ZipSafetyError(Exception): + """zip 不安全/超限/损坏。code ∈ {ZIP_UNSAFE, ZIP_BOMB, ZIP_INVALID}。""" + + def __init__(self, code: str, message: str, entry: str = "") -> None: + super().__init__(message) + self.code = code + self.message = message + self.entry = entry + + +@dataclass +class SkippedEntry: + name: str + reason: str + + +@dataclass +class ExtractResult: + files: int = 0 + tex: int = 0 + images: int = 0 + cls: int = 0 + bib: int = 0 + other: int = 0 + bytes: int = 0 + skipped: list[SkippedEntry] = field(default_factory=list) + skipped_count: int = 0 + + +_IMG_EXT = {".png", ".jpg", ".jpeg", ".eps", ".svg"} +_BARE_RE = re.compile(r"^[A-Za-z]:") # 盘符 +_CTRL_RE = re.compile(r"[\x00-\x1f]") # 控制字符 + + +def sanitize_entry_name(raw: str) -> str: + """归一 + 校验条目名，返回 posix 相对路径；不安全 → ZIP_UNSAFE。 + + 拒绝：绝对路径（/ 或 \\ 开头）、盘符（C:）、任一段 `..`、空段、NUL/控制字符。 + """ + name = raw.replace("\\", "/") + if not name or name in ("/", "."): + raise ZipSafetyError("ZIP_UNSAFE", f"zip 含空条目名：{raw!r}", raw) + if name.startswith("/") or _BARE_RE.match(name): + raise ZipSafetyError("ZIP_UNSAFE", f"zip 含绝对路径条目：{raw}", raw) + if _CTRL_RE.search(name): + raise ZipSafetyError("ZIP_UNSAFE", f"zip 条目名含控制字符：{raw!r}", raw) + segs = name.split("/") + if any(s == ".." for s in segs): + raise ZipSafetyError("ZIP_UNSAFE", f"zip 含路径穿越条目：{raw}", raw) + # 空段（//）拒绝；尾段为 ""（目录 / 结尾）在除末位外的段上已由 split 覆盖 + if any(s == "" for s in segs[:-1]) or (segs and segs[0] == ""): + raise ZipSafetyError("ZIP_UNSAFE", f"zip 含非法路径条目：{raw}", raw) + depth = len([s for s in segs if s]) + if depth > 16: + raise ZipSafetyError("ZIP_UNSAFE", f"zip 条目路径过深（{depth} 层）：{raw}", raw) + if len(name) > 200: + raise ZipSafetyError("ZIP_UNSAFE", f"zip 条目路径过长（{len(name)} 字符）：{raw}", raw) + return "/".join(s for s in segs if s) + + +def classify_entry(name: str, is_dir: bool) -> Literal["allow", "skip", "deny"]: + """条目分类：目录 → allow（只建）；DENY_EXT → deny（整包拒绝）；白名单 → allow；其余 → skip。""" + if is_dir: + return "allow" + segs = name.replace("\\", "/").split("/") + # 中间目录段命中 SKIP_DIRS → skip（.git/node_modules 整树跳过） + if any(seg in SKIP_DIRS for seg in segs[:-1]): + return "skip" + p = PurePosixPath(name) + ext = p.suffix.lower() + # 多重扩展如 .synctex.gz：取最后一段即可（gz 不在白名单 → skip） + if ext in DENY_EXT: + return "deny" + if ext in ALLOW_EXT: + return "allow" + if p.name.upper() in {b.upper() for b in ALLOW_BARE}: + return "allow" + return "skip" + + +def _is_symlink(info: zipfile.ZipInfo) -> bool: + """external_attr 高 16 位为 Unix mode；S_IFMT == S_IFLNK(0xA000) → 符号链接。""" + mode = (info.external_attr >> 16) & 0xFFFF + return stat.S_IFMT(mode) == stat.S_IFLNK + + +def _has_exec_bit(info: zipfile.ZipInfo) -> bool: + mode = (info.external_attr >> 16) & 0xFFFF + # Windows 造的 zip（external_attr 无 Unix mode）mode 可能为 0 → 无 x 位 + return bool(mode & 0o111) + + +def _prescan(zf: zipfile.ZipFile, settings) -> list[tuple[zipfile.ZipInfo, str]]: + """预扫中央目录：条目数 → 路径 → 符号链接/可执行位 → 分类 → 声明大小/压缩比。 + + 全部通过才允许写盘（两段式第一段）。 + """ + infos = zf.infolist() + if len(infos) > settings.max_zip_entries: + raise ZipSafetyError( + "ZIP_BOMB", f"zip 条目数超限（{len(infos)} > {settings.max_zip_entries}）") + + plan: list[tuple[zipfile.ZipInfo, str]] = [] + declared_total = 0 + for info in infos: + is_dir = info.is_dir() + name = sanitize_entry_name(info.filename) + if is_dir: + continue + # #3 符号链接 → 整包拒绝 + if _is_symlink(info): + raise ZipSafetyError("ZIP_UNSAFE", f"zip 含符号链接条目：{info.filename}", info.filename) + # #4/#6 分类 + kind = classify_entry(name, is_dir=False) + if kind == "deny": + raise ZipSafetyError("ZIP_UNSAFE", f"zip 含危险条目：{info.filename}", info.filename) + if kind == "skip": + continue + # #5 可执行位 → 整包拒绝（仅普通文件；目录已 continue） + if _has_exec_bit(info): + raise ZipSafetyError( + "ZIP_UNSAFE", f"zip 含可执行权限条目：{info.filename}", info.filename) + # #9 单条目声明大小 + if info.file_size > settings.max_zip_entry_bytes: + raise ZipSafetyError( + "ZIP_BOMB", + f"zip 条目过大：{info.filename}（{info.file_size} 字节）", info.filename) + # #7 压缩比（compress_size=0 且 file_size>0 视为异常） + if info.file_size > 0: + cs = info.compress_size + if cs == 0 or info.file_size / cs > settings.max_zip_ratio: + raise ZipSafetyError( + "ZIP_BOMB", + f"zip 条目压缩比异常（疑似 zip bomb）：{info.filename}", info.filename) + declared_total += info.file_size + # #8 声明累计 + if declared_total > settings.max_extract_bytes: + raise ZipSafetyError( + "ZIP_BOMB", + f"zip 解压总大小超限（声明 {declared_total} > " + f"{settings.max_extract_bytes} 字节）") + plan.append((info, name)) + return plan + + +def safe_extract(zip_bytes: bytes, dest: Path, settings) -> ExtractResult: + """安全解压 zip_bytes → dest（新建目录）。 + + 两段式：①预扫（结构性检查全过）②流式写盘（8KB 块 + 实字节计数）。 + 任何 ZipSafetyError → 调用方负责 rmtree(dest) 清残留。 + """ + try: + zf = zipfile.ZipFile(io.BytesIO(zip_bytes)) + except zipfile.BadZipFile as exc: + raise ZipSafetyError("ZIP_INVALID", f"zip 文件损坏或不是有效 zip：{exc}") from exc + + result = ExtractResult() + with zf: + # ---- 第一段：纯预扫 ---- + plan = _prescan(zf, settings) + + # ---- 第二段：流式写盘 ---- + dest.mkdir(parents=True, exist_ok=True) + written_total = 0 + for info, rel in plan: + target = dest / rel + # #1 双保险：resolve 后必须仍在 dest 内 + resolved_parent = target.parent.resolve() + if not str(resolved_parent).startswith(str(dest.resolve())): + raise ZipSafetyError( + "ZIP_UNSAFE", f"zip 条目逃逸目标目录：{info.filename}", info.filename) + target.parent.mkdir(parents=True, exist_ok=True) + try: + src = zf.open(info) + except RuntimeError as exc: # 加密条目 + raise ZipSafetyError("ZIP_INVALID", f"zip 条目受密码保护：{info.filename}") from exc + try: + with src, open(target, "wb") as out: + while True: + chunk = src.read(8192) + if not chunk: + break + # #8/#13 实写字节计数（不信头部声明） + written_total += len(chunk) + if written_total > settings.max_extract_bytes: + raise ZipSafetyError( + "ZIP_BOMB", + f"zip 实际解压超限（> {settings.max_extract_bytes} 字节），已中止") + out.write(chunk) + except zipfile.BadZipFile as exc: # CRC 校验失败（#12/#13） + raise ZipSafetyError( + "ZIP_INVALID", f"zip 条目损坏（CRC 校验失败）：{info.filename}", + info.filename) from exc + # #5 绝不保留可执行位 + target.chmod(0o644) + result.files += 1 + _tally(result, rel) + result.bytes = written_total # 实写累计（#8 计数器终值） + + # 跳过统计（限 20 条 + 总数） + skipped: list[SkippedEntry] = [] + skipped_count = 0 + for info in zf.infolist(): + if info.is_dir(): + continue + try: + name = sanitize_entry_name(info.filename) + except ZipSafetyError: + continue # 预扫已拒绝，不会到这里 + if classify_entry(name, is_dir=False) == "skip": + skipped_count += 1 + if len(skipped) < 20: + skipped.append(SkippedEntry(name=name, reason="unsupported")) + result.skipped = skipped + result.skipped_count = skipped_count + + return result + + +def _tally(result: ExtractResult, rel: str) -> None: + ext = PurePosixPath(rel).suffix.lower() + if ext == ".tex": + result.tex += 1 + elif ext in _IMG_EXT or ext == ".png": + result.images += 1 + elif ext == ".cls": + result.cls += 1 + elif ext == ".bib": + result.bib += 1 + else: + result.other += 1 diff --git a/review_service/tests/test_compile_zip.py b/review_service/tests/test_compile_zip.py new file mode 100644 index 0000000..5faa7ed --- /dev/null +++ b/review_service/tests/test_compile_zip.py @@ -0,0 +1,527 @@ +"""tests/test_compile_zip.py —— P4 项目 zip 两段式编译集成测试。 + +真实调用 tools/tectonic（宏包已预热缓存）；二进制缺失时集成用例 skipif 跳过， +校验/消费/清扫等纯逻辑用例照常运行。 +运行: cd review_service && /bin/python -m pytest -q +""" +from __future__ import annotations + +import io +import json +import os +import time +import zipfile +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from app.config import get_settings +from app.main import app +from app.services import latex_compile as lc + +MINIMAL = "\\documentclass[11pt]{article}\n\\begin{document}\nhello p4\n\\end{document}\n" + +_TECTONIC_OK = get_settings().tectonic_abs.is_file() + +needs_tectonic = pytest.mark.skipif( + not _TECTONIC_OK, reason="tools/tectonic 未安装（bash tools/install_tectonic.sh）" +) + + +def _zip_bytes(entries: dict[str, str | bytes]) -> bytes: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: + for name, data in entries.items(): + zf.writestr(name, data) + return buf.getvalue() + + +def _upload(client: TestClient, payload: bytes, name: str = "project.zip"): + return client.post( + "/api/review/compile/inspect", + files={"file": (name, payload, "application/zip")}, + ) + + +def _tiny_pdf() -> bytes: + """极小合法 PDF（1 页 20x20 蓝色方块），供图片内嵌集成用例。""" + stream = b"0 0 1 rg 2 2 10 10 re f\n" + objs = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 20 20]" + b" /Contents 4 0 R /Resources << >> >>", + b"<< /Length " + str(len(stream)).encode() + b" >>\nstream\n" + + stream + b"endstream", + ] + out = bytearray(b"%PDF-1.4\n") + offsets = [] + for i, body in enumerate(objs, 1): + offsets.append(len(out)) + out += f"{i} 0 obj\n".encode() + body + b"\nendobj\n" + xref_pos = len(out) + out += f"xref\n0 {len(objs) + 1}\n".encode() + out += b"0000000000 65535 f \n" + for off in offsets: + out += f"{off:010d} 00000 n \n".encode() + out += (f"trailer\n<< /Size {len(objs) + 1} /Root 1 0 R >>\n" + f"startxref\n{xref_pos}\n%%EOF\n").encode() + return bytes(out) + + +@pytest.fixture() +def client(tmp_path, monkeypatch): + settings = get_settings() + monkeypatch.setattr(settings, "reformat_dir", str(tmp_path / "jobs")) + monkeypatch.setattr(settings, "inspect_dir", str(tmp_path / "inspect")) + monkeypatch.setattr(settings, "compile_jobs_dir", str(tmp_path / "compile_jobs")) + monkeypatch.setattr(settings, "compile_timeout_s", 60.0) + monkeypatch.setattr(settings, "compile_project_timeout_s", 60.0) + monkeypatch.setattr(lc, "_pending", 0) + with TestClient(app) as c: + yield c + + +# ================================================================ inspect 校验 + +def test_inspect_not_zip(client: TestClient): + r = client.post( + "/api/review/compile/inspect", + files={"file": ("x.zip", b"notazip", "application/zip")}, + ) + assert r.status_code == 200 and r.json()["error"]["code"] == "ZIP_INVALID" + + +def test_inspect_empty(client: TestClient): + r = _upload(client, b"") + assert r.json()["error"]["code"] == "VALIDATION" + + +def test_inspect_oversize(client: TestClient, monkeypatch): + monkeypatch.setattr(get_settings(), "max_zip_bytes", 10) + r = _upload(client, _zip_bytes({"main.tex": MINIMAL})) + assert r.json()["error"]["code"] == "VALIDATION" + assert "上限" in r.json()["error"]["message"] + + +def test_inspect_bad_filename(client: TestClient): + r = _upload(client, _zip_bytes({"main.tex": MINIMAL}), name="../evil.zip") + assert r.json()["error"]["code"] == "VALIDATION" + + +def test_inspect_path_traversal(client: TestClient): + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr("main.tex", MINIMAL) + info = zipfile.ZipInfo("../../evil.txt") + zf.writestr(info, "x") + r = _upload(client, buf.getvalue()) + assert r.json()["error"]["code"] == "ZIP_UNSAFE" + + +def test_inspect_denied_ext(client: TestClient): + r = _upload(client, _zip_bytes({"main.tex": MINIMAL, "evil.sh": "#!/bin/sh\n"})) + assert r.json()["error"]["code"] == "ZIP_UNSAFE" + assert "evil.sh" in r.json()["error"]["message"] + + +def test_inspect_ratio_bomb(client: TestClient, monkeypatch): + monkeypatch.setattr(get_settings(), "max_zip_ratio", 1.5) + r = _upload(client, _zip_bytes({"main.tex": "a" * 100_000})) + assert r.json()["error"]["code"] == "ZIP_BOMB" + + +def test_inspect_no_tex(client: TestClient): + r = _upload(client, _zip_bytes({"readme.txt": "hi"})) + assert r.json()["error"]["code"] == "VALIDATION" + assert "没有 .tex" in r.json()["error"]["message"] + + +def test_inspect_no_documentclass(client: TestClient): + r = _upload(client, _zip_bytes({"chapters/a.tex": "just text\n"})) + assert r.json()["error"]["code"] == "VALIDATION" + assert "documentclass" in r.json()["error"]["message"] + + +def test_inspect_multiple_mains(client: TestClient): + r = _upload(client, _zip_bytes({ + "main.tex": MINIMAL, "appendix.tex": MINIMAL})) + body = r.json() + assert body["error"]["code"] == "VALIDATION" + assert "多个主文件" in body["error"]["message"] + + +# ================================================================ inspect 正常 + 卫生 + +def test_inspect_happy_path(client: TestClient): + payload = _zip_bytes({ + "main.tex": MINIMAL, + "figs/fig1.png": b"\x89PNG\r\n\x1a\nfake", + "refs.bib": "@article{a,\n title={t}\n}\n", + "journal.cls": "% fake cls\n", + "notes.docx": "skip me", + ".DS_Store": "junk", + "__MACOSX/._main.tex": "resource fork junk", + }) + r = _upload(client, payload) + body = r.json() + assert body["ok"] is True, body + meta = body["meta"] + assert meta["main"] == "main.tex" + stats = meta["stats"] + assert stats["tex"] == 1 and stats["images"] == 1 + assert stats["cls"] == 1 and stats["bib"] == 1 + assert meta["skipped_count"] >= 2 + names = {s["name"] for s in meta["skipped"]} + assert any("notes.docx" in n for n in names) + + job_id = meta["job_id"] + assert len(job_id) == 12 + jdir = get_settings().inspect_dir_abs / job_id + assert (jdir / "inspect.json").is_file() + assert (jdir / "project" / "main.tex").is_file() + + +def test_inspect_fail_leaves_no_residue(client: TestClient): + r = _upload(client, _zip_bytes({"main.tex": MINIMAL, "evil.sh": "x"})) + assert r.json()["ok"] is False + base = get_settings().inspect_dir_abs + if base.is_dir(): + assert not any(base.iterdir()), "inspect 失败后磁盘有残留" + + +# ================================================================ run：校验 / 原子消费 + +def test_run_bad_job_id(client: TestClient): + r = client.post("/api/review/compile/run", json={"job_id": "zz"}) + assert r.json()["error"]["code"] == "VALIDATION" + + +def test_run_unknown_job(client: TestClient): + r = client.post("/api/review/compile/run", json={"job_id": "0123456789ab"}) + assert r.json()["error"]["code"] == "VALIDATION" + + +def test_run_double_consume_rejected(client: TestClient): + """补充：原子消费——同一 job 第二次 run 必须 VALIDATION（防并发双跑）。""" + job_id = _upload(client, _zip_bytes({"main.tex": MINIMAL})).json()["meta"]["job_id"] + first = client.post("/api/review/compile/run", json={"job_id": job_id}).json() + first_err = first.get("error") or {} + if first_err.get("code") == "TOOL_UNAVAILABLE": + pytest.skip("tectonic 未安装，消费逻辑未触达") + # 首次无论编译成败（消费已发生），二次必失败 + assert first["ok"] is True or first["error"]["code"] in ( + "COMPILE_FAILED", "TIMEOUT") + second = client.post("/api/review/compile/run", json={"job_id": job_id}).json() + assert second["ok"] is False + assert second["error"]["code"] == "VALIDATION" + # inspect 残壳也不应存在 + assert not (get_settings().inspect_dir_abs / job_id / "project").exists() + + +def test_run_tool_unavailable_keeps_job(client: TestClient, monkeypatch): + """队列/工具检查先于消费 → TOOL_UNAVAILABLE 时 inspect 保留可重试。""" + job_id = _upload(client, _zip_bytes({"main.tex": MINIMAL})).json()["meta"]["job_id"] + with monkeypatch.context() as m: + m.setattr(get_settings(), "tectonic_path", "tools/definitely-missing") + r = client.post("/api/review/compile/run", json={"job_id": job_id}).json() + assert r["error"]["code"] == "TOOL_UNAVAILABLE" + assert (get_settings().inspect_dir_abs / job_id / "inspect.json").is_file() + + +def test_run_queue_full_keeps_job(client: TestClient, monkeypatch): + job_id = _upload(client, _zip_bytes({"main.tex": MINIMAL})).json()["meta"]["job_id"] + monkeypatch.setattr(lc, "_pending", 5) + monkeypatch.setattr(get_settings(), "compile_queue_limit", 5) + r = client.post("/api/review/compile/run", json={"job_id": job_id}) + assert r.status_code == 429 + assert r.json()["error"]["code"] == "QUEUE_FULL" + assert (get_settings().inspect_dir_abs / job_id / "inspect.json").is_file() + + +# ================================================================ run：真实 tectonic + +@needs_tectonic +def test_run_project_happy_path(client: TestClient): + payload = _zip_bytes({"main.tex": MINIMAL}) + job_id = _upload(client, payload).json()["meta"]["job_id"] + r = client.post("/api/review/compile/run", json={"job_id": job_id}) + body = r.json() + assert body["ok"] is True, body + cid = body["compile_id"] + assert len(cid) == 12 + + pdf = client.get(body["pdf_url"]) + assert pdf.status_code == 200 + assert pdf.headers["content-type"].startswith("application/pdf") + assert pdf.content[:5] == b"%PDF-" + + log = client.get(body["log_url"]) + assert log.status_code == 200 + assert "LaTeX2e" in log.text and "Output written" in log.text + + meta = client.get(f"/api/review/compile/zip/{cid}") + m = meta.json() + assert m["ok"] is True + assert m["meta"]["project"] is True + assert m["meta"]["main"] == "main.tex" + assert m["meta"]["pdf_bytes"] > 0 + + # project 树已删，只留产物 + cdir = get_settings().compile_jobs_dir_abs / cid + assert not (cdir / "project").exists() + assert (cdir / "output.pdf").is_file() + + +@needs_tectonic +def test_run_failed_envelope_has_log_url(client: TestClient): + """补充 3：编译失败信封带 compile_id + log_url，且 log 可下载。""" + bad = "\\documentclass{article}\n\\begin{document}\n\\undefinedcommandxyz\n" + job_id = _upload(client, _zip_bytes({"main.tex": bad})).json()["meta"]["job_id"] + body = client.post("/api/review/compile/run", json={"job_id": job_id}).json() + assert body["ok"] is False + assert body["error"]["code"] == "COMPILE_FAILED" + cid = body.get("compile_id") + assert cid and len(cid) == 12 + assert body["log_url"] == f"/api/review/compile/zip/{cid}/log" + log = client.get(body["log_url"]) + assert log.status_code == 200 and log.text.strip() + + +@needs_tectonic +def test_run_failed_keeps_project_scene(client: TestClient): + """失败保留现场（诊断）：COMPILE_FAILED → compile_jobs/{cid}/project/ 不删。""" + bad = "\\documentclass{article}\n\\begin{document}\n\\undefinedcommandxyz\n" + job_id = _upload(client, _zip_bytes({"main.tex": bad})).json()["meta"]["job_id"] + body = client.post("/api/review/compile/run", json={"job_id": job_id}).json() + assert body["ok"] is False and body["error"]["code"] == "COMPILE_FAILED" + cid = body["compile_id"] + cdir = get_settings().compile_jobs_dir_abs / cid + assert (cdir / "project" / "main.tex").is_file() # 现场保留 + assert (cdir / "output.log").is_file() + + +@needs_tectonic +def test_run_missing_cls_friendly_message(client: TestClient): + """classify_compile_error：缺 .cls → 期刊模板提示（Q4/§4.4）。""" + tex = "\\documentclass{noexist-cls}\n\\begin{document}\nx\n" + job_id = _upload(client, _zip_bytes({"main.tex": tex})).json()["meta"]["job_id"] + body = client.post("/api/review/compile/run", json={"job_id": job_id}).json() + assert body["ok"] is False and body["error"]["code"] == "COMPILE_FAILED" + assert "noexist-cls" in body["error"]["message"] + + +@needs_tectonic +def test_run_project_malicious_write18(client: TestClient, tmp_path): + canary = tmp_path / "canary_p4" + evil = ( + "\\documentclass{article}\n" + f"\\write18{{touch {canary}}}\n" + "\\begin{document}\nno shell please\n\\end{document}\n" + ) + job_id = _upload(client, _zip_bytes({"main.tex": evil})).json()["meta"]["job_id"] + client.post("/api/review/compile/run", json={"job_id": job_id}) + assert not canary.exists(), r"\write18 竟然执行了！" + + +@needs_tectonic +def test_run_project_timeout(client: TestClient, monkeypatch): + monkeypatch.setattr(get_settings(), "compile_project_timeout_s", 5.0) + loop_tex = ( + "\\documentclass{article}\n\\begin{document}\n" + "\\count255=0\n\\loop\\iftrue\\advance\\count255 by 1\\repeat\n" + "\\end{document}\n" + ) + job_id = _upload(client, _zip_bytes({"main.tex": loop_tex})).json()["meta"]["job_id"] + started = time.monotonic() + body = client.post("/api/review/compile/run", json={"job_id": job_id}).json() + elapsed = time.monotonic() - started + assert body["ok"] is False and body["error"]["code"] == "TIMEOUT" + assert elapsed < 15.0, f"超时回收过慢：{elapsed:.1f}s" + # 超时也必落 log（补充 3） + log = client.get(body["log_url"]) + assert log.status_code == 200 + assert "[timeout]" in log.text + + +# ================================================================ EPS 回退（xdvipdfmx 不支持内嵌 EPS） + +def test_fallback_eps_converted_pdf(tmp_path): + """存在 *-eps-converted-to.pdf → 引用自动切换（.cls/.tex 都扫，字节替换不坏编码）。""" + proj = tmp_path / "p" + defs = proj / "Definitions" + defs.mkdir(parents=True) + (defs / "logo-mdpi.eps").write_bytes(b"%!PS-Adobe-3.0 EPSF-3.0\n") + (defs / "logo-mdpi-eps-converted-to.pdf").write_bytes(_tiny_pdf()) + (defs / "other.eps").write_bytes(b"%!PS\n") # 无转换版 → 不动 + (defs / "mdpi.cls").write_text( + "\\includegraphics{Definitions/logo-mdpi.eps}\n" + "\\includegraphics{Definitions/other.eps}\n", encoding="utf-8") + (proj / "main.tex").write_text( + "\\documentclass{Definitions/mdpi}\n\\begin{document}\nx\n\\end{document}\n", + encoding="utf-8") + + changed = lc.fallback_eps_to_converted_pdf(proj) + assert changed == ["Definitions/mdpi.cls"] + cls = (defs / "mdpi.cls").read_text(encoding="utf-8") + assert "logo-mdpi-eps-converted-to.pdf" in cls + assert "Definitions/other.eps" in cls # 无转换版的引用保持原样 + assert (defs / "logo-mdpi.eps").is_file() # 源文件不删（诊断现场保留） + # 缺省扩展名解析规避：有转换版 → 复制出 X.pdf（graphicx 先试 .pdf） + assert (defs / "logo-mdpi.pdf").is_file() + assert not (defs / "other.pdf").exists() # 无转换版 → 不复制 + + +def test_fallback_eps_noop_without_converted(tmp_path): + proj = tmp_path / "p" + proj.mkdir() + (proj / "fig.eps").write_bytes(b"%!PS\n") + (proj / "main.tex").write_text("x", encoding="utf-8") + assert lc.fallback_eps_to_converted_pdf(proj) == [] + assert (proj / "main.tex").read_text(encoding="utf-8") == "x" + + +@needs_tectonic +def test_run_eps_with_converted_pdf_succeeds(client: TestClient): + """EPS + 预转换 PDF → 显式 .eps 引用与缺省扩展名引用都编译成功（MDPI logo 场景）。""" + tex = ( + "\\documentclass{article}\n\\usepackage{graphicx}\n" + "\\begin{document}\n" + "\\includegraphics[width=1cm]{fig.eps}\n" # 显式 .eps → 文本替换 + "\\includegraphics[width=1cm]{fig}\n" # 缺省扩展名 → 复制 X.pdf 规避 .eps + "\\end{document}\n" + ) + payload = _zip_bytes({ + "main.tex": tex, + "fig.eps": b"%!PS-Adobe-3.0 EPSF-3.0\n%%BoundingBox: 0 0 10 10\n", + "fig-eps-converted-to.pdf": _tiny_pdf(), + }) + job_id = _upload(client, payload).json()["meta"]["job_id"] + body = client.post("/api/review/compile/run", json={"job_id": job_id}).json() + assert body["ok"] is True, body + pdf = client.get(body["pdf_url"]) + assert pdf.content[:5] == b"%PDF-" + + +@needs_tectonic +def test_run_eps_without_converted_friendly_message(client: TestClient): + """只有 EPS 无转换版 → xdvipdfmx 内嵌失败 → 友好提示（指向预转换 PDF）。""" + tex = ( + "\\documentclass{article}\n\\usepackage{graphicx}\n" + "\\begin{document}\n\\includegraphics[width=1cm]{fig.eps}\n\\end{document}\n" + ) + payload = _zip_bytes({ + "main.tex": tex, + "fig.eps": b"%!PS-Adobe-3.0 EPSF-3.0\n%%BoundingBox: 0 0 10 10\n", + }) + job_id = _upload(client, payload).json()["meta"]["job_id"] + body = client.post("/api/review/compile/run", json={"job_id": job_id}).json() + assert body["ok"] is False and body["error"]["code"] == "COMPILE_FAILED" + assert "无法内嵌" in body["error"]["message"] + assert "fig-eps-converted-to.pdf" in body["error"]["message"] + + +# ================================================================ zip 只读端点 404 + +def test_zip_meta_404(client: TestClient): + r = client.get("/api/review/compile/zip/0123456789ab") + assert r.json()["error"]["code"] == "NOT_FOUND" + + +def test_zip_pdf_404(client: TestClient): + r = client.get("/api/review/compile/zip/0123456789ab/pdf") + assert r.json()["error"]["code"] == "NOT_FOUND" + + +def test_zip_log_404(client: TestClient): + r = client.get("/api/review/compile/zip/0123456789ab/log") + assert r.json()["error"]["code"] == "NOT_FOUND" + + +# ================================================================ 清扫 + +def test_sweep_inspect_expired(tmp_path, monkeypatch): + settings = get_settings() + monkeypatch.setattr(settings, "inspect_dir", str(tmp_path / "inspect")) + base = tmp_path / "inspect" + stale = base / "aabbccddeeff" + stale.mkdir(parents=True) + (stale / "inspect.json").write_text("{}", encoding="utf-8") + (stale / "project").mkdir() + old = time.time() - 2 * 3600 + os.utime(stale, (old, old)) + + fresh = base / "112233445566" + fresh.mkdir() + noise = base / "not-a-job" + noise.mkdir() + + removed = lc.sweep_inspect_expired() + assert removed == 1 + assert not stale.exists() + assert fresh.exists() + assert noise.exists() + + +def test_sweep_compile_jobs_expired(tmp_path, monkeypatch): + settings = get_settings() + monkeypatch.setattr(settings, "compile_jobs_dir", str(tmp_path / "cj")) + base = tmp_path / "cj" + stale = base / "aabbccddeeff" + stale.mkdir(parents=True) + (stale / "output.pdf").write_bytes(b"%PDF-fake") + old = time.time() - 8 * 86400 + os.utime(stale, (old, old)) + + fresh = base / "112233445566" + fresh.mkdir() + (fresh / "output.pdf").write_bytes(b"%PDF-fake") + noise = base / "not-a-job" + noise.mkdir() + + removed = lc.sweep_compile_jobs_expired() + assert removed == 1 + assert not stale.exists() + assert fresh.exists() + assert noise.exists() + + +def test_sweep_failed_project_ttl(tmp_path, monkeypatch): + """两级清扫①：失败现场 project/ 超 failed_project_ttl_s → 只删子目录，log 保留。""" + settings = get_settings() + monkeypatch.setattr(settings, "compile_jobs_dir", str(tmp_path / "cj")) + monkeypatch.delenv("KEEP_FAILED_PROJECTS", raising=False) + base = tmp_path / "cj" + stale = base / "aabbccddeeff" + (stale / "project").mkdir(parents=True) + (stale / "output.log").write_text("log", encoding="utf-8") + old = time.time() - settings.failed_project_ttl_s - 3600 + os.utime(stale / "project", (old, old)) + os.utime(stale, (old, old)) # 整目录也旧，但未超 7 天 → ② 不触发 + + fresh = base / "112233445566" + (fresh / "project").mkdir(parents=True) + (fresh / "output.log").write_text("log", encoding="utf-8") + + removed = lc.sweep_compile_jobs_expired() + assert removed == 1 + assert not (stale / "project").exists() # 现场删 + assert (stale / "output.log").is_file() # log 留（完整 log 下载） + assert (fresh / "project").exists() + + +def test_sweep_failed_project_kept_when_flag(tmp_path, monkeypatch): + """KEEP_FAILED_PROJECTS=1：失败现场永久保留（跳过①）。""" + settings = get_settings() + monkeypatch.setattr(settings, "compile_jobs_dir", str(tmp_path / "cj")) + monkeypatch.setenv("KEEP_FAILED_PROJECTS", "1") + base = tmp_path / "cj" + stale = base / "aabbccddeeff" + (stale / "project").mkdir(parents=True) + old = time.time() - settings.failed_project_ttl_s - 3600 + os.utime(stale / "project", (old, old)) + + removed = lc.sweep_compile_jobs_expired() + assert removed == 0 + assert (stale / "project").exists() diff --git a/review_service/tests/test_zip_safety.py b/review_service/tests/test_zip_safety.py new file mode 100644 index 0000000..25bb186 --- /dev/null +++ b/review_service/tests/test_zip_safety.py @@ -0,0 +1,195 @@ +"""tests/test_zip_safety.py —— P4 zip 安全模块纯逻辑测试（无需 tectonic）。 + +覆盖设计 §2 安全清单 #1-#13（#14 tectonic --untrusted 归 test_compile.py， +#15 由调用方 to_thread 承担，模块本身只测上限封顶行为）。 + +运行: cd review_service && /bin/python -m pytest -q tests/test_zip_safety.py +""" +from __future__ import annotations + +import io +import stat +import zipfile +from pathlib import Path + +import pytest + +from app.config import get_settings +from app.services.zip_safety import ( + ExtractResult, + ZipSafetyError, + classify_entry, + safe_extract, + sanitize_entry_name, +) + + +def _settings(**over): + """取真实 settings 并覆盖上限（dataclass 风格 monkeypatch）。""" + s = get_settings() + for k, v in over.items(): + setattr(s, k, v) + return s + + +def _make_zip(entries: dict[str, bytes], exec_bits: dict[str, int] | None = None) -> bytes: + """entries: name → content；exec_bits: name → unix mode（如 0o100755）。 + + 必须显式 ZIP_DEFLATED：手动构造的 ZipInfo 默认 ZIP_STORED（ratio=1，测不出压缩比炸弹）。 + """ + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: + for name, data in entries.items(): + info = zipfile.ZipInfo(name) + info.compress_type = zipfile.ZIP_DEFLATED + if exec_bits and name in exec_bits: + info.external_attr = exec_bits[name] << 16 + zf.writestr(info, data) + return buf.getvalue() + + +def _symlink_zip(name: str = "link") -> bytes: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + info = zipfile.ZipInfo(name) + info.external_attr = (stat.S_IFLNK | 0o777) << 16 # 0xA1FF + zf.writestr(info, "../../etc/passwd") + return buf.getvalue() + + +# ================================================================ sanitize（#1/#2/#11） + +class TestSanitize: + def test_normal_path_ok(self): + assert sanitize_entry_name("main.tex") == "main.tex" + assert sanitize_entry_name("figs/a.png") == "figs/a.png" + assert sanitize_entry_name("a/b/c/") == "a/b/c" # 目录尾斜杠归一 + + def test_path_traversal_rejected(self): # #1 + with pytest.raises(ZipSafetyError) as e: + sanitize_entry_name("../../etc/passwd") + assert e.value.code == "ZIP_UNSAFE" + + def test_backslash_traversal_rejected(self): # #2 Windows 变体 + with pytest.raises(ZipSafetyError) as e: + sanitize_entry_name("..\\..\\evil.sh") + assert e.value.code == "ZIP_UNSAFE" + + def test_absolute_path_rejected(self): + with pytest.raises(ZipSafetyError): + sanitize_entry_name("/etc/passwd") + with pytest.raises(ZipSafetyError): + sanitize_entry_name("C:\\Windows\\evil") + + def test_control_char_rejected(self): + with pytest.raises(ZipSafetyError): + sanitize_entry_name("ma\x00in.tex") + + def test_depth_limit(self): # #11 + deep = "/".join(f"d{i}" for i in range(20)) + "/x.tex" + with pytest.raises(ZipSafetyError) as e: + sanitize_entry_name(deep) + assert e.value.code == "ZIP_UNSAFE" + + def test_length_limit(self): + with pytest.raises(ZipSafetyError): + sanitize_entry_name("a" * 250 + ".tex") + + +# ================================================================ classify（#4/#6） + +class TestClassify: + def test_allow(self): + for n in ("main.tex", "journal.cls", "refs.bib", "figs/a.png", "x.sty"): + assert classify_entry(n, False) == "allow" + assert classify_entry("figs/", True) == "allow" + + def test_deny_dangerous(self): # #4 + for n in ("run.sh", "evil.exe", "x.so", "x.dll", "a.py", "b.js"): + assert classify_entry(n, False) == "deny" + + def test_skip_harmless_non_whitelist(self): # #6 + for n in ("notes.docx", "old.log", "archive.zip", "img.gif"): + assert classify_entry(n, False) == "skip" + + def test_skip_dirs(self): # #6 附属 + assert classify_entry("node_modules/x.js", False) == "skip" + assert classify_entry(".git/config", False) == "skip" + assert classify_entry("__MACOSX/._a.tex", False) == "skip" + + +# ================================================================ safe_extract（#3/#5/#7-#10/#12/#13） + +class TestSafeExtract: + def test_happy_path(self, tmp_path: Path): + zb = _make_zip({"main.tex": b"hi", "figs/a.png": b"\x89PNG"}) + r = safe_extract(zb, tmp_path / "p", _settings()) + assert isinstance(r, ExtractResult) + assert r.files == 2 and r.tex == 1 and r.images == 1 + assert (tmp_path / "p" / "main.tex").read_text() == "hi" + assert (tmp_path / "p" / "figs" / "a.png").exists() + + def test_symlink_rejected(self, tmp_path: Path): # #3 + with pytest.raises(ZipSafetyError) as e: + safe_extract(_symlink_zip(), tmp_path / "p", _settings()) + assert e.value.code == "ZIP_UNSAFE" + assert not (tmp_path / "p").exists() # 预扫拒绝 → 未写盘 + + def test_deny_ext_rejects_whole_zip(self, tmp_path: Path): # #4 + zb = _make_zip({"main.tex": b"x", "run.sh": b"#!/bin/sh"}) + with pytest.raises(ZipSafetyError) as e: + safe_extract(zb, tmp_path / "p", _settings()) + assert e.value.code == "ZIP_UNSAFE" and "run.sh" in e.value.message + assert not (tmp_path / "p").exists() + + def test_exec_bit_rejects(self, tmp_path: Path): # #5 + zb = _make_zip({"main.tex": b"x"}, exec_bits={"main.tex": 0o100755}) + with pytest.raises(ZipSafetyError) as e: + safe_extract(zb, tmp_path / "p", _settings()) + assert e.value.code == "ZIP_UNSAFE" + + def test_harmless_skipped_not_rejected(self, tmp_path: Path): # #6 决策1 + zb = _make_zip({"main.tex": b"x", "notes.docx": b"DOC", "old.zip": b"PK"}) + r = safe_extract(zb, tmp_path / "p", _settings()) + assert r.files == 1 + assert r.skipped_count == 2 + assert {s.name for s in r.skipped} == {"notes.docx", "old.zip"} + assert not (tmp_path / "p" / "notes.docx").exists() + + def test_ratio_bomb_rejected(self, tmp_path: Path): # #7 + zb = _make_zip({"big.txt": b"\x00" * (2 * 1024 * 1024)}) # 全零 → ratio ≫100 + with pytest.raises(ZipSafetyError) as e: + safe_extract(zb, tmp_path / "p", _settings()) + assert e.value.code == "ZIP_BOMB" + + def test_total_bytes_rejected_runtime(self, tmp_path: Path): # #8 声明上限 + zb = _make_zip({"a.txt": b"x" * 5000, "b.txt": b"y" * 5000}) + with pytest.raises(ZipSafetyError) as e: + safe_extract(zb, tmp_path / "p", _settings(max_extract_bytes=6000)) + assert e.value.code == "ZIP_BOMB" + + def test_entry_bytes_rejected(self, tmp_path: Path): # #9 + zb = _make_zip({"a.txt": b"x" * 10000}) + with pytest.raises(ZipSafetyError) as e: + safe_extract(zb, tmp_path / "p", _settings(max_zip_entry_bytes=1000)) + assert e.value.code == "ZIP_BOMB" + + def test_entry_count_rejected(self, tmp_path: Path): # #10 + zb = _make_zip({f"f{i}.txt": b"x" for i in range(30)}) + with pytest.raises(ZipSafetyError) as e: + safe_extract(zb, tmp_path / "p", _settings(max_zip_entries=10)) + assert e.value.code == "ZIP_BOMB" + assert not (tmp_path / "p").exists() # 预扫先于写盘 + + def test_corrupt_zip(self, tmp_path: Path): + with pytest.raises(ZipSafetyError) as e: + safe_extract(b"not a zip at all", tmp_path / "p", _settings()) + assert e.value.code == "ZIP_INVALID" + + def test_traversal_entry_rejected_end_to_end(self, tmp_path: Path): # #1 端到端 + zb = _make_zip({"../evil.txt": b"pwned"}) + with pytest.raises(ZipSafetyError) as e: + safe_extract(zb, tmp_path / "p", _settings()) + assert e.value.code == "ZIP_UNSAFE" + assert not (tmp_path / "evil.txt").exists() # 磁盘无残留 + assert not (tmp_path / "p").exists() diff --git a/tests/frontend/unit/edu/reformat.zip.test.js b/tests/frontend/unit/edu/reformat.zip.test.js new file mode 100644 index 0000000..5eeb377 --- /dev/null +++ b/tests/frontend/unit/edu/reformat.zip.test.js @@ -0,0 +1,188 @@ +// tests/frontend/unit/edu/reformat.zip.test.js —— ReformatPanel 项目 zip 两段式（P4） +// 1) 默认 .tex 模式；切"项目 .zip"→ zip 上传区 + 打包建议文案 + inspect 按钮 +// 2) 选 .zip → inspect 成功 → 摘要（main/统计/跳过）+ 编译按钮 +// 3) inspect 失败（ZIP_UNSAFE 后端原话）→ 错误展示 +// 4) 点编译 → loading → 成功：iframe 预览 + 耗时 +// 5) 编译失败：错误摘要 + log 折叠区 + 查看完整 log（失败信封带 log_url） +// 6) 模式互斥：切回 .tex 清 zip 状态 +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { mount } from '@vue/test-utils'; +import { createPinia, setActivePinia } from 'pinia'; +import ReformatPanel from '../../../../ui/src/components/edu/panels/ReformatPanel.vue'; + +const inspectZipMock = vi.fn(); +const runZipCompileMock = vi.fn(); +vi.mock('../../../../ui/src/api/review.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + getJournals: vi.fn().mockResolvedValue({ ok: true, items: [] }), + getDirectory: vi.fn(), + reformatTex: vi.fn(), + compileTex: vi.fn(), + inspectZip: (...args) => inspectZipMock(...args), + runZipCompile: (...args) => runZipCompileMock(...args), + }; +}); + +import { useEduStore } from '../../../../ui/src/stores/edu.js'; +import { ApiError } from '../../../../ui/src/api/http.js'; + +const INSPECT_OK = { + ok: true, + meta: { + job_id: 'abc123def456', + main: 'main.tex', + stats: { files: 5, tex: 1, images: 1, cls: 1, bib: 1, other: 1, bytes: 4096 }, + skipped: [{ name: '__MACOSX/._main.tex', reason: 'skip_dir' }], + skipped_count: 1, + created_at: '2026-10-13T00:00:00+00:00', + }, +}; + +function zipFile(name = 'project.zip') { + return new File([new Uint8Array([0x50, 0x4b, 0x03, 0x04])], name, { + type: 'application/zip', + }); +} + +async function mountZipPanel(edu) { + const pinia = createPinia(); + setActivePinia(pinia); + edu = useEduStore(); + const wrapper = mount(ReformatPanel, { global: { plugins: [pinia] } }); + await wrapper.find('[data-testid="rf-mode-zip"]').trigger('click'); + return { wrapper, edu }; +} + +async function pickZip(wrapper, edu, file = zipFile()) { + const input = wrapper.find('input[type="file"]'); + Object.defineProperty(input.element, 'files', { value: [file], configurable: true }); + await input.trigger('change'); + await wrapper.vm.$nextTick(); +} + +describe('ReformatPanel 项目 zip 两段式（P4）', () => { + beforeEach(() => { + inspectZipMock.mockReset(); + runZipCompileMock.mockReset(); + if (!URL.createObjectURL) URL.createObjectURL = vi.fn(() => 'blob:mock'); + if (!URL.revokeObjectURL) URL.revokeObjectURL = vi.fn(); + global.fetch = vi.fn().mockRejectedValue(new Error('no network')); + }); + + it('1) 默认 .tex 模式；切 zip → zip 上传区 + 打包建议 + inspect 按钮', async () => { + const pinia = createPinia(); + setActivePinia(pinia); + const wrapper = mount(ReformatPanel, { global: { plugins: [pinia] } }); + expect(wrapper.find('[data-testid="rf-mode-tex"]').classes()).toContain('active'); + expect(wrapper.find('[data-testid="rf-zip-inspect-btn"]').exists()).toBe(false); + // tex 提交按钮仍在 + expect(wrapper.findAll('button').some((b) => b.text() === '开始自动重排版')).toBe(true); + + await wrapper.find('[data-testid="rf-mode-zip"]').trigger('click'); + const zipBtn = wrapper.find('[data-testid="rf-zip-inspect-btn"]'); + expect(zipBtn.exists()).toBe(true); + expect(zipBtn.attributes('disabled')).toBeDefined(); // 未选文件不可点 + expect(wrapper.text()).toContain('打包建议：main.tex 放 zip 根目录'); + // tex 提交按钮消失（互斥） + expect(wrapper.findAll('button').some((b) => b.text() === '开始自动重排版')).toBe(false); + }); + + it('2) 选 .zip → inspect 成功 → 摘要 + 编译按钮', async () => { + inspectZipMock.mockResolvedValue(INSPECT_OK); + const { wrapper, edu } = await mountZipPanel(null); + await pickZip(wrapper, edu); + await wrapper.find('[data-testid="rf-zip-inspect-btn"]').trigger('click'); + await vi.waitFor(() => { + expect(wrapper.find('[data-testid="rf-zip-meta"]').exists()).toBe(true); + }); + expect(inspectZipMock).toHaveBeenCalledTimes(1); + const meta = wrapper.find('[data-testid="rf-zip-meta"]'); + expect(meta.text()).toContain('main.tex'); + expect(meta.text()).toContain('文件 5 个'); + expect(meta.text()).toContain('1 个文件已跳过'); + expect(wrapper.find('[data-testid="rf-zip-skipped"]').text()).toContain('__MACOSX'); + expect(wrapper.find('[data-testid="rf-zip-compile-btn"]').exists()).toBe(true); + }); + + it('3) inspect 失败（ZIP_UNSAFE）→ 错误展示', async () => { + inspectZipMock.mockRejectedValue( + new ApiError('zip 含危险条目：evil.sh', { code: 'ZIP_UNSAFE', status: 200 }), + ); + const { wrapper, edu } = await mountZipPanel(null); + await pickZip(wrapper, edu); + await wrapper.find('[data-testid="rf-zip-inspect-btn"]').trigger('click'); + await vi.waitFor(() => { + expect(wrapper.find('[data-testid="rf-zip-error"]').exists()).toBe(true); + }); + expect(wrapper.find('[data-testid="rf-zip-error"]').text()).toContain('evil.sh'); + expect(wrapper.find('[data-testid="rf-zip-meta"]').exists()).toBe(false); + }); + + it('4) 编译 → loading → 成功：iframe 预览 + 耗时', async () => { + inspectZipMock.mockResolvedValue(INSPECT_OK); + let resolveRun; + runZipCompileMock.mockReturnValue(new Promise((r) => { resolveRun = r; })); + const { wrapper, edu } = await mountZipPanel(null); + await pickZip(wrapper, edu); + await wrapper.find('[data-testid="rf-zip-inspect-btn"]').trigger('click'); + await vi.waitFor(() => { + expect(wrapper.find('[data-testid="rf-zip-compile-btn"]').exists()).toBe(true); + }); + await wrapper.find('[data-testid="rf-zip-compile-btn"]').trigger('click'); + await wrapper.vm.$nextTick(); + expect(wrapper.find('[data-testid="rf-compile-loading"]').exists()).toBe(true); + + resolveRun({ + ok: true, + compile_id: 'aabbccddeeff', + pdf_url: '/api/review/compile/zip/aabbccddeeff/pdf', + log_url: '/api/review/compile/zip/aabbccddeeff/log', + meta: { seconds: 3.2, pdf_bytes: 2048, project: true }, + }); + await vi.waitFor(() => { + expect(wrapper.find('[data-testid="rf-pdf-frame"]').exists()).toBe(true); + }); + expect(wrapper.find('[data-testid="rf-compile-secs"]').text()).toContain('3.2 秒'); + expect(wrapper.find('[data-testid="rf-download-pdf"]').exists()).toBe(true); + }); + + it('5) 编译失败：错误摘要 + log 折叠区 + 查看完整 log', async () => { + inspectZipMock.mockResolvedValue(INSPECT_OK); + const compileErr = new ApiError('LaTeX 源码有语法错误', { + code: 'COMPILE_FAILED', status: 200, + }); + compileErr.detail = 'l.12 \\badcmd'; // unwrap 从 error.detail 挂载（log 尾） + runZipCompileMock.mockRejectedValue(compileErr); + const { wrapper, edu } = await mountZipPanel(null); + await pickZip(wrapper, edu); + await wrapper.find('[data-testid="rf-zip-inspect-btn"]').trigger('click'); + await vi.waitFor(() => { + expect(wrapper.find('[data-testid="rf-zip-compile-btn"]').exists()).toBe(true); + }); + await wrapper.find('[data-testid="rf-zip-compile-btn"]').trigger('click'); + await vi.waitFor(() => { + expect(wrapper.find('[data-testid="rf-compile-error"]').exists()).toBe(true); + }); + expect(wrapper.find('[data-testid="rf-compile-error"]').text()).toContain('语法错误'); + expect(wrapper.find('[data-testid="rf-log-body"]').text()).toContain('l.12'); + expect(wrapper.find('[data-testid="rf-download-log"]').exists()).toBe(true); + }); + + it('6) 模式互斥：切回 .tex 清 zip 状态（inspect 摘要消失）', async () => { + inspectZipMock.mockResolvedValue(INSPECT_OK); + const { wrapper, edu } = await mountZipPanel(null); + await pickZip(wrapper, edu); + await wrapper.find('[data-testid="rf-zip-inspect-btn"]').trigger('click'); + await vi.waitFor(() => { + expect(wrapper.find('[data-testid="rf-zip-meta"]').exists()).toBe(true); + }); + await wrapper.find('[data-testid="rf-mode-tex"]').trigger('click'); + expect(wrapper.find('[data-testid="rf-zip-meta"]').exists()).toBe(false); + expect(edu.zipInspect).toBeNull(); + expect(edu.zipFile).toBeNull(); + // tex 上传区回来了 + expect(wrapper.findAll('button').some((b) => b.text() === '开始自动重排版')).toBe(true); + }); +}); diff --git a/ui/src/api/review.js b/ui/src/api/review.js index 0a6c67b..d0cdd5d 100644 --- a/ui/src/api/review.js +++ b/ui/src/api/review.js @@ -19,6 +19,9 @@ function unwrap(promise) { degraded: !!payload.error.degraded, }); if (payload.error.detail) err.detail = payload.error.detail; // 编译失败的 log 尾等 + // P4：项目编译失败信封带 compile_id + log_url（补充 3，前端"查看完整 log"用） + if (payload.compile_id) err.compileId = payload.compile_id; + if (payload.log_url) err.logUrl = payload.log_url; throw err; } return payload; @@ -104,6 +107,23 @@ export function compileTex(formData, opts = {}) { return unwrap(post(`${BASE}/compile`, formData, opts)); } +/** P4 POST /api/review/compile/inspect（multipart .zip ≤50MB，两段式之①） + * → {ok, meta:{job_id, main, stats:{files,tex,images,cls,bib,other,bytes}, + * skipped:[{name,reason}], skipped_count, created_at}} + * 错误信封：VALIDATION / ZIP_UNSAFE / ZIP_BOMB / ZIP_INVALID（HTTP 200）。 + * job_id 1 小时 TTL；run 前不占编译队列。 */ +export function inspectZip(formData, opts = {}) { + return unwrap(post(`${BASE}/compile/inspect`, formData, opts)); +} + +/** P4 POST /api/review/compile/run（JSON {job_id}，两段式之②） + * → 与 compileTex 同构信封 {ok, compile_id, pdf_url, log_url, meta}； + * pdf/log 路径带 /zip/ 前缀。失败（COMPILE_FAILED/TIMEOUT）也带 compile_id+log_url。 + * QUEUE_FULL → HTTP 429（此时 job 未消费，可直接重试）。 */ +export function runZipCompile(payload, opts = {}) { + return unwrap(post(`${BASE}/compile/run`, payload, opts)); +} + /** ⑤ format-check 专用错误文案：解档/校验类 PARSE_FAILED/VALIDATION 用后端原话 * （与 presubmit 的"扫描件"提示是不同语境，不能复用 CODE_HINTS），其余走统一口径。 */ export function formatErrorMessage(err) { diff --git a/ui/src/components/edu/panels/ReformatPanel.vue b/ui/src/components/edu/panels/ReformatPanel.vue index e3d0d2c..2c92013 100644 --- a/ui/src/components/edu/panels/ReformatPanel.vue +++ b/ui/src/components/edu/panels/ReformatPanel.vue @@ -13,6 +13,9 @@ 首次拉 bundle 1-3 分钟——store.compileFirstHint 黄条提示可切面板后台跑）； 成功→blob URL 内嵌 iframe 预览 + 下载 + 耗时/重编；失败→错误摘要 + log 尾 200 行折叠 + 完整 log 下载；QUEUE_FULL→固定文案。 + P4 项目 zip（上传区模式切换，与单文件互斥）：POST /compile/inspect（≤50MB， + 1h TTL，run 前不占编译队列）→ 主文件/统计/跳过文件摘要 → POST /compile/run + （原子消费 inspect；信封与 P3b 同构复用 compile* 字段；失败也带 log_url）。 结果跨菜单保留（store reformatResult/compile*）；样式 rf- 前缀，零新增依赖。 -->