adminandClaude Fable 5
79b4c7c52c
- cli/run_extension.py:容器通用入口(宿主挂载 /opt/bioflow-entry,镜像零改动);
extension.json → 模板渲染(白名单 12 占位符 + param_*,{output_dir} 必含)→
subprocess 执行(无 shell,进程组 SIGTERM→3s→SIGKILL)→ 输出契约校验(空目录退出码 3);
零第三方依赖(镜像内标准库)
- extension_exec.py:服务端 docker 执行器(list-form;三数据卷+入口卷;
--cpus/--memory 白名单独立复制不 import kernel.py;terminate→kill 同约定;
返回 {exit_code,timed_out,log_tail,cmd};run_stream 逐行增量)
- extension_image.py:指纹 sha256(base+tag+排序依赖) 前 12 位 →
bioflow-ext-<fp>:dev;inspect 命中复用;Dockerfile FROM base + apt + /opt/conda/bin/pip;
失败分类(pip/apt/network/timeout/docker_missing)+ LABEL 定位中间层清理;
registry 台账 + 引用计数 + collect_garbage
- extension_streams.py:契约 §4 事件流(trial/run async 生成器,增量日志节流,
EventBus 双发,防御性失败帧,aclose 回收)
- 单测 53 用例(15 unittest + 38 pytest)全绿;现有 bioflow 226 用例回归通过
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
292 lines
12 KiB
Python
292 lines
12 KiB
Python
"""tests/test_extension_image.py —— 自动镜像生成(指纹去重/命令形态/失败清理/回收)。"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from app.services.bioflow.extension_image import (
|
|
ExtensionImageBuilder,
|
|
ImageBuildError,
|
|
ImageConfig,
|
|
image_fingerprint,
|
|
image_name_for,
|
|
render_dockerfile,
|
|
)
|
|
|
|
pytestmark = pytest.mark.bioflow_unit
|
|
|
|
|
|
@pytest.fixture()
|
|
def image_cfg(tmp_path):
|
|
return ImageConfig(data_root=str(tmp_path / "data"), build_timeout_sec=1800.0)
|
|
|
|
|
|
@pytest.fixture()
|
|
def builder(image_cfg):
|
|
return ExtensionImageBuilder(image_cfg)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 指纹 / 命名 / Dockerfile
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_fingerprint_deterministic():
|
|
req = {"pip": ["numpy==1.26.0", "pandas"], "apt": ["libgomp1"]}
|
|
assert image_fingerprint("bioflow-gf", req) == image_fingerprint("bioflow-gf", req)
|
|
|
|
|
|
def test_fingerprint_order_independent():
|
|
assert image_fingerprint("bioflow-gf", {"pip": ["b", "a"]}) == \
|
|
image_fingerprint("bioflow-gf", {"pip": ["a", "b"]})
|
|
|
|
|
|
def test_fingerprint_changes_with_base_and_deps():
|
|
base = image_fingerprint("bioflow-gf", {"pip": ["numpy"]})
|
|
assert base != image_fingerprint("bioflow-gfpy", {"pip": ["numpy"]})
|
|
assert base != image_fingerprint("bioflow-gf", {"pip": ["numpy", "pandas"]})
|
|
assert base != image_fingerprint("bioflow-gf", {"apt": ["libgomp1"]})
|
|
|
|
|
|
def test_fingerprint_python_alias_form():
|
|
assert image_fingerprint("bioflow-gf", {"python": ["numpy"]}) == \
|
|
image_fingerprint("bioflow-gf", {"pip": ["numpy"]})
|
|
assert image_fingerprint("bioflow-gf", {"system": ["libgomp1"]}) == \
|
|
image_fingerprint("bioflow-gf", {"apt": ["libgomp1"]})
|
|
|
|
|
|
def test_image_name_format():
|
|
assert image_name_for("abcd12345678") == "bioflow-ext-abcd12345678:dev"
|
|
|
|
|
|
def test_render_dockerfile_pip_only():
|
|
text = render_dockerfile("bioflow-gf", {"pip": ["numpy==1.26.0", "pandas"]})
|
|
assert text.startswith("# auto-generated")
|
|
assert "FROM bioflow-gf:dev" in text
|
|
assert ("RUN /opt/conda/bin/pip install --no-cache-dir "
|
|
"-i https://pypi.org/simple numpy==1.26.0 pandas") in text
|
|
assert "apt-get" not in text
|
|
|
|
|
|
def test_render_dockerfile_apt_and_pip():
|
|
text = render_dockerfile("bioflow-gf", {"apt": ["libgomp1"], "pip": ["scipy"]})
|
|
# apt 行:首个包带续行反斜杠(唯一依赖时 bash 忽略尾反斜杠,安全)
|
|
assert "apt-get install" in text
|
|
assert "libgomp1" in text
|
|
assert "/opt/conda/bin/pip install" in text
|
|
|
|
|
|
def test_render_dockerfile_no_deps():
|
|
text = render_dockerfile("bioflow-gf", {})
|
|
assert "FROM bioflow-gf:dev" in text
|
|
assert "RUN true" in text
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ensure_image:命中复用 / 构建命令形态 / 失败分类 / 清理
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class FakeDocker:
|
|
"""fake docker:inspect 可编程命中、build 可编程失败、rmi 记录调用。"""
|
|
|
|
def __init__(self, builder):
|
|
self.calls: list[list[str]] = []
|
|
self.rmi_calls: list[str] = []
|
|
self._building = False
|
|
self.hit = False
|
|
self.build_stderr = ""
|
|
self._builder = builder
|
|
|
|
def inspect_result(self, name):
|
|
return {"exit_code": 0, "stdout": json.dumps([{"Id": "sha256:abc"}]), "stderr": ""}
|
|
|
|
def inspect_fail(self, name):
|
|
return {"exit_code": 1, "stdout": "", "stderr": "no such image\n"}
|
|
|
|
def run(self, cmd, timeout_sec=None, **kwargs):
|
|
"""未命中路径:inspect 失败 → build → 构建后 inspect 取 Id。"""
|
|
self.calls.append(list(cmd))
|
|
if cmd[:2] == ["docker", "inspect"]:
|
|
if self._building:
|
|
return self.inspect_result(cmd[2])
|
|
if self.hit:
|
|
return self.inspect_result(cmd[2])
|
|
return self.inspect_fail(cmd[2])
|
|
if cmd[:2] == ["docker", "build"]:
|
|
self._building = True
|
|
return {"exit_code": 0, "stdout": "built", "stderr": ""}
|
|
if cmd[:2] == ["docker", "rmi"]:
|
|
self.rmi_calls.append(cmd[2])
|
|
return {"exit_code": 0, "stdout": "", "stderr": ""}
|
|
return {"exit_code": 1, "stdout": "", "stderr": f"unknown cmd {cmd}"}
|
|
|
|
def run_fail(self, cmd, timeout_sec=None, **kwargs):
|
|
"""构建失败路径:inspect 可命中;build 返回 stderr;中间层定位;rmi 记录。"""
|
|
self.calls.append(list(cmd))
|
|
if cmd[:2] == ["docker", "inspect"]:
|
|
if self.hit:
|
|
return self.inspect_result(cmd[2])
|
|
return self.inspect_fail(cmd[2])
|
|
if cmd[:3] == ["docker", "images", "-q"]:
|
|
return {"exit_code": 0, "stdout": "sha256:inter\n", "stderr": ""}
|
|
if cmd[:2] == ["docker", "build"]:
|
|
self._building = True
|
|
return {"exit_code": 1, "stdout": "", "stderr": self.build_stderr}
|
|
if cmd[:2] == ["docker", "rmi"]:
|
|
self.rmi_calls.append(cmd[2])
|
|
return {"exit_code": 0, "stdout": "", "stderr": ""}
|
|
return {"exit_code": 1, "stdout": "", "stderr": f"unknown cmd {cmd}"}
|
|
|
|
def setup(self, builder, hit: bool = False, build_stderr: str = ""):
|
|
self.hit = hit
|
|
self.build_stderr = build_stderr
|
|
builder._run_docker = self.run_fail if build_stderr else self.run
|
|
|
|
|
|
def test_ensure_image_miss_builds_and_registers(builder, image_cfg):
|
|
fake = FakeDocker(builder)
|
|
fake.setup(builder)
|
|
name = builder.ensure_image("bioflow-gf", {"pip": ["numpy==1.26.0"]})
|
|
assert name == "bioflow-ext-" + image_fingerprint("bioflow-gf", {"pip": ["numpy==1.26.0"]}) + ":dev"
|
|
build_calls = [c for c in fake.calls if c[:2] == ["docker", "build"]]
|
|
assert len(build_calls) == 1
|
|
assert build_calls[0][2] == "-t"
|
|
assert build_calls[0][3] == name
|
|
ctx_dir = build_calls[0][4]
|
|
assert ctx_dir.endswith(name.split(":")[0].split("-")[-1]) # 上下文=指纹目录
|
|
assert (builder.registry_path()).is_file()
|
|
registry = json.loads(builder.registry_path().read_text(encoding="utf-8"))
|
|
assert name in registry["images"]
|
|
assert registry["images"][name]["base_image"] == "bioflow-gf"
|
|
assert registry["images"][name]["requirements"] == {"pip": ["numpy==1.26.0"], "apt": []}
|
|
|
|
|
|
def test_ensure_image_hit_reuses(builder, image_cfg):
|
|
fake = FakeDocker(builder)
|
|
fake.setup(builder, hit=True)
|
|
req = {"pip": ["pandas"]}
|
|
name = builder.ensure_image("bioflow-gf", req)
|
|
build_calls = [c for c in fake.calls if c[:2] == ["docker", "build"]]
|
|
assert build_calls == []
|
|
assert name == "bioflow-ext-" + image_fingerprint("bioflow-gf", req) + ":dev"
|
|
# inspect 命中但 registry 无记录 → 补登记
|
|
assert name in json.loads(builder.registry_path().read_text(encoding="utf-8"))["images"]
|
|
|
|
|
|
def test_ensure_image_hit_uses_existing_registry(builder, image_cfg):
|
|
fake = FakeDocker(builder)
|
|
fake.setup(builder, hit=True)
|
|
req = {"pip": ["pandas"]}
|
|
name = builder.ensure_image("bioflow-gf", req)
|
|
# 已登记:再 ensure 不产生新 docker 调用(纯 registry 读 + inspect 命中)
|
|
fake.calls.clear()
|
|
name2 = builder.ensure_image("bioflow-gf", req)
|
|
assert name2 == name
|
|
assert [c for c in fake.calls if c[:2] == ["docker", "build"]] == []
|
|
|
|
|
|
def test_ensure_image_no_deps_reuses_base(builder, image_cfg):
|
|
fake = FakeDocker(builder)
|
|
fake.setup(builder, hit=False) # 无依赖:inspect 失败 → 直接复用 base,不 build
|
|
name = builder.ensure_image("bioflow-gf", {})
|
|
assert name == "bioflow-gf:dev"
|
|
assert [c for c in fake.calls if c[:2] == ["docker", "build"]] == []
|
|
|
|
|
|
def test_ensure_image_pip_unavailable(builder, image_cfg):
|
|
fake = FakeDocker(builder)
|
|
fake.setup(builder, build_stderr="ERROR: No matching distribution found for nope-pkg\n")
|
|
with pytest.raises(ImageBuildError) as exc_info:
|
|
builder.ensure_image("bioflow-gf", {"pip": ["nope-pkg"]})
|
|
assert exc_info.value.kind == "pip_unavailable"
|
|
assert "No matching distribution" in exc_info.value.message
|
|
# 失败清理:build 上下文目录已删除 + rmi 调用(无 tag 中间镜像)
|
|
ctx = [c for c in fake.calls if c[:2] == ["docker", "build"]]
|
|
import pathlib
|
|
assert not pathlib.Path(ctx[0][4]).exists()
|
|
assert any(c[:2] == ["docker", "rmi"] for c in fake.calls)
|
|
|
|
|
|
def test_ensure_image_network_failure(builder, image_cfg):
|
|
fake = FakeDocker(builder)
|
|
fake.setup(builder, build_stderr="Failed to fetch http://pypi.org/nope\n")
|
|
with pytest.raises(ImageBuildError) as exc_info:
|
|
builder.ensure_image("bioflow-gf", {"pip": ["pandas"]})
|
|
assert exc_info.value.kind == "network"
|
|
|
|
|
|
def test_ensure_image_docker_missing(builder, image_cfg):
|
|
def missing(cmd, timeout_sec=None, **kwargs):
|
|
raise FileNotFoundError("docker")
|
|
|
|
builder._run_docker = missing
|
|
with pytest.raises(ImageBuildError) as exc_info:
|
|
builder.ensure_image("bioflow-gf", {"pip": ["pandas"]})
|
|
assert exc_info.value.kind == "docker_missing"
|
|
|
|
|
|
def test_ensure_image_timeout(builder, image_cfg):
|
|
import subprocess
|
|
|
|
def slow(cmd, timeout_sec=None, **kwargs):
|
|
raise subprocess.TimeoutExpired(cmd, timeout_sec)
|
|
|
|
builder._run_docker = slow
|
|
with pytest.raises(ImageBuildError) as exc_info:
|
|
builder.ensure_image("bioflow-gf", {"pip": ["pandas"]})
|
|
assert exc_info.value.kind == "timeout"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 引用计数 / 回收
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _seed_registry(builder, image_cfg):
|
|
registry = {
|
|
"version": "1",
|
|
"images": {
|
|
"bioflow-ext-aaaaaaaaaaaa:dev": {"image": "bioflow-ext-aaaaaaaaaaaa:dev",
|
|
"refs": ["ext_1"]},
|
|
"bioflow-ext-bbbbbbbbbbbb:dev": {"image": "bioflow-ext-bbbbbbbbbbbb:dev",
|
|
"refs": []},
|
|
"base:bioflow-gf:dev": {"image": "bioflow-gf", "refs": []},
|
|
},
|
|
}
|
|
path = builder.registry_path()
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(registry), encoding="utf-8")
|
|
builder.refresh_registry()
|
|
|
|
|
|
def test_collect_garbage(builder, image_cfg):
|
|
_seed_registry(builder, image_cfg)
|
|
fake = FakeDocker(builder)
|
|
fake.setup(builder, hit=True)
|
|
collected = builder.collect_garbage({"ext_1"})
|
|
names = [c["image"] for c in collected]
|
|
assert "bioflow-ext-aaaaaaaaaaaa:dev" not in names # 有引用保留
|
|
assert "bioflow-ext-bbbbbbbbbbbb:dev" in names # 无引用回收
|
|
assert "base:bioflow-gf:dev" not in names # base 映射豁免
|
|
# rmi 调用确认
|
|
assert any("bioflow-ext-bbbbbbbbbbbb:dev" in c for c in fake.rmi_calls)
|
|
|
|
|
|
def test_collect_garbage_dry_run(builder, image_cfg):
|
|
_seed_registry(builder, image_cfg)
|
|
fake = FakeDocker(builder)
|
|
fake.setup(builder, hit=True)
|
|
collected = builder.collect_garbage({"ext_1"}, dry_run=True)
|
|
assert any(c["image"] == "bioflow-ext-bbbbbbbbbbbb:dev" and not c["removed"]
|
|
for c in collected)
|
|
assert fake.rmi_calls == []
|
|
|
|
|
|
def test_register_and_unregister_ref(builder, image_cfg):
|
|
builder.registry().put("bioflow-ext-aaaaaaaaaaaa:dev",
|
|
{"image": "bioflow-ext-aaaaaaaaaaaa:dev", "refs": []})
|
|
builder.register_ref("bioflow-ext-aaaaaaaaaaaa:dev", "ext_1")
|
|
builder.register_ref("bioflow-ext-aaaaaaaaaaaa:dev", "ext_1") # 幂等
|
|
assert builder.registry().get("bioflow-ext-aaaaaaaaaaaa:dev")["refs"] == ["ext_1"]
|
|
builder.unregister_ref("bioflow-ext-aaaaaaaaaaaa:dev", "ext_1")
|
|
assert builder.registry().get("bioflow-ext-aaaaaaaaaaaa:dev")["refs"] == []
|