From 27946edc2b846e403928758b07f814f21b1bafed Mon Sep 17 00:00:00 2001 From: jerry Date: Mon, 31 Aug 2026 12:59:29 +0800 Subject: [PATCH] =?UTF-8?q?P0/M1:=20=E6=94=B6=E7=BC=96=20encryption-test?= =?UTF-8?q?=20=E5=B7=A5=E5=85=B7=E9=93=BE=E5=85=A5=20tooling/?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - legacy_plantai_server_package/: phase_a–f + compile_* + bin/plantai_admin.py + tools/(audit/elf_visibility/smoke/build_delivery_manifest) 完整基线 - legacy_docker_packager/: protected 形态参考 - crypto/client/: plantai_core.py(AES-GCM 原语) + install_plantai.sh 参考 - secrets/provision 严格排除(gitignore),入 git 前已脱敏清理 - LEGACY_MAPPING.md 记录源与改造方向 Co-Authored-By: Claude Fable 5 --- tooling/.gitignore | 10 + tooling/LEGACY_MAPPING.md | 15 + tooling/crypto/client/README.md | 186 +++ tooling/crypto/client/install_plantai.sh | 250 ++++ tooling/crypto/client/plantai_core.py | 1018 +++++++++++++++++ .../legacy_docker_packager/Dockerfile.plantai | 52 + tooling/legacy_docker_packager/Dockerfile.qa | 27 + tooling/legacy_docker_packager/Dockerfile.rag | 26 + tooling/legacy_docker_packager/README.md | 96 ++ .../legacy_docker_packager/bbav5.env.example | 39 + .../legacy_docker_packager/build_package.sh | 548 +++++++++ .../legacy_docker_packager/docker-compose.yml | 224 ++++ .../legacy_docker_packager/pack.env.example | 35 + .../requirements/plantai.txt | 32 + .../requirements/qa.txt | 40 + .../requirements/rag.txt | 27 + .../runtime/health-check.sh | 76 ++ .../legacy_docker_packager/runtime/install.sh | 55 + .../legacy_docker_packager/runtime/logs.sh | 5 + .../legacy_docker_packager/runtime/start.sh | 24 + .../legacy_docker_packager/runtime/status.sh | 6 + .../legacy_docker_packager/runtime/stop.sh | 6 + .../legacy_plantai_server_package/.gitignore | 5 + .../legacy_plantai_server_package/README.md | 106 ++ .../bin/plantai_admin.py | 802 +++++++++++++ .../compile_plantai.sh | 160 +++ .../compile_qa.sh | 135 +++ .../compile_rag.sh | 165 +++ .../issue_client.sh | 129 +++ .../phase_a_compile_all.sh | 260 +++++ .../phase_b_init_keys.sh | 45 + .../phase_c_build_dual_release.sh | 272 +++++ .../phase_e_issue_client.sh | 35 + .../phase_f_migrate_data.sh | 187 +++ .../legacy_plantai_server_package/server.env | 57 + .../start_all.sh | 206 ++++ .../legacy_plantai_server_package/stop_all.sh | 95 ++ .../templates/start_all_client.sh | 181 +++ .../tests/conftest.py | 8 + .../tests/test_admin_phase_e.py | 254 ++++ .../tests/tpm_fixtures.py | 110 ++ .../tools/audit_compiled_tree.py | 242 ++++ .../tools/build_delivery_manifest.py | 80 ++ .../tools/elf_visibility.py | 205 ++++ .../tools/export_ollama_models.py | 72 ++ .../tools/generate_stubs.py | 88 ++ .../tools/smoke_dual_build.py | 171 +++ tooling/package_client.sh | 52 + 48 files changed, 6919 insertions(+) create mode 100644 tooling/.gitignore create mode 100644 tooling/LEGACY_MAPPING.md create mode 100644 tooling/crypto/client/README.md create mode 100755 tooling/crypto/client/install_plantai.sh create mode 100644 tooling/crypto/client/plantai_core.py create mode 100644 tooling/legacy_docker_packager/Dockerfile.plantai create mode 100644 tooling/legacy_docker_packager/Dockerfile.qa create mode 100644 tooling/legacy_docker_packager/Dockerfile.rag create mode 100644 tooling/legacy_docker_packager/README.md create mode 100644 tooling/legacy_docker_packager/bbav5.env.example create mode 100755 tooling/legacy_docker_packager/build_package.sh create mode 100644 tooling/legacy_docker_packager/docker-compose.yml create mode 100644 tooling/legacy_docker_packager/pack.env.example create mode 100644 tooling/legacy_docker_packager/requirements/plantai.txt create mode 100644 tooling/legacy_docker_packager/requirements/qa.txt create mode 100644 tooling/legacy_docker_packager/requirements/rag.txt create mode 100755 tooling/legacy_docker_packager/runtime/health-check.sh create mode 100755 tooling/legacy_docker_packager/runtime/install.sh create mode 100755 tooling/legacy_docker_packager/runtime/logs.sh create mode 100755 tooling/legacy_docker_packager/runtime/start.sh create mode 100755 tooling/legacy_docker_packager/runtime/status.sh create mode 100755 tooling/legacy_docker_packager/runtime/stop.sh create mode 100644 tooling/legacy_plantai_server_package/.gitignore create mode 100644 tooling/legacy_plantai_server_package/README.md create mode 100755 tooling/legacy_plantai_server_package/bin/plantai_admin.py create mode 100755 tooling/legacy_plantai_server_package/compile_plantai.sh create mode 100755 tooling/legacy_plantai_server_package/compile_qa.sh create mode 100755 tooling/legacy_plantai_server_package/compile_rag.sh create mode 100755 tooling/legacy_plantai_server_package/issue_client.sh create mode 100755 tooling/legacy_plantai_server_package/phase_a_compile_all.sh create mode 100755 tooling/legacy_plantai_server_package/phase_b_init_keys.sh create mode 100755 tooling/legacy_plantai_server_package/phase_c_build_dual_release.sh create mode 100755 tooling/legacy_plantai_server_package/phase_e_issue_client.sh create mode 100755 tooling/legacy_plantai_server_package/phase_f_migrate_data.sh create mode 100644 tooling/legacy_plantai_server_package/server.env create mode 100755 tooling/legacy_plantai_server_package/start_all.sh create mode 100755 tooling/legacy_plantai_server_package/stop_all.sh create mode 100755 tooling/legacy_plantai_server_package/templates/start_all_client.sh create mode 100644 tooling/legacy_plantai_server_package/tests/conftest.py create mode 100644 tooling/legacy_plantai_server_package/tests/test_admin_phase_e.py create mode 100644 tooling/legacy_plantai_server_package/tests/tpm_fixtures.py create mode 100755 tooling/legacy_plantai_server_package/tools/audit_compiled_tree.py create mode 100755 tooling/legacy_plantai_server_package/tools/build_delivery_manifest.py create mode 100755 tooling/legacy_plantai_server_package/tools/elf_visibility.py create mode 100755 tooling/legacy_plantai_server_package/tools/export_ollama_models.py create mode 100755 tooling/legacy_plantai_server_package/tools/generate_stubs.py create mode 100755 tooling/legacy_plantai_server_package/tools/smoke_dual_build.py create mode 100755 tooling/package_client.sh diff --git a/tooling/.gitignore b/tooling/.gitignore new file mode 100644 index 0000000..cbbc6d4 --- /dev/null +++ b/tooling/.gitignore @@ -0,0 +1,10 @@ +# 密钥与客户交付物(绝不入 git) +secrets/ +server_fingers/ +provision/ +**/secrets/ +*.bak_* + +# 构建产物 +build/ +out/ diff --git a/tooling/LEGACY_MAPPING.md b/tooling/LEGACY_MAPPING.md new file mode 100644 index 0000000..0e687ec --- /dev/null +++ b/tooling/LEGACY_MAPPING.md @@ -0,0 +1,15 @@ +# 收编映射(encryption-test → tooling) + +源:`/home/xzq/5090/PlanAI-cooperate/hsj-workspace/encryption-test/`(2026-08-31 收编,原工具链未入 git) + +| 源 | 目标 | 说明 | +|---|---|---| +| server/plantai_server_package/ | legacy_plantai_server_package/ | 完整基线:phase_a–f、compile_{plantai,qa,rag}.sh、start/stop_all、bin/plantai_admin.py、tools/(audit_compiled_tree/elf_visibility/smoke_dual_build/build_delivery_manifest) | +| server/plantai_server_package/secrets/ | **不复制**(gitignore) | supplier 私钥/口令/master_keys,绝不入 git | +| server/plantai_server_package/provision/ | **不复制** | 客户交付物(含机器指纹),运行时产物 | +| server/protected_docker_packager/ | legacy_docker_packager/ | 参考实现(docker 形态,本方案走 shell 形态) | +| client/bin/plantai_core.py | crypto/client/plantai_core.py | AES-GCM/SHA256 原语 + Ed25519(后续剥离 TPM/RSA-OAEP) | +| client/test/install_plantai.sh | crypto/client/install_plantai.sh | 安装器参考(后续去指纹/license/TPM) | +| package_client.sh | package_client.sh | 客户端打包参考 | + +改造方向(P1–P6 落地):phase_a → build_all.sh + compile/*;compile_qa.sh → 参数化 ENTRY_SCRIPT + compile.manifest 白名单;phase_b → crypto/init_keys.sh(去供应商 RSA);phase_c → assemble/(去 TPM/license/Ollama);phase_f → kb/encrypt_kb.sh;plantai_core.py → crypto/(AES-GCM+SHA256)。server.env 旧路径由新 tooling.env 覆盖。 diff --git a/tooling/crypto/client/README.md b/tooling/crypto/client/README.md new file mode 100644 index 0000000..d5d4b67 --- /dev/null +++ b/tooling/crypto/client/README.md @@ -0,0 +1,186 @@ +# PlantAI + RAG + QA 客户端部署包 + +本目录可以交给客户：不含供应商私钥和明文 `K_master`，只能采集指纹、配置 TPM、验证供应商签名、解封密钥和解密已授权资产。 + +## 客户专属配置 — `install.env` 变量详解 + +以下表格覆盖 `install.env.example` 中所有变量。每客户交付时据此填写 `install.env`。 + +| 变量 | 默认值 | 作用 | 影响路径 | 每客户必改 | 取值依据 | +|------|--------|------|----------|:----------:|----------| +| `PLANTAI_ROOT` | `/opt/PlantAI` 或 `$HOME/PlantAI` | 安装根目录，其下自动创建所有子目录 | → `bin/` `config/` `assets/` `data/` `releases/` `runtime/` | 通常不改 | 客户目录规范（如 `/data/PlantAI`） | +| `RELEASE_SRC` | `./release` | 供应商发行包解压目录，安装时从此取 VERSION / app/ / assets/ / config/ | → `$PLANTAI_ROOT/releases//` 的源 | 不改 | 保留默认值 | +| `DEVICE_ID` | `$(hostname)` | 设备业务标识，写入许可证和日志，用于绑定设备身份 | → 出现在 `license.json` / 监控系统 | **必改** | `<客户缩写>-<机器角色>-<序号>`，如 `huanuo-web01` | +| `PLANTAI_WEB_PORT` | `7880` | PlantAI Web 服务监听端口 | → 服务绑定端口 | 偶尔改 | 客户现有服务已占用时 | +| `RAG_PORT` | `7900` | RAG 服务监听端口 | → 服务绑定端口 | 偶尔改 | 客户现有服务已占用时 | +| `PLANTAI_PYTHON` | `python3` | PlantAI 后端使用的 Python 解释器路径 | → 执行 `plantai_client.py` / 后端进程 | **必改** | 问客户 conda 环境路径，如 `/opt/miniconda3/envs/plantai/bin/python` | +| `RAG_PYTHON` | 同 `PLANTAI_PYTHON` | RAG 服务使用的 Python 解释器路径 | → 执行 `start_qa_v1_0_0_7908.sh` 等 | **必改** | 同上，通常与 `PLANTAI_PYTHON` 相同 | +| `NEO4J_URI` | `bolt://127.0.0.1:7687` | Neo4j 图数据库连接地址 | → 应用通过此 URI 连接图数据库 | 通常改 IP | 客户 Neo4j 实际主机 IP | +| `NEO4J_USER` | `neo4j` | Neo4j 用户名 | → 数据库连接认证 | 通常不改 | 除非客户改了用户名 | +| `NEO4J_PASSWORD` | 无 | Neo4j 密码 | → 数据库连接认证 | **必改** | 客户管理员提供，**禁用默认/空密码** | +| `QDRANT_URL` | `http://127.0.0.1:6333` | Qdrant 向量数据库 HTTP 地址 | → 应用通过此 URL 读写向量 | 通常改 IP | 客户 Qdrant 实际主机 IP | +| `V15_QDRANT_COLLECTION` | 无 | Qdrant 集合名，反映客户 + 数据版本 | → 读写此集合中的向量数据 | **必改** | `v15_<客户缩写>_<日期>`，如 `v15_huanuo_20260701` | +| `V15_BGE_MODEL` | `bge-m3:567m` | Ollama 中的 embedding 模型名 | → 文本向量化使用的模型 | 不改 | 仅更换 embedding 策略时动 | +| `V15_OLLAMA_EMBED_URL` | `http://127.0.0.1:11434` | Ollama 服务 URL | → embedding API 调用地址 | 偶尔改 | Ollama 不在本机时改 IP | +| `INSTALL_USER_SERVICE` | `0` | 是否注册 systemd --user 自启服务 | → `$HOME/.config/systemd/user/plantai.service` | 可选 | 客户要求开机自启时设 `=1` | + +**安全要求：** +- `install.env` 创建后执行 `chmod 600` +- **只写入真实密码**到 `install.env`，`install.env.example` 永远只含占位符 +- `install.env` 包含客户凭证，禁止提交到代码仓库或分享 + +## 交付结构 + +```text +client/ + bin/plantai_client.py # 客户端工具（采集指纹、TPM、解密） + bin/PlantAI-launcher # 启动器（解密代码运行资产到 tmpfs，启动三项目） + install_plantai.sh # 安装脚本（fingerprint / install / tpm2-setup） + install.env.example # ⬅ 客户据此创建 install.env（每客户不同） + phase_d_tpm.sh # 阶段 D：TPM 初始化 + 指纹采集（硬件） + phase_d_swtpm_test.sh # 软件 TPM 测试版本 + README.md # 本文件 + release/ ⬅ 解压供应商发行包到这里 + VERSION + app/PlantAI_compiled/ + app/RAG/ + app/QA/ + app/start_all.sh + DELIVERY_MANIFEST.json + assets/runtime-assets.enc + assets/runtime-assets.manifest.json + config/supplier_public_key.pem +``` + +另随包交付 `plantai-rag-runtime-data-.tar` 及对应 `.sha256`，其中包含 +Qdrant、Neo4j 数据卷、BGE reranker 和本地 Ollama 模型 +`bge-m3:567m`、`qwen2.5:14b-instruct`。数据库卷不放入 `/dev/shm`，避免约 +70GB 持久数据占满内存文件系统。 + +> `install.env`（含真实密码）由客户在部署时根据 `install.env.example` 创建，不包含在初始交付包中。 + +## 每客户修改指南（交付前必做） + +`install.env.example` → `install.env` 按上方表格填写即可，此处只列 README 中需要客户化的文本： + +| 位置 | 当前内容 | 改为 | +|------|----------|------| +| 准备配置 | `tar -C release -xzf plantai-rag-release-1.0.0.tar.gz` | 版本号改为本次实际版本 | +| 导入授权 | `PROVISION_BUNDLE=/path/to/plantai-provision-DEVICE.tar.gz` | `DEVICE` 替换为 `install.env` 中 `DEVICE_ID` | +| 正式启动 | `PLANTAI_ROOT=/opt/PlantAI` | 改为 `install.env` 中 `PLANTAI_ROOT` 值 | +| 外部服务 | `ollama pull qwen3:0.6b` | 模型版本按本次实际版本改 | + +另在 README 末尾追加客户档案： +```markdown +## 本次交付记录 +- **客户名称**: +- **DEVICE_ID**: +- **发行包版本**: +- **交付日期**: +- **供应商负责人**: +``` + +### 交付前 checklist + +- [ ] `client/` 中删除了 `install.env`（只留 `.example`） +- [ ] `client/` 中删除了 `package_client.sh` +- [ ] `install.env.example` 中无真实密码 +- [ ] 发行包 `plantai-rag-release-.tar.gz` 已单独发给客户 + +## 1. 准备配置与发行包 + +```bash +cp install.env.example install.env +chmod 600 install.env +mkdir -p release +tar -C release -xzf plantai-rag-release-1.0.0.tar.gz +sha256sum -c release/SHA256SUMS +``` + +在 `install.env` 设置安装路径、Python 解释器以及 Neo4j/Qdrant/Ollama 变量。真实密码不要写入 `install.env.example`。 + +## 2. TPM 2.0 初始化（每台目标机只做一次） + +推荐直接执行阶段 D 一键脚本： + +```bash +./phase_d_tpm.sh +``` + +它会依次检查 TPM、复用或创建 TPM 密钥、采集指纹，并拒绝缺少 TPM 公钥/attestation 的弱指纹。 + +下面是等价的手工命令，仅用于排错： + +```bash +ls -l /dev/tpmrm0 /dev/tpm0 +tpm2_getrandom 8 +./install_plantai.sh tpm2-setup +``` + +不要删除 `$PLANTAI_ROOT/config/tpm/`，也不要反复运行初始化。更换 TPM/主板或删除持久化 TPM key 后，旧授权将无法解封。 + +## 3. 采集指纹并交给供应商 + +### 无真实 TPM 时的软件测试入口 + +```bash +./phase_d_swtpm_test.sh +``` + +该入口会使用端口 2321/2322 的 swtpm，自动运行阶段 D，并在指纹中写入 `test_only_swtpm=true`。它只能验证流程，不能提供硬件级保护。阶段 E 测试签发必须显式设置 `ALLOW_SOFTWARE_TPM_TEST=1`。 + +```bash +./install_plantai.sh fingerprint +``` + +将 `$PLANTAI_ROOT/config/device_fingerprint.json` 发给供应商。生产授权必须包含 `tpm_public_key` 和 `attestation`；只有普通 DMI/磁盘信息的指纹不能提供 TPM 硬件绑定。 + +## 4. 导入授权并安装 + +```bash +PROVISION_BUNDLE=/path/to/plantai-provision-DEVICE.tar.gz ./install_plantai.sh install +``` + +校验并展开随包交付的持久化运行数据： + +```bash +sha256sum -c plantai-rag-runtime-data-1.0.0.tar.sha256 +mkdir -p "${PLANTAI_ROOT:-/opt/PlantAI}/data" +tar -C "${PLANTAI_ROOT:-/opt/PlantAI}/data" \ + -xf plantai-rag-runtime-data-1.0.0.tar +``` + +安装结束会执行 `PlantAI-launcher --check-only`：验证许可证签名、硬件匹配、TPM 解封、资产清单和 AES-GCM 解密。任何一步失败都不应启动服务。 + +正式启动： + +```bash +PLANTAI_ROOT=/opt/PlantAI /opt/PlantAI/bin/PlantAI-launcher +``` + +启动器默认把加密的 PlantAI/RAG `models/prompts` 解密到 `/dev/shm`；QA 的 +Qdrant、Neo4j 与大模型保留在 `$PLANTAI_ROOT/data/runtime-assets`。客户启动 +脚本会依次检查/启动 Qdrant、Neo4j、独立端口的本地 Ollama、QA、RAG、 +PlantAI。不要设置 `PLANTAI_PERSIST_RUNTIME=1`，除非明确接受代码运行资产明文落盘风险。 + +## 5. 依赖与端口 + +客户机需预装 Docker 与 Ollama。启动脚本会使用交付数据自动拉起依赖，无需 +联网拉取模型。默认端口如下： + +```bash +curl -fsS http://127.0.0.1:16333/collections # Qdrant +curl -fsS http://127.0.0.1:17474 # Neo4j HTTP +curl -fsS http://127.0.0.1:11435/api/tags # 交付专用 Ollama +curl -fsS http://127.0.0.1:7925/api/v1/qa-v1-1-73/health +curl -fsS http://127.0.0.1:7900/health +curl -fsS http://127.0.0.1:7880/ +``` + +## 安全边界 + +- `.so` 是逆向加固，不是密码学加密。 +- TPM 保护的是解封 `K_master` 的能力；私钥从不离开 TPM。 +- 许可证和 `asset.key.enc` 通过 SHA-256 绑定，不能跨设备混用。 +- 客户端没有生成供应商密钥、自签许可证或重新加密资产的命令。 +- 供应商的 `plantai_server_package`、私钥、口令文件、`master_keys/` 永不进入 client。 diff --git a/tooling/crypto/client/install_plantai.sh b/tooling/crypto/client/install_plantai.sh new file mode 100755 index 0000000..480d7bf --- /dev/null +++ b/tooling/crypto/client/install_plantai.sh @@ -0,0 +1,250 @@ +#!/usr/bin/env bash +# PlantAI client-side installer. Safe to ship to any customer: this script +# and its bin/ folder contain no private key and no master-key logic, only +# the ability to collect a fingerprint and consume vendor-issued files. +# +# Usage: +# ./install_plantai.sh fingerprint +# -> collects this machine's hardware fingerprint and tells you where +# to send it to the vendor. Run this FIRST, before you have a +# license. +# +# ./install_plantai.sh install +# -> sets up the directory tree, installs the app + pre-encrypted +# assets from ./release/, and imports the vendor-issued +# license.json / asset.key.enc if you've placed the provision +# bundle at ./release/provision/ (or point PROVISION_BUNDLE at the +# .tar.gz the vendor sent you). +set -euo pipefail + +umask 027 # Config files owner rwx, group rx; prevent world-readable by default + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +for cfg in "/etc/plantai/install.env" "$SCRIPT_DIR/install.env"; do + [ -f "$cfg" ] && . "$cfg" +done + +# --------------------------------------------------------------------------- +# Safety: validate that a path is safe to rm -rf +# --------------------------------------------------------------------------- +validate_path_under() { + local target="$1" + local root="$2" + local desc="${3:-path}" + + [ -z "$target" ] && { echo "SECURITY: $desc is empty — refusing to rm -rf" >&2; exit 1; } + [ "$target" = "/" ] && { echo "SECURITY: $desc is '/' — refusing to rm -rf" >&2; exit 1; } + + local resolved + resolved="$(cd "$(dirname "$target" 2>/dev/null || echo /)" 2>/dev/null && pwd)/$(basename "$target")" || true + local root_resolved + root_resolved="$(cd "$root" 2>/dev/null && pwd)" || root_resolved="" + + if [ -n "$root_resolved" ] && [ -n "$resolved" ]; then + case "$resolved" in + "$root_resolved"|"$root_resolved/"*) ;; + *) + echo "SECURITY: $desc ($resolved) is NOT under expected root ($root_resolved) — refusing to rm -rf" >&2 + exit 1 + ;; + esac + fi + + [ "$target" = "$HOME" ] && { echo "SECURITY: $desc is \$HOME — refusing to rm -rf" >&2; exit 1; } + return 0 +} + +validate_version() { + local v="$1" + [ -z "$v" ] && { echo "SECURITY: version string is empty" >&2; exit 1; } + [[ "$v" =~ ^[A-Za-z0-9._-]+$ ]] || { + echo "SECURITY: version string '$v' contains illegal characters — only [A-Za-z0-9._-] allowed" >&2 + exit 1 + } + [[ "$v" == *".."* ]] && { echo "SECURITY: version string '$v' contains '..'" >&2; exit 1; } + return 0 +} + +default_root() { + if [ "$(id -u)" = "0" ] || [ -w "/opt" ]; then + echo "/opt/PlantAI" + else + echo "${HOME}/PlantAI" + fi +} + +PLANTAI_ROOT="${PLANTAI_ROOT:-$(default_root)}" +RELEASE_SRC="${RELEASE_SRC:-$SCRIPT_DIR/release}" +DEVICE_ID="${DEVICE_ID:-$(hostname)}" +INSTALL_USER_SERVICE="${INSTALL_USER_SERVICE:-0}" +WEB_PORT="${PLANTAI_WEB_PORT:-7880}" +PYTHON="${PLANTAI_PYTHON:-python3}" + +if [ -z "${BBAV5_CONDA_BASE:-}" ] && command -v conda >/dev/null 2>&1; then + BBAV5_CONDA_BASE="$(conda info --base 2>/dev/null || true)" +fi +BBAV5_CONDA_BASE="${BBAV5_CONDA_BASE:-}" +BBAV5_DEFAULT_ENV="${BBAV5_DEFAULT_ENV:-plantai}" +BBAV5_QUEUE_PREFIX="${BBAV5_QUEUE_PREFIX:-plantai_${WEB_PORT}}" +BBAV5_REDIS_URL="${BBAV5_REDIS_URL:-redis://localhost:6379/0}" + +cmd_fingerprint() { + mkdir -p "$PLANTAI_ROOT/config" + "$PYTHON" "$SCRIPT_DIR/bin/plantai_client.py" collect-fingerprint \ + --device-id "$DEVICE_ID" \ + --root "$PLANTAI_ROOT" \ + --output "$PLANTAI_ROOT/config/device_fingerprint.json" + echo + echo "Fingerprint written to: $PLANTAI_ROOT/config/device_fingerprint.json" + echo "Send this file to your vendor. They will send back a provision bundle" + echo "(license.json + asset.key.enc) -- place it at $RELEASE_SRC/provision/" + echo "or set PROVISION_BUNDLE=/path/to/plantai-provision-*.tar.gz, then run:" + echo " ./install_plantai.sh install" +} + +cmd_install() { + need_file() { [ -e "$1" ] || { echo "Required path not found: $1" >&2; exit 1; }; } + + need_file "$RELEASE_SRC/app" + need_file "$SCRIPT_DIR/bin/plantai_client.py" + need_file "$SCRIPT_DIR/bin/plantai_core.py" + need_file "$SCRIPT_DIR/bin/PlantAI-launcher" + + VERSION="$(cat "$RELEASE_SRC/VERSION" 2>/dev/null || echo unknown)" + validate_version "$VERSION" + RELEASE_DIR="$PLANTAI_ROOT/releases/$VERSION" + APP_DIR="$RELEASE_DIR/app" + + echo "[1/6] Create PlantAI directory tree: $PLANTAI_ROOT" + mkdir -p \ + "$PLANTAI_ROOT/bin" "$PLANTAI_ROOT/config" "$PLANTAI_ROOT/assets" \ + "$PLANTAI_ROOT/data/input" "$PLANTAI_ROOT/data/workspace" "$PLANTAI_ROOT/data/final" \ + "$PLANTAI_ROOT/data/logs" "$PLANTAI_ROOT/data/memory_storage" "$PLANTAI_ROOT/data/reproducibility" \ + "$PLANTAI_ROOT/var/cache" "$PLANTAI_ROOT/var/tmp" "$PLANTAI_ROOT/var/lock" "$PLANTAI_ROOT/var/audit" \ + "$PLANTAI_ROOT/runtime" "$PLANTAI_ROOT/tools" "$RELEASE_DIR" + + echo "[2/6] Install app code from release payload (already excludes models/prompts)" + validate_path_under "$APP_DIR" "$PLANTAI_ROOT/releases" "APP_DIR" + rm -rf "$APP_DIR" + mkdir -p "$APP_DIR" + rsync -a --delete "$RELEASE_SRC/app/" "$APP_DIR/" + ln -sfn "$RELEASE_DIR" "$PLANTAI_ROOT/releases/current" + + echo "[3/6] Install launcher + client tool" + install -m 0755 "$SCRIPT_DIR/bin/plantai_client.py" "$PLANTAI_ROOT/bin/plantai_client.py" + install -m 0644 "$SCRIPT_DIR/bin/plantai_core.py" "$PLANTAI_ROOT/bin/plantai_core.py" + install -m 0755 "$SCRIPT_DIR/bin/PlantAI-launcher" "$PLANTAI_ROOT/bin/PlantAI-launcher" + + echo "[4/6] Install pre-encrypted assets + public key from release payload" + rsync -a "$RELEASE_SRC/assets/" "$PLANTAI_ROOT/assets/" + install -m 0644 "$RELEASE_SRC/config/supplier_public_key.pem" "$PLANTAI_ROOT/config/supplier_public_key.pem" + + echo "[5/6] Import vendor-issued license + asset key, if available" + PROVISION_BUNDLE="${PROVISION_BUNDLE:-}" + if [ -n "$PROVISION_BUNDLE" ] && [ -f "$PROVISION_BUNDLE" ]; then + # Extract to a temp dir first so we can inspect filenames before + # installing to the real config directory. + PROV_TMP="$(mktemp -d "${TMPDIR:-/tmp}/plantai-provision.XXXXXX")" + tar -C "$PROV_TMP" -xzf "$PROVISION_BUNDLE" license.json asset.key.enc + for fn in license.json asset.key.enc; do + if [ ! -f "$PROV_TMP/$fn" ]; then + echo "ERROR: provision bundle is missing required file: $fn" >&2 + validate_path_under "$PROV_TMP" "${TMPDIR:-/tmp}" "PROV_TMP" + rm -rf "$PROV_TMP" + exit 1 + fi + if [ -L "$PROV_TMP/$fn" ]; then + echo "ERROR: provision bundle contains a symlink for $fn (unsafe)" >&2 + validate_path_under "$PROV_TMP" "${TMPDIR:-/tmp}" "PROV_TMP" + rm -rf "$PROV_TMP" + exit 1 + fi + done + install -m 0644 "$PROV_TMP/license.json" "$PLANTAI_ROOT/config/license.json" + install -m 0644 "$PROV_TMP/asset.key.enc" "$PLANTAI_ROOT/config/asset.key.enc" + validate_path_under "$PROV_TMP" "${TMPDIR:-/tmp}" "PROV_TMP" + rm -rf "$PROV_TMP" + elif [ -f "$RELEASE_SRC/provision/license.json" ] && [ -f "$RELEASE_SRC/provision/asset.key.enc" ]; then + install -m 0644 "$RELEASE_SRC/provision/license.json" "$PLANTAI_ROOT/config/license.json" + install -m 0644 "$RELEASE_SRC/provision/asset.key.enc" "$PLANTAI_ROOT/config/asset.key.enc" + else + echo " No license.json/asset.key.enc found yet." + echo " Run './install_plantai.sh fingerprint' first if you haven't, send it to" + echo " your vendor, then place their reply at $RELEASE_SRC/provision/ and re-run install." + fi + + { + echo "PLANTAI_ROOT=$PLANTAI_ROOT" + echo "PLANTAI_RUNTIME=$PLANTAI_ROOT/runtime" + echo "PLANTAI_WEB_PORT=$WEB_PORT" + echo "PLANTAI_PYTHON=$PYTHON" + [ -n "$BBAV5_CONDA_BASE" ] && echo "BBAV5_CONDA_BASE=$BBAV5_CONDA_BASE" + echo "BBAV5_DEFAULT_ENV=$BBAV5_DEFAULT_ENV" + echo "BBAV5_QUEUE_PREFIX=$BBAV5_QUEUE_PREFIX" + echo "BBAV5_REDIS_URL=$BBAV5_REDIS_URL" + for name in \ + NEO4J_URI NEO4J_USER NEO4J_PASSWORD QA_NEO4J_PASSWORD \ + QDRANT_URL V15_QDRANT_COLLECTION V15_BGE_MODEL V15_OLLAMA_EMBED_URL \ + RAG_PYTHON RAG_PORT QA_PYTHON QA_PORT \ + QA_QDRANT_PORT QA_QDRANT_GRPC_PORT \ + QA_NEO4J_HTTP_PORT QA_NEO4J_BOLT_PORT \ + OLLAMA_PORT PLANTAI_RUNTIME_DATA \ + QA_OLLAMA_GENERATION_MODEL QA_OLLAMA_EMBED_MODEL + do + if [ -n "${!name:-}" ]; then + printf '%s=%s\n' "$name" "${!name}" + fi + done + } > "$PLANTAI_ROOT/config/service.env" + chmod 600 "$PLANTAI_ROOT/config/service.env" + + echo "[6/6] Create systemd user service file" + mkdir -p "$PLANTAI_ROOT/config/systemd" + cat > "$PLANTAI_ROOT/config/systemd/plantai.service" <&2; exit 1 ;; +esac diff --git a/tooling/crypto/client/plantai_core.py b/tooling/crypto/client/plantai_core.py new file mode 100644 index 0000000..94a3863 --- /dev/null +++ b/tooling/crypto/client/plantai_core.py @@ -0,0 +1,1018 @@ +#!/usr/bin/env python3 +""" +PlantAI CLIENT-SIDE core library. + +Splitting logic out of plantai_client.py makes the two workflow phases +independently testable: + + Phase A (client): fingerprint collection + TPM provisioning + (collect_fingerprint / tpm2_setup_key / tpm2_certify_wrapkey) + Phase E (server): K_master wrapping + license issuance using a fingerprint + (handled by plantai_admin.py, which only needs a JSON file + produced by this library — no client import required) + +By design this module contains NO private key handling and NO random key +generation for licenses or assets. It can only: + - collect this machine's hardware fingerprint (to send to the vendor) + - verify a license signature using the supplier PUBLIC key + - unwrap an already-issued asset key envelope and decrypt asset packages + +Signing and master-key handling only exist in plantai_admin.py, which lives +on the vendor's server and is never shipped. +""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import os +import shutil +import socket +import subprocess +import tarfile +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.kdf.hkdf import HKDF + + +# --------------------------------------------------------------------------- +# generic helpers +# --------------------------------------------------------------------------- +def now_iso() -> str: + return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds") + + +def read_text(path: Path) -> str: + try: + return path.read_text(encoding="utf-8").strip() + except Exception: + return "" + + +def run(cmd: list[str]) -> str: + try: + return subprocess.check_output(cmd, text=True, stderr=subprocess.DEVNULL).strip() + except Exception: + return "" + + +def canonical_bytes(data: Any) -> bytes: + return json.dumps(data, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def sha256_hex(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def write_json(path: Path, data: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def log_audit(root: Path, event: dict[str, Any]) -> None: + event = dict(event) + event.setdefault("at", now_iso()) + path = root / "var" / "audit" / "launcher-audit.jsonl" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as f: + f.write(json.dumps(event, ensure_ascii=False) + "\n") + + +# =========================================================================== +# TPM 2.0 support — hardware-bound K_master unwrapping +# =========================================================================== +# Problem this solves: +# The legacy (v2) wrapping uses HKDF(core_id_hash, salt) where BOTH +# parameters are stored in asset.key.enc. Anyone with the client package +# can recompute the wrap key and decrypt K_master — this is NOT hardware +# binding, it's just obfuscation. +# +# TPM-based (v3) wrapping provides REAL hardware binding: +# 1. Client runs "tpm2-setup" ONCE: +# → A non-exportable RSA-2048 key is created INSIDE the TPM chip. +# → The private key NEVER exists in plaintext outside the TPM. +# → The public key is exported and sent to the vendor. +# 2. Vendor wraps K_master with RSA-OAEP using the client's TPM public key. +# → Only the TPM holding the corresponding private key can unwrap it. +# 3. At runtime, the client asks the TPM to decrypt: +# → tpm2_rsadecrypt returns K_master; the private key never leaves the TPM. +# +# Requirements: +# - Linux kernel >= 4.12 with TPM 2.0 (/dev/tpm0 or /dev/tpmrm0) +# - tpm2-tools >= 5.x (apt install tpm2-tools / dnf install tpm2-tools) +# - Read access to TPM device (user in tss group, or run as root) +# +# Without a TPM, the system falls back to legacy HKDF wrapping and prints +# a clear warning that this is weak protection. +# =========================================================================== + +_TPM_HANDLE_BASE = 0x81008000 # Well-known persistent-handle range +_TPM_STATE_NAME = "tpm_wrapkey_state.json" + + +class TpmBackend: + """Abstraction over the tpm2-tools command layer. + + The real backend shells out to tpm2-tools. Tests substitute a fake + backend (or monkeypatch the module-level ``_TPM`` instance) so the + provisioning logic can run without a physical TPM / swtpm. + """ + + def run(self, *args: str) -> str: + """Run a tpm2-tools command. Returns stdout. Raises SystemExit on failure.""" + cmd = list(args) + try: + output = subprocess.check_output(cmd, stderr=subprocess.PIPE) + # Some tpm2-tools commands emit binary status/signature bytes on + # stdout even when their useful artifacts are directed to files. + # Text-producing commands remain UTF-8; ignored binary stdout must + # never crash the workflow with UnicodeDecodeError. + return output.decode("utf-8", errors="replace").strip() + except subprocess.CalledProcessError as e: + stderr_text = (e.stderr or "").strip() + raise SystemExit( + f"TPM command failed: {' '.join(cmd)}\n" + f" exit={e.returncode} stderr={stderr_text}" + ) + except FileNotFoundError: + raise SystemExit( + "tpm2-tools is not installed.\n" + " Debian/Ubuntu: sudo apt install tpm2-tools\n" + " RHEL/CentOS: sudo dnf install tpm2-tools\n" + " Arch: sudo pacman -S tpm2-tools" + ) + + def is_available(self) -> bool: + """Check whether TPM 2.0 userspace + kernel device are functional.""" + try: + subprocess.run( + ["tpm2_getrandom", "4"], + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + timeout=10, + ) + return True + except Exception: + return False + + def flush_transients(self) -> None: + """Free libtpms' three transient slots between CLI context loads. + + tpm2-tools may leave a loaded copy behind when a saved context file + is passed to the next command. Only do this for the explicit swtpm + test backend; flushing all transient objects on a real/shared TPM is + unsafe. + """ + if os.environ.get("TPM2TOOLS_TCTI", "").startswith("swtpm:"): + subprocess.run( + ["tpm2_flushcontext", "-t"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + + +# Default backend used by the production CLI. Tests replace this instance. +_TPM = TpmBackend() + + +def tpm2_is_available() -> bool: + """Check whether TPM 2.0 userspace + kernel device are functional.""" + return _TPM.is_available() + + +def read_public_key_pem(handle: int, tpm: TpmBackend | None = None) -> str: + """Export the public key (PEM) of a persistent TPM-resident key. + + NOTE: tpm2-readtools 5.x NEVER prints the PEM on stdout — stdout always + carries the human-readable TPMT_PUBLIC dump regardless of -f. The PEM + only lands in the file given via -o, so we write it to a temp file and + read it back. (Capturing stdout instead produced the text dump and made + the server-side load_pem_public_key() fail with MalformedFraming.) + """ + tpm = tpm or _TPM + tpm.flush_transients() + with tempfile.TemporaryDirectory(prefix="plantai-pem-") as tmp: + pem_path = Path(tmp) / "wrapkey_pub.pem" + tpm.run("tpm2_readpublic", "-c", f"0x{handle:08X}", "-f", "pem", + "-o", str(pem_path)) + return pem_path.read_text(encoding="utf-8").strip() + + +def tpm2_try_load_state(root: Path) -> dict[str, Any] | None: + """If a TPM wrapping key state file exists, return it. Otherwise None.""" + state_path = root / "config" / "tpm" / _TPM_STATE_NAME + if state_path.exists(): + return load_json(state_path) + return None + + +def tpm2_setup_key(root: Path, tpm: TpmBackend | None = None) -> dict[str, Any]: + """One-time TPM provisioning: create a non-exportable RSA-2048 key inside + the TPM, persist it across reboots, and return its public metadata. + + Idempotent — if a key already exists, just re-export the public key + instead of creating a new one (which would invalidate existing licenses). + """ + tpm = tpm or _TPM + if not tpm.is_available(): + raise SystemExit( + "TPM 2.0 is not available on this machine.\n\n" + "Check with:\n" + " ls /dev/tpm* # should show /dev/tpm0 or /dev/tpmrm0\n" + " sudo dmesg | grep -i tpm # kernel detection log\n" + " groups # should include 'tss'\n\n" + "If this machine has no TPM (common on cloud VMs without vTPM),\n" + "use the legacy path — it provides weaker (file-level) protection." + ) + + tpm_dir = root / "config" / "tpm" + tpm_dir.mkdir(parents=True, exist_ok=True) + state_path = tpm_dir / _TPM_STATE_NAME + + # ---- Already set up? ---- + if state_path.exists(): + state = load_json(state_path) + handle = state["persistent_handle"] + try: + pubkey_pem = read_public_key_pem(handle, tpm) + except SystemExit: + raise SystemExit( + f"TPM persistent key at handle 0x{handle:08X} is missing.\n" + "The TPM may have been cleared or the key evicted.\n" + f"Remove {state_path} and re-run tpm2-setup." + ) + # Repair a previous partial setup without rotating the wrapping key. + # This is especially useful for swtpm, where transient-slot exhaustion + # may have allowed persistence but interrupted attestation generation. + if not state.get("attestation"): + tpm.flush_transients() + print("Existing TPM key has no attestation — generating it now …") + state["attestation"] = tpm2_certify_wrapkey(root, tpm) + write_json(state_path, state) + print(f"TPM key already exists at persistent handle 0x{handle:08X} — reusing it.") + result = { + "tpm_version": "2.0", + "persistent_handle": handle, + "algorithm": "RSA-2048-OAEP-SHA256", + "public_key_pem": pubkey_pem, + "public_key_sha256": sha256_hex(pubkey_pem.encode()), + } + if state.get("attestation"): + result["attestation"] = state["attestation"] + return result + + # ---- Create a new non-exportable key ---- + primary_ctx = tpm_dir / "primary.ctx" + child_pub = tpm_dir / "wrapkey.pub" + child_priv = tpm_dir / "wrapkey.priv" + child_ctx = tpm_dir / "wrapkey.ctx" + + # 1. Create a primary key under the storage (owner) hierarchy. + # This "primary key" is a TPM-internal root that never leaves the chip. + print("Creating TPM primary key (storage hierarchy) …") + tpm.run("tpm2_createprimary", "-C", "o", + "-g", "sha256", "-G", "rsa", + "-c", str(primary_ctx)) + tpm.flush_transients() + + # 2. Create a child RSA-2048 key for K_master wrapping. + # Key attributes (each one is critical): + # fixedtpm — cannot be duplicated to another TPM chip + # fixedparent — bound to this specific primary key + # sensitivedataorigin — private key material generated by TPM's RNG + # userwithauth — allows setting an auth value (we leave it empty) + # decrypt — key can perform RSA decryption (OAEP) + print("Creating non-exportable TPM wrapping key (RSA-2048) …") + tpm.run("tpm2_create", "-C", str(primary_ctx), + # This is an unrestricted RSA decryption child used with OAEP. + # Its symmetric definition must be NULL. aes128cfb is appropriate + # for a restricted storage parent, but is rejected for this child by + # conforming TPMs (including swtpm/libtpms) with TPM_RC_SYMMETRIC. + "-g", "sha256", "-G", "rsa2048:null", + "-a", "fixedtpm|fixedparent|sensitivedataorigin|userwithauth|decrypt", + "-u", str(child_pub), + "-r", str(child_priv)) + + tpm.flush_transients() + + # 3. Load the key into the TPM (it's still transient — gone after reboot) + tpm.run("tpm2_load", "-C", str(primary_ctx), + "-u", str(child_pub), "-r", str(child_priv), + "-c", str(child_ctx)) + + tpm.flush_transients() + + # 4. Persist the key so it survives reboots. + # Persistent handles live in the range 0x81000000–0x81FFFFFF. + persistent_handle = None + for offset in range(3): + handle = _TPM_HANDLE_BASE + offset + try: + tpm.run("tpm2_evictcontrol", "-C", "o", + "-c", str(child_ctx), + f"0x{handle:08X}") + persistent_handle = handle + break + except SystemExit: + continue + + if persistent_handle is None: + raise SystemExit( + "Could not persist TPM key — all candidate handles are occupied.\n" + "List persistent handles: tpm2_getcap handles-persistent\n" + "Free one with: tpm2_evictcontrol -C o -c " + ) + + # 5. Export the public key (PEM) — this is what we send to the vendor. + # The private key stays inside the TPM; it CANNOT be exported. + pubkey_pem = read_public_key_pem(persistent_handle, tpm) + (tpm_dir / "wrapkey_pub.pem").write_text(pubkey_pem) + + # 6. Save state so the unwrap path knows which handle to use. + state: dict[str, Any] = { + "persistent_handle": persistent_handle, + "algorithm": "RSA-2048-OAEP-SHA256", + "public_key_sha256": sha256_hex(pubkey_pem.encode()), + "created_at": now_iso(), + } + write_json(state_path, state) + + # 7. Generate a TPM attestation that cryptographically proves the + # wrapping key is TPM-resident with the claimed attributes. + # The server uses this to verify the key is not a software fake. + try: + attestation = tpm2_certify_wrapkey(root, tpm) + state["attestation"] = attestation + write_json(state_path, state) + except SystemExit as e: + print(f"WARNING: Could not create TPM attestation: {e}") + print("The wrapping key is functional, but the server will not be") + print("able to cryptographically verify it is TPM-resident.") + print("Re-run tpm2-setup after resolving the issue to enable attestation.") + state["attestation"] = None + write_json(state_path, state) + + # Clean up intermediate files — the real key lives in TPM silicon now + for p in [primary_ctx, child_pub, child_priv, child_ctx]: + p.unlink(missing_ok=True) + + print(f"TPM wrapping key created and persisted at handle 0x{persistent_handle:08X}") + return { + "tpm_version": "2.0", + "persistent_handle": persistent_handle, + "algorithm": "RSA-2048-OAEP-SHA256", + "public_key_pem": pubkey_pem, + "public_key_sha256": state["public_key_sha256"], + } + + +def tpm2_certify_wrapkey(root: Path, tpm: TpmBackend | None = None) -> dict[str, Any]: + """Use TPM2_Certify to cryptographically prove the wrapping key's attributes. + + Creates a temporary signing key inside the TPM, then asks the TPM to sign + a TPMS_ATTEST structure that certifies: + - The wrapping key is TPM-resident (not a software-generated key) + - The wrapping key was created with the claimed attributes + (fixedtpm | sensitivedataorigin | decrypt | …) + + The server can verify this attestation signature to confirm the key + really lives inside a TPM — something a software-only key cannot fake. + + LIMITATION: Without a full TPM remote-attestation chain (EK → AK → certify), + a software TPM emulator could still forge this. For production, combine + this with a trusted provisioning environment or TPM EK certificate + verification, or pre-provision TPM keys at the factory. + """ + tpm = tpm or _TPM + state = tpm2_try_load_state(root) + if state is None: + raise SystemExit("No TPM state found. Run tpm2-setup first.") + + tpm_dir = root / "config" / "tpm" + handle = state["persistent_handle"] + + # We create a fresh primary key + signing key each time because the + # certifying key is temporary — only its public key is exported for + # the server to verify the attestation signature. + cert_primary_ctx = tpm_dir / "_cert_primary.ctx" + sign_pub = tpm_dir / "_cert_sign.pub" + sign_priv = tpm_dir / "_cert_sign.priv" + sign_ctx = tpm_dir / "_cert_sign.ctx" + attest_path = tpm_dir / "wrapkey.attest" + sig_path = tpm_dir / "wrapkey.attest.sig" + + print("Creating TPM attestation for wrapping key …") + + tpm.run("tpm2_createprimary", "-C", "o", + "-g", "sha256", "-G", "rsa", + "-c", str(cert_primary_ctx)) + tpm.flush_transients() + + # Create a signing-only RSA key to act as the certifying key. + # Attributes mirror the wrapping key: fixedtpm | fixedparent | + # sensitivedataorigin | userwithauth | sign. + tpm.run("tpm2_create", "-C", str(cert_primary_ctx), + "-g", "sha256", "-G", "rsa2048:rsassa:null", + "-a", "fixedtpm|fixedparent|sensitivedataorigin|userwithauth|sign", + "-u", str(sign_pub), + "-r", str(sign_priv)) + + tpm.flush_transients() + + tpm.run("tpm2_load", "-C", str(cert_primary_ctx), + "-u", str(sign_pub), "-r", str(sign_priv), + "-c", str(sign_ctx)) + + tpm.flush_transients() + + # Export the certifying key's public key so the server can verify + # the attestation signature. Same temp-file requirement as the + # wrapping key PEM (see read_public_key_pem). + sign_pub_pem_path = tpm_dir / "_cert_sign_pub.pem" + tpm.run("tpm2_readpublic", "-c", str(sign_ctx), "-f", "pem", + "-o", str(sign_pub_pem_path)) + sign_pub_pem = sign_pub_pem_path.read_text(encoding="utf-8").strip() + sign_pub_pem_path.unlink(missing_ok=True) + + # TPM2_Certify: the TPM signs a TPMS_ATTEST structure that binds the + # wrapping key's public area (including its attributes) to this TPM. + # Use long option names to be unambiguous — short flag -C vs -c + # conventions have changed across tpm2-tools versions. + # --certifiedkey-context = the object whose properties are being + # certified (the wrapping key) + # --signingkey-context = the key that signs the attestation + # (the temporary certifying key) + tpm.run("tpm2_certify", + "--certifiedkey-context", f"0x{handle:08X}", + "--signingkey-context", str(sign_ctx), + "-g", "sha256", + "--attestation", str(attest_path), + "--signature", str(sig_path), + "--format", "tss") + + # Export the wrapping key's raw public area (TPMT_PUBLIC) so the server + # can compute its TPM Name and verify it matches the Name embedded in + # the attestation. This proves the attestation is about *this* key, + # not some unrelated key. + # + # tpm2_readpublic -o writes TPM2B_PUBLIC (UINT16 size + TPMT_PUBLIC). + # The TPM Name = SHA-256(marshaled TPMT_PUBLIC), so we strip the 2-byte + # size prefix before sending. + pub_raw_path = tpm_dir / "_wrapkey_tpmt_public.bin" + tpm.run("tpm2_readpublic", "-c", f"0x{handle:08X}", + "-o", str(pub_raw_path)) + tpm2b_public = pub_raw_path.read_bytes() + # Strip the 2-byte UINT16 size prefix to get TPMT_PUBLIC + tpmt_public = tpm2b_public[2:] + pub_raw_path.unlink(missing_ok=True) + + # Try to parse the attestation blob to JSON for human-readable + # verification on the server side. + try: + attest_json_str = tpm.run("tpm2_print", "-t", "TPMS_ATTEST", str(attest_path)) + attest_data = json.loads(attest_json_str) + except (SystemExit, json.JSONDecodeError): + attest_data = None + + attestation_b64 = base64.b64encode(attest_path.read_bytes()).decode("ascii") + + # tpm2_certify -o outputs the signature in TPMT_SIGNATURE format, not + # as a bare RSA signature. We must strip the TPMT_SIGNATURE header so + # the server can verify it with standard cryptography APIs. + # + # TPMT_SIGNATURE (TPM_ALG_RSASSA = 0x0014): + # UINT16 sigAlg (2 bytes) — 0x0014 + # UINT16 hashAlg (2 bytes) — 0x000B = SHA-256 + # UINT16 sigSize (2 bytes) — e.g. 256 for RSA-2048 + # BYTE[sigSize] — the actual RSASSA-PKCS1-v1_5 signature + _sig_raw = sig_path.read_bytes() + if len(_sig_raw) < 6: + raise SystemExit( + f"tpm2_certify signature is too short ({len(_sig_raw)} bytes) — " + "expected TPMT_SIGNATURE format with at least a 6-byte header" + ) + _sig_alg = int.from_bytes(_sig_raw[0:2], "big") + if _sig_alg != 0x0014: + raise SystemExit( + f"tpm2_certify signature has unexpected algorithm 0x{_sig_alg:04X} — " + "expected TPM_ALG_RSASSA (0x0014)" + ) + _hash_alg = int.from_bytes(_sig_raw[2:4], "big") + if _hash_alg != 0x000B: + raise SystemExit( + f"tpm2_certify signature has unexpected hash algorithm 0x{_hash_alg:04X} — " + "expected TPM_ALG_SHA256 (0x000B)" + ) + _sig_size = int.from_bytes(_sig_raw[4:6], "big") + if len(_sig_raw) != 6 + _sig_size: + raise SystemExit( + f"tpm2_certify signature size mismatch: header says {_sig_size} bytes, " + f"but file is {len(_sig_raw)} bytes (expected {6 + _sig_size})" + ) + _rsa_sig = _sig_raw[6:] + signature_b64 = base64.b64encode(_rsa_sig).decode("ascii") + + # Clean up temporary TPM objects — the attestation + signature files + # are plaintext blobs we keep on disk. + for p in [cert_primary_ctx, sign_pub, sign_priv, sign_ctx]: + p.unlink(missing_ok=True) + + result: dict[str, Any] = { + "attestation_blob_b64": attestation_b64, + "attestation_signature_b64": signature_b64, + "certifying_key_pem": sign_pub_pem, + "certifying_key_sha256": sha256_hex(sign_pub_pem.encode()), + # Raw TPMT_PUBLIC of the wrapping key — server computes + # SHA-256 of this to get the TPM Name and cross-checks it + # against the Name inside the attestation blob. + "wrapkey_tpmt_public_b64": base64.b64encode(tpmt_public).decode("ascii"), + } + if attest_data is not None: + result["attestation_json"] = attest_data + + print("TPM attestation created — wrapping key attributes are now cryptographically provable.") + return result + + +def tpm2_unwrap_master_key(root: Path, wrapped_key_b64: str, + tpm: TpmBackend | None = None) -> bytes: + """Ask the TPM to RSA-OAEP-decrypt the wrapped K_master. + + The TPM performs the decryption internally using the non-exportable + private key. The plaintext K_master is returned to us — the private + key itself never leaves the TPM chip. + """ + tpm = tpm or _TPM + state = tpm2_try_load_state(root) + if state is None: + raise SystemExit( + "TPM-wrapped asset key found (v3 envelope) but no TPM state " + "exists on this machine. Run 'plantai_client.py tpm2-setup' first." + ) + + handle = state["persistent_handle"] + # Use /dev/shm (tmpfs) so the plaintext K_master never touches a + # persistent disk. Even if the finally block is skipped, a reboot + # wipes /dev/shm completely. PID in the filename prevents collisions. + ct_path = Path(f"/dev/shm/_unwrap_ct_{os.getpid()}.bin") + pt_path = Path(f"/dev/shm/_unwrap_pt_{os.getpid()}.bin") + ct_path.write_bytes(base64.b64decode(wrapped_key_b64)) + + try: + tpm.run("tpm2_rsadecrypt", + "-c", f"0x{handle:08X}", + "-s", "oaep", + "-o", str(pt_path), + str(ct_path)) + return pt_path.read_bytes() + finally: + ct_path.unlink(missing_ok=True) + pt_path.unlink(missing_ok=True) + + +# --------------------------------------------------------------------------- +# fingerprint collection, hardened against volatile/virtual components +# --------------------------------------------------------------------------- +def _is_physical_nic(iface: Path) -> bool: + # Physical NICs have a real PCI/USB device behind them; virtual + # interfaces created by docker/libvirt/veth/tun/bridges do not. + return (iface / "device").exists() + + +def _collect_network() -> list[dict[str, str]]: + rows = [] + for iface in sorted(Path("/sys/class/net").glob("*")): + name = iface.name + if name == "lo": + continue + if not _is_physical_nic(iface): + continue + mac = read_text(iface / "address") + if mac and mac != "00:00:00:00:00:00": + rows.append({"iface": name, "mac": mac.lower()}) + return rows + + +def _is_removable_disk(name: str) -> bool: + flag = read_text(Path(f"/sys/block/{name}/removable")) + return flag == "1" + + +def _collect_disks() -> list[dict[str, str]]: + rows = [] + lsblk = run(["lsblk", "-ndo", "NAME,SERIAL,MODEL,TYPE"]) + for line in lsblk.splitlines(): + cols = line.split() + if len(cols) >= 2 and cols[-1] == "disk": + name = cols[0] + if _is_removable_disk(name): + continue + rows.append({ + "name": name, + "serial": cols[1] if len(cols) >= 3 else "", + "model": " ".join(cols[2:-1]) if len(cols) > 3 else "", + }) + return rows + + +def collect_fingerprint(device_id: str, root: Path | None = None, + tpm: TpmBackend | None = None) -> dict[str, Any]: + tpm = tpm or _TPM + dmi_root = Path("/sys/class/dmi/id") + cpu_model = "" + for line in Path("/proc/cpuinfo").read_text(errors="ignore").splitlines(): + if line.lower().startswith("model name"): + cpu_model = line.split(":", 1)[-1].strip() + break + + gpu_rows = [] + gpu_out = run(["nvidia-smi", "--query-gpu=name,uuid", "--format=csv,noheader"]) + for line in gpu_out.splitlines(): + parts = [p.strip() for p in line.split(",", 1)] + if len(parts) == 2: + gpu_rows.append({"name": parts[0], "uuid": parts[1]}) + + disk_rows = _collect_disks() + network_rows = _collect_network() + + product_uuid = read_text(dmi_root / "product_uuid") + board_serial = read_text(dmi_root / "board_serial") + + fp = { + "device_id": device_id, + "collected_at": now_iso(), + "hostname": socket.gethostname(), + "dmi": { + "product_uuid": product_uuid, + "board_serial": board_serial, + "bios_vendor": read_text(dmi_root / "bios_vendor"), + "bios_version": read_text(dmi_root / "bios_version"), + }, + "cpu": {"model": cpu_model}, + "gpu": gpu_rows, + "disk": disk_rows, + "network": network_rows, + } + + # Stable "core identity" anchor used ONLY for cryptographic key wrapping. + # Picked from the most stable identifier actually available on this box, + # in order of preference. This deliberately excludes anything that can + # change without the physical machine changing (NICs, USB disks, hostname). + bad_uuid_markers = ("", "00000000-0000-0000-0000-000000000000", "Not Specified") + if product_uuid and product_uuid not in bad_uuid_markers: + anchor_type, anchor_value = "dmi_product_uuid", product_uuid + elif board_serial and board_serial not in bad_uuid_markers: + anchor_type, anchor_value = "dmi_board_serial", board_serial + elif disk_rows and any(d["serial"] for d in disk_rows): + anchor_type = "disk_serial_set" + anchor_value = "|".join(sorted(d["serial"] for d in disk_rows if d["serial"])) + else: + # Weak fallback: still deterministic, but flagged so the server can + # decide whether to accept it (e.g. bare-metal-less VM with no DMI). + anchor_type, anchor_value = "hostname_weak", socket.gethostname() + + fp["core_id_anchor"] = anchor_type + fp["core_id_hash"] = sha256_hex(canonical_bytes({"type": anchor_type, "value": anchor_value})) + + fp["fingerprint_material"] = { + "hostname": fp["hostname"], + "dmi": fp["dmi"], + "cpu_model": cpu_model, + "gpu_uuid": sorted(g.get("uuid", "") for g in gpu_rows if g.get("uuid")), + "disk_serial": sorted(d.get("serial", "") for d in disk_rows if d.get("serial")), + "network_mac": sorted(n.get("mac", "") for n in network_rows if n.get("mac")), + } + + # Attach TPM public key if available. This tells the vendor "wrap K_master + # with this public key so only my TPM can unwrap it." + if root is not None: + tpm_state = tpm2_try_load_state(root) + if tpm_state is not None: + try: + pubkey_pem = read_public_key_pem(tpm_state["persistent_handle"], tpm) + fp["tpm_public_key"] = { + "tpm_version": "2.0", + "persistent_handle": tpm_state["persistent_handle"], + "algorithm": "RSA-2048-OAEP-SHA256", + "public_key_pem": pubkey_pem, + "public_key_sha256": sha256_hex(pubkey_pem.encode()), + } + # Include attestation data so the server can cryptographically + # verify the key is truly TPM-resident, not a software fake. + tpm_attestation = tpm_state.get("attestation") + if tpm_attestation: + fp["tpm_public_key"]["attestation"] = tpm_attestation + else: + print("WARNING: No TPM attestation found. The server will not") + print("be able to verify this key is truly hardware-backed.") + print("Re-run 'plantai_client.py tpm2-setup --root ' to create one.") + except SystemExit: + pass # TPM unavailable at fingerprint time — just omit + + return fp + + +def match_score(expected: dict[str, Any], current: dict[str, Any]) -> int: + score = 0 + if (expected.get("dmi") or {}).get("product_uuid") and (expected.get("dmi") or {}).get("product_uuid") == (current.get("dmi") or {}).get("product_uuid"): + score += 30 + expected_gpu = set(expected.get("gpu_uuid", [])) + current_gpu = set(current.get("gpu_uuid", [])) + if expected_gpu and expected_gpu & current_gpu: + score += 25 + expected_disk = set(expected.get("disk_serial", [])) + current_disk = set(current.get("disk_serial", [])) + if expected_disk and expected_disk & current_disk: + score += 20 + if expected.get("dmi") and expected.get("dmi") == current.get("dmi"): + score += 10 + expected_mac = set(expected.get("network_mac", [])) + current_mac = set(current.get("network_mac", [])) + if expected_mac and expected_mac & current_mac: + score += 10 + if expected.get("hostname") and expected.get("hostname") == current.get("hostname"): + score += 5 + return min(score, 100) + + +# --------------------------------------------------------------------------- +# verification / key unwrap / decrypt (public-key + already-issued files only) +# --------------------------------------------------------------------------- +def load_public(path: Path) -> Ed25519PublicKey: + return serialization.load_pem_public_key(path.read_bytes()) + + +def verify_payload(public_key_path: Path, payload: dict[str, Any]) -> None: + signature = base64.b64decode(payload.get("signature", "")) + unsigned = dict(payload) + unsigned.pop("signature", None) + load_public(public_key_path).verify(signature, canonical_bytes(unsigned)) + + +def derive_wrap_key(anchor_hash: str, salt: bytes) -> bytes: + return HKDF( + algorithm=hashes.SHA256(), + length=32, + salt=salt, + info=b"PlantAI K_master wrap v2", + ).derive(anchor_hash.encode("ascii")) + + +def unwrap_master_key(asset_key_path: Path, current_fp: dict[str, Any], + root: Path) -> bytes: + """Unwrap K_master. Supports two envelope versions: + + v3 — TPM-based (hardware-bound): + K_master is RSA-OAEP-encrypted to a non-exportable key inside the + TPM. Only this specific TPM chip can decrypt it. + + v2 — Legacy HKDF (WEAK, no hardware binding): + wrap_key = HKDF(core_id_hash, salt) where both parameters are + stored in this file. Anyone with the file can recompute the key. + """ + envelope = load_json(asset_key_path) + version = envelope.get("version", 1) + + # ---- v3: TPM-based unwrapping (REAL hardware binding) ---- + if version == 3: + return tpm2_unwrap_master_key( + root, + envelope["wrapped_key_b64"], + ) + + # ---- v2: Legacy HKDF-based unwrapping (WEAK) ---- + print("WARNING: This license uses legacy HKDF key wrapping — this provides") + print("NO hardware binding. Anyone with the asset.key.enc file can derive") + print("the unwrap key. If this machine has a TPM 2.0, run:") + print(" plantai_client.py tpm2-setup --root ") + print("and request a new asset.key.enc from the vendor.") + print() + + if envelope["core_id_hash"] != current_fp["core_id_hash"]: + raise SystemExit( + "Hardware core identity has changed since this license was issued " + f"(anchor={envelope.get('core_id_anchor')}). This machine cannot unwrap " + "the asset key. Contact vendor support for a license migration." + ) + salt = base64.b64decode(envelope["salt"]) + nonce = base64.b64decode(envelope["nonce"]) + ciphertext = base64.b64decode(envelope["ciphertext"]) + wrap_key = derive_wrap_key(envelope["core_id_hash"], salt) + return AESGCM(wrap_key).decrypt(nonce, ciphertext, envelope["core_id_hash"].encode("ascii")) + + +def _safe_extract(tf: tarfile.TarFile, output_dir: Path) -> None: + """Extract tar members safely: reject absolute paths, parent-dir + traversal, and symlinks / hardlinks / device nodes.""" + resolved_base = output_dir.resolve() + for member in tf.getmembers(): + # Reject absolute paths and .. traversal + if member.name.startswith("/") or ".." in Path(member.name).parts: + raise SystemExit( + f"tar member '{member.name}' has an unsafe path" + ) + target = (resolved_base / member.name).resolve() + if not str(target).startswith(str(resolved_base) + "/") and target != resolved_base: + raise SystemExit( + f"tar member '{member.name}' escapes output directory" + ) + # Reject anything that isn't a regular file or directory + if not (member.isreg() or member.isdir()): + raise SystemExit( + f"tar member '{member.name}' is not a regular file or directory" + ) + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + else: + target.parent.mkdir(parents=True, exist_ok=True) + with tf.extractfile(member) as src, open(target, "wb") as dst: + shutil.copyfileobj(src, dst) + + +def decrypt_asset(root: Path, master_key: bytes, enc_path: Path, output_dir: Path, + keep_compressed_tmp: bool = False, + allow_missing_manifest: bool = False) -> None: + name = enc_path.name.removesuffix(".enc") + + # --- verify asset manifest before touching the ciphertext --- + pubkey_path = root / "config" / "supplier_public_key.pem" + license_data = load_json(root / "config" / "license.json") + manifest_path = enc_path.parent / f"{name}.manifest.json" + + if manifest_path.exists(): + manifest = load_json(manifest_path) + verify_payload(pubkey_path, manifest) + + # release_version binding + lic_release = license_data.get("asset_policy", {}).get("release_version", "") + mf_release = manifest.get("release_version", "") + if lic_release and mf_release != lic_release: + raise SystemExit( + f"manifest release_version '{mf_release}' does not match " + f"license asset_policy.release_version '{lic_release}' for asset '{name}'" + ) + + # Asset package names ("models", "prompts") are NOT the same as + # business modules ("identify", "rnaseq_qc", …). Use asset_policy. + # allowed_assets to control which encrypted packages a license permits. + allowed_assets: list[str] = license_data.get("asset_policy", {}).get("allowed_assets", []) + if allowed_assets and name not in allowed_assets: + raise SystemExit( + f"asset '{name}' is not in license asset_policy.allowed_assets" + ) + else: + if allow_missing_manifest: + print(f"WARNING: no manifest found for '{name}', skipping integrity check " + f"(--allow-missing-manifest is set)") + else: + raise SystemExit( + f"No manifest found for '{name}' — every asset must have a signed " + f"manifest. If this is a test/dev environment, pass " + f"--allow-missing-manifest to decrypt-all." + ) + + raw = enc_path.read_bytes() + nonce_b64, ciphertext = raw.split(b"\n", 1) + + # verify cipher_hash before decrypt (fast, detects corruption / tampering) + if manifest_path.exists(): + expected_cipher_hash = manifest.get("cipher_hash", "") + if expected_cipher_hash and sha256_hex(ciphertext) != expected_cipher_hash: + raise SystemExit( + f"cipher_hash mismatch for '{name}' — the .enc file may be " + "corrupted or tampered with" + ) + + plaintext = AESGCM(master_key).decrypt(base64.b64decode(nonce_b64), ciphertext, name.encode("utf-8")) + + # verify content_hash after decrypt (guards against key misuse / corruption) + if manifest_path.exists(): + expected_content_hash = manifest.get("content_hash", "") + if expected_content_hash and sha256_hex(plaintext) != expected_content_hash: + raise SystemExit( + f"content_hash mismatch for '{name}' — the decrypted asset does " + "not match what the vendor signed" + ) + output_dir.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory() as tmp: + archive_path = Path(tmp) / f"{name}.tar.zst" + archive_path.write_bytes(plaintext) + if shutil.which("zstd"): + tar_path = Path(tmp) / f"{name}.tar" + subprocess.check_call(["zstd", "-q", "-d", "-f", str(archive_path), "-o", str(tar_path)]) + else: + tar_path = archive_path + with tarfile.open(tar_path, "r") as tf: + _safe_extract(tf, output_dir) + + +def verify_install(args: argparse.Namespace) -> dict[str, Any]: + root = Path(args.root) + license_data = load_json(root / "config" / "license.json") + try: + verify_payload(root / "config" / "supplier_public_key.pem", license_data) + except InvalidSignature as exc: + raise SystemExit("license signature verification failed") from exc + + current_fp = collect_fingerprint(license_data["device_id"], root) + hw = license_data["hardware_policy"] + score = match_score(hw["fingerprint_snapshot"], current_fp["fingerprint_material"]) + min_score = int(hw["min_match_score"]) + warn_score = int(hw.get("allow_warning_score", min_score)) + + if score < min_score: + raise SystemExit(f"hardware match score {score} is below required {min_score}") + + # Verify that asset.key.enc matches the hash baked into the license. + # This prevents an attacker from swapping in a different envelope. + # For license_version >= 2.0, asset_key_sha256 is REQUIRED — if it's + # missing the license was not properly bound (e.g. operator forgot + # --asset-key-envelope when running issue-license). + asset_key_path = root / "config" / "asset.key.enc" + expected_ak_hash = license_data.get("asset_policy", {}).get("asset_key_sha256", "") + if not expected_ak_hash: + raise SystemExit( + "license is missing asset_policy.asset_key_sha256 — the license " + "was not cryptographically bound to an asset key envelope.\n" + "The vendor must re-issue the license with --asset-key-envelope " + "pointing to the same asset.key.enc that ships with this license." + ) + actual_ak_hash = sha256_hex(asset_key_path.read_bytes()) + if actual_ak_hash != expected_ak_hash: + raise SystemExit( + f"asset.key.enc hash does not match license asset_key_sha256.\n" + f" Expected: {expected_ak_hash}\n" + f" Actual: {actual_ak_hash}\n" + f"The license and asset key may have been tampered with or are " + f"from different issuance runs." + ) + + # Verify that the TPM key used for wrapping matches the one in the license. + # This prevents a TPM-key downgrade between license issuance and runtime. + expected_tpm_hash = license_data.get("asset_policy", {}).get("tpm_public_key_sha256", "") + if expected_tpm_hash: + current_tpm = current_fp.get("tpm_public_key") + if not current_tpm: + raise SystemExit( + "License requires TPM-based protection (tpm_public_key_sha256 is set) " + "but this machine has no TPM key configured. Run tpm2-setup." + ) + if current_tpm["public_key_sha256"] != expected_tpm_hash: + raise SystemExit( + f"TPM public key has changed since license issuance.\n" + f" License: {expected_tpm_hash}\n" + f" Current: {current_tpm['public_key_sha256']}\n" + f"Was tpm2-setup re-run after the license was issued?" + ) + + result = {"ok": True, "hardware_match_score": score, "root": str(root)} + if score < warn_score: + result["warning"] = f"hardware match score {score} is below the comfortable threshold {warn_score}; license still valid but drifting" + + # Enforce min_wrap_version before unwrap. If the license says v3 is + # required (TPM-based), reject a v2 (HKDF) envelope even if a v2 + # unwrap would technically succeed. This prevents downgrade attacks + # where an attacker replaces asset.key.enc with a v2 envelope. + min_wrap = license_data.get("asset_policy", {}).get("min_wrap_version", 0) + envelope = load_json(asset_key_path) + env_version = envelope.get("version", 1) + if env_version < min_wrap: + raise SystemExit( + f"asset.key.enc wrap version ({env_version}) is below the " + f"minimum required by this license ({min_wrap}).\n" + f"The license requires v{min_wrap} (TPM hardware-bound) but " + f"the asset key envelope is v{env_version}.\n" + f"Contact the vendor for a properly wrapped asset key." + ) + + master_key = unwrap_master_key(asset_key_path, current_fp, root) + result["asset_key_ready"] = True + log_audit(root, {"event": "verify-install", **result}) + print(json.dumps(result, ensure_ascii=False)) + return {"license_data": license_data, "master_key": master_key, "result": result} + + +def decrypt_all(root: Path, output: Path, + allow_missing_manifest: bool = False) -> None: + """Decrypt every *.enc under /assets into output. Pure logic — the + CLI wrapper and the launcher both call this.""" + state = verify_install(argparse.Namespace(root=str(root))) + master_key = state["master_key"] + output_dir = Path(output) + for enc in sorted(Path(root, "assets").glob("*.enc")): + decrypt_asset(root, master_key, enc, output_dir, + allow_missing_manifest=allow_missing_manifest) + print(f"decrypted {enc.name} -> {output_dir}") diff --git a/tooling/legacy_docker_packager/Dockerfile.plantai b/tooling/legacy_docker_packager/Dockerfile.plantai new file mode 100644 index 0000000..7eafceb --- /dev/null +++ b/tooling/legacy_docker_packager/Dockerfile.plantai @@ -0,0 +1,52 @@ +ARG BASE_IMAGE=python:3.11-bookworm +FROM ${BASE_IMAGE} + +ARG PIP_INDEX_URL=https://pypi.org/simple +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_NO_CACHE_DIR=1 \ + PLANTAI_PROJECT_ROOT=/opt/plantai \ + BBAV5_PROJECT_ROOT=/opt/plantai \ + PLANTAI_RAG_ROOT=/opt/rag \ + GENE_FAMILY_ROOT=/opt/qa \ + PYTHONPATH=/opt/plantai + +# Identify requires the HMMER and NCBI BLAST command-line tools. This is the +# minimum native runtime, not the old allenv/Conda environment. +RUN set -eux; \ + if [ -f /etc/apt/sources.list.d/debian.sources ]; then \ + sed -i 's|http://deb.debian.org|https://deb.debian.org|g; s|http://security.debian.org|https://security.debian.org|g' /etc/apt/sources.list.d/debian.sources; \ + fi; \ + if [ -f /etc/apt/sources.list ]; then \ + sed -i 's|http://deb.debian.org|https://deb.debian.org|g; s|http://security.debian.org|https://security.debian.org|g' /etc/apt/sources.list; \ + fi; \ + apt-get -o Acquire::Retries=5 -o Acquire::https::Timeout=60 update; \ + apt-get install -y --no-install-recommends hmmer ncbi-blast+; \ + rm -rf /var/lib/apt/lists/* + +COPY requirements/plantai.txt /tmp/requirements.txt +RUN python -m pip install --no-cache-dir -r /tmp/requirements.txt \ + && rm -f /tmp/requirements.txt + +COPY work/context/PlantAI/ /opt/plantai/ +RUN test -f /opt/plantai/.hardened-build.json \ + && python -c "import sys; assert sys.version_info[:2] == (3, 11)" \ + && test "$(find /opt/plantai -type f -name '*.so' | wc -l)" -gt 0 \ + && command -v hmmscan \ + && command -v hmmsearch \ + && command -v blastp \ + && command -v makeblastdb \ + && test -s /opt/plantai/data/hmm/Pfam-A.hmm \ + && test -s /opt/plantai/data/hmm/Pfam-A.hmm.h3f \ + && test -s /opt/plantai/data/hmm/Pfam-A.hmm.h3i \ + && test -s /opt/plantai/data/hmm/Pfam-A.hmm.h3m \ + && test -s /opt/plantai/data/hmm/Pfam-A.hmm.h3p \ + && test -s /opt/plantai/data/rag_db/pfam_mapping.json \ + && python -c "import json; p=json.load(open('/opt/plantai/data/rag_db/pfam_mapping.json')); assert p['SPL'] == 'PF03110'" \ + && mkdir -p /runtime/input /runtime/workspace /runtime/data /runtime/logs \ + /runtime/service_data/memory_storage /runtime/service_data/cost_logs \ + /runtime/service_data/reproducibility + +WORKDIR /opt/plantai +CMD ["python", "apps/web/vue3/main.py"] diff --git a/tooling/legacy_docker_packager/Dockerfile.qa b/tooling/legacy_docker_packager/Dockerfile.qa new file mode 100644 index 0000000..15ba005 --- /dev/null +++ b/tooling/legacy_docker_packager/Dockerfile.qa @@ -0,0 +1,27 @@ +ARG BASE_IMAGE=python:3.12-bookworm +FROM ${BASE_IMAGE} + +ARG PIP_INDEX_URL=https://pypi.org/simple +ARG TORCH_INDEX_URL=https://download.pytorch.org/whl/cpu +ARG TORCH_CPU_VERSION=2.13.0+cpu +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_NO_CACHE_DIR=1 \ + PYTHONPATH=/opt/qa \ + GENE_FAMILY_ROOT=/opt/qa + +COPY requirements/qa.txt /tmp/requirements.txt +RUN python -m pip install --no-cache-dir \ + --index-url "$TORCH_INDEX_URL" "torch==$TORCH_CPU_VERSION" \ + && python -m pip install --no-cache-dir -r /tmp/requirements.txt \ + && rm -f /tmp/requirements.txt + +COPY work/context/QA/ /opt/qa/ +RUN test -f /opt/qa/.hardened-build.json \ + && python -c "import sys; assert sys.version_info[:2] == (3, 12)" \ + && python -c "import torch; assert torch.__version__ == '$TORCH_CPU_VERSION'; assert torch.version.cuda is None" \ + && test "$(find /opt/qa -type f -name '*.so' | wc -l)" -gt 0 + +WORKDIR /opt/qa +CMD ["python", "-m", "uvicorn", "scripts.qa_v1_1_73_server:app", "--host", "0.0.0.0", "--port", "7925"] diff --git a/tooling/legacy_docker_packager/Dockerfile.rag b/tooling/legacy_docker_packager/Dockerfile.rag new file mode 100644 index 0000000..dbed091 --- /dev/null +++ b/tooling/legacy_docker_packager/Dockerfile.rag @@ -0,0 +1,26 @@ +ARG BASE_IMAGE=python:3.11-bookworm +FROM ${BASE_IMAGE} + +ARG PIP_INDEX_URL=https://pypi.org/simple +ARG TORCH_INDEX_URL=https://download.pytorch.org/whl/cpu +ARG TORCH_CPU_VERSION=2.13.0+cpu +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_NO_CACHE_DIR=1 \ + PYTHONPATH=/opt/rag:/opt/rag/src + +COPY requirements/rag.txt /tmp/requirements.txt +RUN python -m pip install --no-cache-dir \ + --index-url "$TORCH_INDEX_URL" "torch==$TORCH_CPU_VERSION" \ + && python -m pip install --no-cache-dir -r /tmp/requirements.txt \ + && rm -f /tmp/requirements.txt + +COPY work/context/RAG/ /opt/rag/ +RUN test -f /opt/rag/.hardened-build.json \ + && python -c "import sys; assert sys.version_info[:2] == (3, 11)" \ + && python -c "import torch; assert torch.__version__ == '$TORCH_CPU_VERSION'; assert torch.version.cuda is None" \ + && test "$(find /opt/rag -type f -name '*.so' | wc -l)" -gt 0 + +WORKDIR /opt/rag +CMD ["python", "-m", "uvicorn", "api.app:app", "--host", "0.0.0.0", "--port", "7900", "--workers", "2"] diff --git a/tooling/legacy_docker_packager/README.md b/tooling/legacy_docker_packager/README.md new file mode 100644 index 0000000..cfc30c2 --- /dev/null +++ b/tooling/legacy_docker_packager/README.md @@ -0,0 +1,96 @@ +# PlantAI / RAG / RAG-QA Docker 一键打包工具 + +本工具只封装已经完成阶段 A 加固的三个产物，不重新编译、不重新加密、不生成密钥、不签发许可证，也不启动现有服务。 + +## 当前构建策略 + +- 不构建或复制“全环境镜像”，镜像中不包含 `/opt/conda`、`test4py311`、`rnaseq`、`rnaR` 或 `gfpy312`。 +- 不复用 `plantai-allenv` 或任何既有 PlantAI 应用镜像。 +- PlantAI、RAG、RAG-QA 分别构建为三个独立镜像。 +- PlantAI/RAG 从官方 `python:3.11-bookworm` 开始，RAG-QA 从官方 `python:3.12-bookworm` 开始。 +- 正式构建固定使用 `docker pull`、`docker build --pull --no-cache`，确保重新解析基础镜像并从头安装运行时 Python 依赖。 +- PlantAI 只额外安装 Identify 必需的 `hmmer` 和 `ncbi-blast+`，不恢复旧 Conda/allenv。 +- RAG/QA 固定从 PyTorch 官方 CPU 索引安装 `torch==2.13.0+cpu`，不从通用 + `torch>=2.0` 解析并下载 CUDA 运行库。 +- Debian 源切换为 HTTPS，并结合 `BUILD_NETWORK=host`、下载重试与超时控制，避免此前 Docker bridge 访问 `deb.debian.org:80` 超时。 + +官方 Python 基础镜像仍然是 Docker 构建所需的干净操作系统/Python 起点，但它不是已有业务镜像，也不包含旧 PlantAI 文件。 + +## 一键使用 + +```bash +cd /home/xzq/5090/PlanAI-cooperate/hsj-workspace/encryption-test/server/protected_docker_packager +cp pack.env.example pack.env +vi pack.env +CHECK_ONLY=1 bash build_package.sh +bash build_package.sh +``` + +正式构建输出： + +```text +dist/plantai-protected-docker-1.0.0-linux-amd64.tar.zst +dist/plantai-protected-docker-1.0.0-linux-amd64.tar.zst.sha256 +``` + +同名包已存在时默认拒绝覆盖；确认只覆盖本工具自己的旧输出后使用： + +```bash +OVERWRITE=1 bash build_package.sh +``` + +## 构建流程 + +1. 校验三个 `.hardened-build.json` 与 CPython ABI。 +2. 建立隔离 staging，排除输入、历史输出、日志、缓存、Git、测试、密钥、许可证和符号链接，并加入最小离线 Pfam 家族映射。 +3. 删除已有同名 Cython `.so` 的 `.py` 明文副本。 +4. 拉取两个干净的官方 Python 基础镜像。 +5. 使用三个 Dockerfile 和三个独立依赖清单执行无缓存构建。 +6. 验证三个镜像的 Python 版本、加固标记、`.so` 和敏感文件名；额外验证 PlantAI + 的 `hmmscan`、`hmmsearch`、`blastp`、`makeblastdb`、完整 Pfam-A 数据库与离线家族映射， + 并验证 RAG/QA 使用固定 CPU-only Torch。 +7. 导出三个应用镜像；按配置刷新并导出 Redis/Qdrant/Neo4j 上游镜像。 +8. 组装客户运行包并生成 SHA-256。 + +`CHECK_ONLY=1` 只验证路径、ABI、排除规则、staging、Docker 构建策略和 Compose，不拉取或构建镜像。 + +## 非全环境模式的边界 + +当前镜像只安装 Python 应用运行依赖。依赖以下专用环境或外部二进制的任务不会由本包提供： + +- RNA-seq 专用 Conda 环境； +- R 及 RNA-R 分析环境； +- gene-family 专用 Python/系统工具链； +- 其他系统级生信软件、数据库客户端或 GPU/CUDA 运行库。 + +例外是 Identify 的最小必要工具：`hmmer` 和 `ncbi-blast+` 已直接安装进 PlantAI 镜像。 +这不代表 RNA-seq、R 绘图或其他全环境已经恢复。 + +为避免已知家族在 RAG 数据暂不可用时直接失败，构建工具还会写入以下经过源码现有 +别名表核对的最小离线映射：SPL/SBP → `PF03110`、WRKY → `PF03106`、 +NAC → `PF02365`。这只修复这些已知家族的 Identify 入口，不代替完整知识库。 + +因此 Compose 的 `worker-medium` 只监听 `.medium` 队列，不监听 `.rnaseq`、`.rnar`、`.gfpy`。如果客户确实需要这些任务，应把相应工具做成独立受控镜像或挂载为客户现场依赖，不能再把整个 allenv 塞回应用镜像。 + +客户仍需单独准备 Ollama/模型、RAG-QA reranker 模型、Qdrant/Neo4j 业务数据、密码和 API 配置。 +当前模板中的 `NEO4J_PASSWORD=test1234` 仅用于本次测试，正式交付前必须替换。 + +## 客户侧安装 + +```bash +sha256sum -c plantai-protected-docker-1.0.0-linux-amd64.tar.zst.sha256 +tar --zstd -xf plantai-protected-docker-1.0.0-linux-amd64.tar.zst +cd plantai_protected_product +./install.sh +vi bbav5.env +./start.sh +./health-check.sh +``` + +`health-check.sh` 不再只判断端口是否能访问，还会拒绝“Qdrant collection 为空”或 +“Neo4j 节点数为 0”的伪健康状态。若在全新安装目录启动但未恢复业务数据，该检查 +会明确失败；这表示 RAG 数据未就绪，而不是容器未启动。 + +## 安全说明 + +Docker 是封装，不是新的密码学加密层。本包不做 TPM/硬件绑定，镜像持有者能够加载和检查镜像；Cython `.so` 只能提高逆向成本。不得把 `server/secrets`、私钥、主密钥、`license.json` 或真实 `.env` 放入本工具目录或构建上下文。Compose 默认只绑定 `127.0.0.1`。 diff --git a/tooling/legacy_docker_packager/bbav5.env.example b/tooling/legacy_docker_packager/bbav5.env.example new file mode 100644 index 0000000..e632a75 --- /dev/null +++ b/tooling/legacy_docker_packager/bbav5.env.example @@ -0,0 +1,39 @@ +# build_package.sh replaces these placeholders with the actual image tags. +PLANTAI_IMAGE=@@PLANTAI_IMAGE_REF@@ +RAG_IMAGE=@@RAG_IMAGE_REF@@ +QA_IMAGE=@@QA_IMAGE_REF@@ +REDIS_IMAGE=redis:7 +QDRANT_IMAGE=qdrant/qdrant:latest +NEO4J_IMAGE=neo4j:5.15.0-community + +BIND_ADDRESS=127.0.0.1 +PLANTAI_PORT=7880 +RAG_PORT=7900 +QA_PORT=7925 +QDRANT_PORT=16333 +QDRANT_GRPC_PORT=16334 +NEO4J_HTTP_PORT=17474 +NEO4J_BOLT_PORT=17687 + +# Test-only password requested for this package. Replace it for real delivery. +NEO4J_PASSWORD=test1234 + +BBAV5_QUEUE_PREFIX=plantai_protected +BBAV5_CELERY_LOGLEVEL=info +BBAV5_DEFAULT_CONCURRENCY=1 +BBAV5_HEAVY_CONCURRENCY=1 +BBAV5_MEDIUM_CONCURRENCY=2 +BBAV5_LIGHT_CONCURRENCY=2 + +V15_QDRANT_COLLECTION=plant_srt_direct_v1_bge_m3_20260702 +QA_V1_LLM_API_KEY=ollama-local +QA_V1_LLM_BASE_URL=http://host.docker.internal:11434/v1 +QA_V1_LLM_MODEL=qwen2.5:14b-instruct +V15_OLLAMA_EMBED_URL=http://host.docker.internal:11434/api/embed +V15_OLLAMA_EMBED_MODEL=bge-m3:567m + +ZHIPUAI_API_KEY= +MINIMAX_API_KEY= +MOONSHOT_API_KEY= +DEEPSEEK_API_KEY= +DASHSCOPE_API_KEY= diff --git a/tooling/legacy_docker_packager/build_package.sh b/tooling/legacy_docker_packager/build_package.sh new file mode 100755 index 0000000..0bdd5d5 --- /dev/null +++ b/tooling/legacy_docker_packager/build_package.sh @@ -0,0 +1,548 @@ +#!/usr/bin/env bash +# Build three independent runtime images from clean official Python bases. +set -euo pipefail +umask 027 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${PACK_ENV_FILE:-$SCRIPT_DIR/pack.env}" +if [[ ! -f "$ENV_FILE" ]]; then + ENV_FILE="$SCRIPT_DIR/pack.env.example" + echo "[config] pack.env not found; using pack.env.example" +fi +# shellcheck source=/dev/null +. "$ENV_FILE" + +: "${ARTIFACT_ROOT:?missing ARTIFACT_ROOT}" +: "${PLANTAI_ARTIFACT:?missing PLANTAI_ARTIFACT}" +: "${RAG_ARTIFACT:?missing RAG_ARTIFACT}" +: "${QA_ARTIFACT:?missing QA_ARTIFACT}" +: "${PLANTAI_BASE_IMAGE:?missing PLANTAI_BASE_IMAGE}" +: "${RAG_BASE_IMAGE:?missing RAG_BASE_IMAGE}" +: "${QA_BASE_IMAGE:?missing QA_BASE_IMAGE}" +: "${PLANTAI_IMAGE_REPOSITORY:?missing PLANTAI_IMAGE_REPOSITORY}" +: "${RAG_IMAGE_REPOSITORY:?missing RAG_IMAGE_REPOSITORY}" +: "${QA_IMAGE_REPOSITORY:?missing QA_IMAGE_REPOSITORY}" +: "${VERSION:?missing VERSION}" + +PLATFORM="${PLATFORM:-linux/amd64}" +BUILD_NETWORK="${BUILD_NETWORK:-host}" +PIP_INDEX_URL="${PIP_INDEX_URL:-https://pypi.org/simple}" +TORCH_INDEX_URL="${TORCH_INDEX_URL:-https://download.pytorch.org/whl/cpu}" +TORCH_CPU_VERSION="${TORCH_CPU_VERSION:-2.13.0+cpu}" +INCLUDE_INFRA_IMAGES="${INCLUDE_INFRA_IMAGES:-1}" +PULL_INFRA_IMAGES="${PULL_INFRA_IMAGES:-1}" +INCLUDE_PLANTAI_STATIC_DATA="${INCLUDE_PLANTAI_STATIC_DATA:-1}" +DROP_COMPILED_PY_SIBLINGS="${DROP_COMPILED_PY_SIBLINGS:-1}" +CHECK_ONLY="${CHECK_ONLY:-0}" +MIN_FREE_GB="${MIN_FREE_GB:-60}" +REDIS_IMAGE="${REDIS_IMAGE:-redis:7}" +QDRANT_IMAGE="${QDRANT_IMAGE:-qdrant/qdrant:latest}" +NEO4J_IMAGE="${NEO4J_IMAGE:-neo4j:5.15.0-community}" + +PLANTAI_IMAGE_REF="${PLANTAI_IMAGE_REPOSITORY}:${VERSION}" +RAG_IMAGE_REF="${RAG_IMAGE_REPOSITORY}:${VERSION}" +QA_IMAGE_REF="${QA_IMAGE_REPOSITORY}:${VERSION}" + +WORK_DIR="$SCRIPT_DIR/work" +CONTEXT_DIR="$WORK_DIR/context" +PACKAGE_STAGE="$WORK_DIR/package" +PRODUCT_NAME="plantai_protected_product" +PRODUCT_DIR="$PACKAGE_STAGE/$PRODUCT_NAME" +DIST_DIR="$SCRIPT_DIR/dist" +ARCHIVE="$DIST_DIR/plantai-protected-docker-${VERSION}-linux-amd64.tar.zst" +IMAGE_ARCHIVE_NAME="plantai-protected-images-${VERSION}.tar.zst" + +die() { + echo "[error] $*" >&2 + exit 1 +} + +[[ "$VERSION" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || + die "unsafe VERSION value: $VERSION" +for value in "$PLANTAI_IMAGE_REPOSITORY" "$RAG_IMAGE_REPOSITORY" "$QA_IMAGE_REPOSITORY"; do + [[ "$value" =~ ^[A-Za-z0-9][A-Za-z0-9._/-]*$ ]] || + die "unsafe version or image repository value: $value" +done +[[ "$PLATFORM" == "linux/amd64" ]] || + die "current hardened .so artifacts only support linux/amd64: $PLATFORM" +[[ "$BUILD_NETWORK" =~ ^[A-Za-z0-9_.:-]+$ ]] || + die "unsafe BUILD_NETWORK value: $BUILD_NETWORK" +for flag_name in INCLUDE_INFRA_IMAGES PULL_INFRA_IMAGES INCLUDE_PLANTAI_STATIC_DATA DROP_COMPILED_PY_SIBLINGS CHECK_ONLY; do + flag_value="${!flag_name}" + [[ "$flag_value" == "0" || "$flag_value" == "1" ]] || + die "$flag_name must be 0 or 1" +done +[[ "$MIN_FREE_GB" =~ ^[0-9]+$ && "$MIN_FREE_GB" -ge 20 ]] || + die "MIN_FREE_GB must be an integer >= 20" +[[ "$PLANTAI_BASE_IMAGE" == python:3.11-* ]] || + die "PLANTAI_BASE_IMAGE must be a clean official python:3.11-* image" +[[ "$RAG_BASE_IMAGE" == python:3.11-* ]] || + die "RAG_BASE_IMAGE must be a clean official python:3.11-* image" +[[ "$QA_BASE_IMAGE" == python:3.12-* ]] || + die "QA_BASE_IMAGE must be a clean official python:3.12-* image" +[[ "$PIP_INDEX_URL" =~ ^https:// ]] || + die "PIP_INDEX_URL must use HTTPS" +[[ "$TORCH_INDEX_URL" =~ ^https:// ]] || + die "TORCH_INDEX_URL must use HTTPS" +[[ "$TORCH_CPU_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+\+cpu$ ]] || + die "TORCH_CPU_VERSION must be an exact CPU wheel version such as 2.13.0+cpu" +[[ "$INCLUDE_PLANTAI_STATIC_DATA" == "1" ]] || + die "INCLUDE_PLANTAI_STATIC_DATA must be 1: Identify requires Pfam-A and the offline family mapping" + +for command_name in docker rsync python3 zstd tar sha256sum realpath find; do + command -v "$command_name" >/dev/null 2>&1 || die "missing command: $command_name" +done +docker info >/dev/null 2>&1 || die "Docker daemon is unavailable" +docker compose version >/dev/null 2>&1 || die "Docker Compose v2 is unavailable" + +for dockerfile in Dockerfile.plantai Dockerfile.rag Dockerfile.qa; do + [[ -f "$SCRIPT_DIR/$dockerfile" ]] || die "missing $dockerfile" + if grep -Eq 'plantai-allenv|FROM[[:space:]]+scratch' "$SCRIPT_DIR/$dockerfile"; then + die "$dockerfile contains a forbidden allenv/snapshot build path" + fi +done +grep -Eq 'apt-get install.*hmmer.*ncbi-blast\+' "$SCRIPT_DIR/Dockerfile.plantai" || + die "Dockerfile.plantai must install the minimum Identify native tools" +for dockerfile in Dockerfile.rag Dockerfile.qa; do + if grep -Eq 'apt-get' "$SCRIPT_DIR/$dockerfile"; then + die "$dockerfile unexpectedly installs system packages" + fi + grep -q 'TORCH_INDEX_URL' "$SCRIPT_DIR/$dockerfile" || + die "$dockerfile must install Torch from the dedicated CPU index" +done +if grep -Eq '^torch([<>=!~]|$)' "$SCRIPT_DIR/requirements/rag.txt" \ + "$SCRIPT_DIR/requirements/qa.txt"; then + die "Torch must not be resolved from the general requirements files" +fi +grep -q '^ARG BASE_IMAGE=python:3.11-' "$SCRIPT_DIR/Dockerfile.plantai" || + die "Dockerfile.plantai does not start from Python 3.11" +grep -q '^ARG BASE_IMAGE=python:3.11-' "$SCRIPT_DIR/Dockerfile.rag" || + die "Dockerfile.rag does not start from Python 3.11" +grep -q '^ARG BASE_IMAGE=python:3.12-' "$SCRIPT_DIR/Dockerfile.qa" || + die "Dockerfile.qa does not start from Python 3.12" + +require_free_space() { + local path="$1" label="$2" available_kb required_kb + available_kb="$(df -Pk "$path" | awk 'NR == 2 {print $4}')" + [[ "$available_kb" =~ ^[0-9]+$ ]] || die "cannot read free space: $path" + required_kb=$((MIN_FREE_GB * 1024 * 1024)) + (( available_kb >= required_kb )) || + die "$label needs at least ${MIN_FREE_GB}GB free: $path" + echo "[disk] $label free=$((available_kb / 1024 / 1024))GB required>=${MIN_FREE_GB}GB" +} + +ARTIFACT_ROOT="$(realpath "$ARTIFACT_ROOT")" +for artifact in "$PLANTAI_ARTIFACT" "$RAG_ARTIFACT" "$QA_ARTIFACT"; do + [[ -d "$artifact" ]] || die "stage A artifact does not exist: $artifact" + resolved="$(realpath "$artifact")" + case "$resolved" in + "$ARTIFACT_ROOT"/*) ;; + *) die "artifact must be inside ARTIFACT_ROOT: $resolved" ;; + esac + [[ -f "$resolved/.hardened-build.json" ]] || + die "artifact is missing .hardened-build.json: $resolved" +done + +python3 - "$PLANTAI_ARTIFACT" "$RAG_ARTIFACT" "$QA_ARTIFACT" <<'PY' +import json +import pathlib +import sys + +expected = [ + ("PlantAI", "3.11", ".cpython-311-"), + ("RAG", "3.11", ".cpython-311-"), + ("QA", "3.12", ".cpython-312-"), +] +for root_arg, (project, version_prefix, suffix_fragment) in zip(sys.argv[1:], expected): + root = pathlib.Path(root_arg) + marker = json.loads((root / ".hardened-build.json").read_text(encoding="utf-8")) + errors = [] + if marker.get("project") != project: + errors.append(f"project={marker.get('project')!r}") + if not str(marker.get("python_version", "")).startswith(version_prefix): + errors.append(f"python_version={marker.get('python_version')!r}") + if suffix_fragment not in str(marker.get("extension_suffix", "")): + errors.append(f"extension_suffix={marker.get('extension_suffix')!r}") + if int(marker.get("extension_count", 0)) <= 0: + errors.append("extension_count<=0") + if errors: + raise SystemExit(f"{root}: invalid build marker: {', '.join(errors)}") + print( + f"[marker] {project}: python={marker['python_version']} " + f"extensions={marker['extension_count']} tree={marker.get('tree_sha256', '')}" + ) +PY + +safe_reset_dir() { + local target="$1" resolved_parent + resolved_parent="$(realpath -m "$(dirname "$target")")" + [[ "$resolved_parent" == "$WORK_DIR" || "$resolved_parent" == "$PACKAGE_STAGE" ]] || + die "refusing to clean outside the tool work directory: $target" + rm -rf -- "$target" + mkdir -p "$target" +} + +mkdir -p "$WORK_DIR" "$DIST_DIR" +DOCKER_ROOT="$(docker info --format '{{.DockerRootDir}}')" +require_free_space "$DOCKER_ROOT" "Docker data directory" +require_free_space "$DIST_DIR" "release directory" +safe_reset_dir "$CONTEXT_DIR" +safe_reset_dir "$PACKAGE_STAGE" + +RSYNC_EXCLUDES=( + --exclude '/input/' + --exclude '/workspace/' + --exclude '/logs/' + --exclude '/cost_logs/' + --exclude '/memory_storage/' + --exclude '/service_data/' + --exclude '/outputs/' + --exclude '/output/' + --exclude '/final/' + --exclude '/backup/' + --exclude '/backups/' + --exclude '/tmp/' + --exclude '/temp/' + --exclude '/tests/' + --exclude '/test_integration/' + --exclude '/benchmarks/' + --exclude '/.git/' + --exclude '/.agents/' + --exclude '/.codex/' + --exclude '/.cursor/' + --exclude '/.idea/' + --exclude '/.pytest_cache/' + --exclude '/__pycache__/' + --exclude '**/__pycache__/' + --exclude '*.pyc' + --exclude '*.pyo' + --exclude '*.c' + --exclude '*.cpp' + --exclude '*.log' + --exclude '*.bak' + --exclude '*.backup' + --exclude '*.backup.*' + --exclude '.env' + --exclude '*.key' + --exclude '*.pem' + --exclude 'license.json' + --exclude 'asset.key.enc' +) + +sync_artifact() { + local source="$1" destination="$2" project="$3" + mkdir -p "$destination" + echo "[stage] $project: $source -> $destination" + rsync -a --delete --safe-links "${RSYNC_EXCLUDES[@]}" "$source/" "$destination/" +} +sync_artifact "$PLANTAI_ARTIFACT" "$CONTEXT_DIR/PlantAI" PlantAI +sync_artifact "$RAG_ARTIFACT" "$CONTEXT_DIR/RAG" RAG +sync_artifact "$QA_ARTIFACT" "$CONTEXT_DIR/QA" QA + +PFAM_MAPPING_SOURCE="$SCRIPT_DIR/runtime-assets/pfam_mapping.json" +[[ -s "$PFAM_MAPPING_SOURCE" ]] || + die "missing offline Pfam family mapping: $PFAM_MAPPING_SOURCE" +mkdir -p "$CONTEXT_DIR/PlantAI/data/rag_db" +cp "$PFAM_MAPPING_SOURCE" "$CONTEXT_DIR/PlantAI/data/rag_db/pfam_mapping.json" +python3 - "$CONTEXT_DIR/PlantAI/data/rag_db/pfam_mapping.json" <<'PY' +import json +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) +mapping = json.loads(path.read_text(encoding="utf-8")) +expected = { + "SPL": "PF03110", + "SBP": "PF03110", + "WRKY": "PF03106", + "NAC": "PF02365", +} +if not isinstance(mapping, dict): + raise SystemExit(f"{path}: expected a JSON object") +for family, accession in expected.items(): + if mapping.get(family) != accession: + raise SystemExit( + f"{path}: expected {family}={accession}, got {mapping.get(family)!r}" + ) +print(f"[stage] offline Pfam family mapping entries={len(mapping)}") +PY + +if [[ "$DROP_COMPILED_PY_SIBLINGS" == "1" ]]; then + python3 - "$CONTEXT_DIR" <<'PY' +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]) +for project in ("PlantAI", "RAG", "QA"): + removed = 0 + for py_path in (root / project).rglob("*.py"): + stem = py_path.name[:-3] + compiled = list(py_path.parent.glob(stem + ".cpython-*.so")) + compiled += list(py_path.parent.glob(stem + ".abi3.so")) + if compiled: + py_path.unlink() + removed += 1 + print(f"[stage] {project}: removed compiled .py siblings={removed}") +PY +fi + +if [[ "$INCLUDE_PLANTAI_STATIC_DATA" != "1" ]]; then + rm -rf -- "$CONTEXT_DIR/PlantAI/data" + echo "[stage] excluded PlantAI/data" +fi + +if [[ "$INCLUDE_PLANTAI_STATIC_DATA" == "1" ]]; then + for pfam_file in \ + Pfam-A.hmm \ + Pfam-A.hmm.h3f \ + Pfam-A.hmm.h3i \ + Pfam-A.hmm.h3m \ + Pfam-A.hmm.h3p; do + [[ -s "$CONTEXT_DIR/PlantAI/data/hmm/$pfam_file" ]] || + die "missing required Identify Pfam asset: PlantAI/data/hmm/$pfam_file" + done + [[ -s "$CONTEXT_DIR/PlantAI/data/rag_db/pfam_mapping.json" ]] || + die "missing required Identify offline mapping: PlantAI/data/rag_db/pfam_mapping.json" +fi + +mapfile -t staged_links < <(find "$CONTEXT_DIR" -type l -print) +if (( ${#staged_links[@]} > 0 )); then + printf '%s\n' "${staged_links[@]}" >&2 + die "isolated build context still contains symbolic links" +fi +mapfile -t forbidden_files < <( + find "$CONTEXT_DIR" -type f \ + \( -name '.env' -o -name '*.key' -o -name '*.pem' \ + -o -name 'license.json' -o -name 'asset.key.enc' \) -print +) +if (( ${#forbidden_files[@]} > 0 )); then + printf '%s\n' "${forbidden_files[@]}" >&2 + die "isolated build context contains forbidden files" +fi + +python3 - "$CONTEXT_DIR" "$PLANTAI_BASE_IMAGE" "$RAG_BASE_IMAGE" "$QA_BASE_IMAGE" \ + "$PLANTAI_IMAGE_REF" "$RAG_IMAGE_REF" "$QA_IMAGE_REF" "$PLATFORM" \ + "$DROP_COMPILED_PY_SIBLINGS" <<'PY' +import datetime +import json +import pathlib +import sys + +context = pathlib.Path(sys.argv[1]) +manifest = { + "package_format": "plantai-protected-docker-v2", + "created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "build_policy": { + "full_environment_in_image": False, + "reuses_existing_application_image": False, + "no_cache": True, + "pull_clean_base_images": True, + }, + "base_images": { + "PlantAI": sys.argv[2], + "RAG": sys.argv[3], + "QA": sys.argv[4], + }, + "application_images": { + "PlantAI": sys.argv[5], + "RAG": sys.argv[6], + "QA": sys.argv[7], + }, + "platform": sys.argv[8], + "projects": { + name: json.loads((context / name / ".hardened-build.json").read_text(encoding="utf-8")) + for name in ("PlantAI", "RAG", "QA") + }, + "excluded_runtime_directories": [ + "input", "workspace", "logs", "cost_logs", "memory_storage", + "service_data", "outputs", "backup", "tmp", "tests" + ], + "compiled_python_siblings_removed": sys.argv[9] == "1", + "identify_offline_pfam_mapping": "PlantAI/data/rag_db/pfam_mapping.json", + "hardware_binding": False, +} +(context / "PACKAGE-MANIFEST.json").write_text( + json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", +) +PY + +du -sh "$CONTEXT_DIR/PlantAI" "$CONTEXT_DIR/RAG" "$CONTEXT_DIR/QA" + +render_compose_env() { + local destination="$1" + sed \ + -e "s|@@PLANTAI_IMAGE_REF@@|$PLANTAI_IMAGE_REF|g" \ + -e "s|@@RAG_IMAGE_REF@@|$RAG_IMAGE_REF|g" \ + -e "s|@@QA_IMAGE_REF@@|$QA_IMAGE_REF|g" \ + "$SCRIPT_DIR/bbav5.env.example" >"$destination" +} + +if [[ "$CHECK_ONLY" == "1" ]]; then + CHECK_ENV="$WORK_DIR/check-bbav5.env" + SERVICE_CHECK_ENV="$SCRIPT_DIR/bbav5.env" + [[ ! -e "$SERVICE_CHECK_ENV" ]] || + die "check-only mode refuses to overwrite existing $SERVICE_CHECK_ENV" + render_compose_env "$CHECK_ENV" + cp "$CHECK_ENV" "$SERVICE_CHECK_ENV" + cleanup_check_env() { + rm -f -- "$SERVICE_CHECK_ENV" "$CHECK_ENV" + } + trap cleanup_check_env EXIT + docker compose --env-file "$CHECK_ENV" -f "$SCRIPT_DIR/docker-compose.yml" config >/dev/null + cleanup_check_env + trap - EXIT + echo "[check-only] paths, ABI, exclusions, staging, Docker policy and Compose passed" + echo "[check-only] no image pull, image build, image save or release compression was run" + exit 0 +fi + +echo "[pull] clean official Python base images" +docker pull --platform "$PLATFORM" "$PLANTAI_BASE_IMAGE" +if [[ "$RAG_BASE_IMAGE" != "$PLANTAI_BASE_IMAGE" ]]; then + docker pull --platform "$PLATFORM" "$RAG_BASE_IMAGE" +fi +docker pull --platform "$PLATFORM" "$QA_BASE_IMAGE" + +export DOCKER_BUILDKIT=1 +build_application_image() { + local project="$1" dockerfile="$2" base_image="$3" image_ref="$4" + echo "[build] $project -> $image_ref from $base_image" + docker build \ + --pull \ + --no-cache \ + --network "$BUILD_NETWORK" \ + --platform "$PLATFORM" \ + --build-arg "BASE_IMAGE=$base_image" \ + --build-arg "PIP_INDEX_URL=$PIP_INDEX_URL" \ + --build-arg "TORCH_INDEX_URL=$TORCH_INDEX_URL" \ + --build-arg "TORCH_CPU_VERSION=$TORCH_CPU_VERSION" \ + -f "$SCRIPT_DIR/$dockerfile" \ + -t "$image_ref" \ + "$SCRIPT_DIR" +} +build_application_image PlantAI Dockerfile.plantai "$PLANTAI_BASE_IMAGE" "$PLANTAI_IMAGE_REF" +build_application_image RAG Dockerfile.rag "$RAG_BASE_IMAGE" "$RAG_IMAGE_REF" +build_application_image QA Dockerfile.qa "$QA_BASE_IMAGE" "$QA_IMAGE_REF" + +verify_image() { + local project="$1" image_ref="$2" root="$3" expected_minor="$4" + echo "[verify] $project image structure and Python $expected_minor" + docker run --rm --entrypoint sh "$image_ref" -c " + set -eu + test -f '$root/.hardened-build.json' + test \"\$(find '$root' -type f -name '*.so' | wc -l)\" -gt 0 + python -c \"import sys; assert '.'.join(map(str, sys.version_info[:2])) == '$expected_minor'\" + if find '$root' -type f \\ + \\( -name '.env' -o -name '*.key' -o -name '*.pem' \\ + -o -name 'license.json' -o -name 'asset.key.enc' \\) \\ + -print -quit | grep -q .; then + echo 'forbidden file found in image' >&2 + exit 1 + fi + " +} +verify_image PlantAI "$PLANTAI_IMAGE_REF" /opt/plantai 3.11 +verify_image RAG "$RAG_IMAGE_REF" /opt/rag 3.11 +verify_image QA "$QA_IMAGE_REF" /opt/qa 3.12 + +verify_cpu_torch() { + local project="$1" image_ref="$2" + echo "[verify] $project uses pinned CPU-only Torch $TORCH_CPU_VERSION" + docker run --rm --entrypoint python "$image_ref" -c " +import torch +assert torch.__version__ == '$TORCH_CPU_VERSION', torch.__version__ +assert torch.version.cuda is None, torch.version.cuda +print(torch.__version__) +" +} +verify_cpu_torch RAG "$RAG_IMAGE_REF" +verify_cpu_torch QA "$QA_IMAGE_REF" + +echo "[verify] PlantAI Identify native tools and Pfam database" +docker run --rm --entrypoint sh "$PLANTAI_IMAGE_REF" -c ' + set -eu + command -v hmmscan + command -v hmmsearch + command -v blastp + command -v makeblastdb + test -s /opt/plantai/data/hmm/Pfam-A.hmm + test -s /opt/plantai/data/hmm/Pfam-A.hmm.h3f + test -s /opt/plantai/data/hmm/Pfam-A.hmm.h3i + test -s /opt/plantai/data/hmm/Pfam-A.hmm.h3m + test -s /opt/plantai/data/hmm/Pfam-A.hmm.h3p + test -s /opt/plantai/data/rag_db/pfam_mapping.json + python -c "import json; p=json.load(open(\"/opt/plantai/data/rag_db/pfam_mapping.json\")); assert p[\"SPL\"]==\"PF03110\"" +' + +images=("$PLANTAI_IMAGE_REF" "$RAG_IMAGE_REF" "$QA_IMAGE_REF") +if [[ "$INCLUDE_INFRA_IMAGES" == "1" ]]; then + for image in "$REDIS_IMAGE" "$QDRANT_IMAGE" "$NEO4J_IMAGE"; do + if [[ "$PULL_INFRA_IMAGES" == "1" ]]; then + docker pull --platform "$PLATFORM" "$image" + else + docker image inspect "$image" >/dev/null 2>&1 || + die "missing infrastructure image and PULL_INFRA_IMAGES=0: $image" + fi + images+=("$image") + done +fi + +mkdir -p \ + "$PRODUCT_DIR/images" \ + "$PRODUCT_DIR/input" \ + "$PRODUCT_DIR/workspace" \ + "$PRODUCT_DIR/data" \ + "$PRODUCT_DIR/logs" \ + "$PRODUCT_DIR/models" \ + "$PRODUCT_DIR/service_data/memory_storage" \ + "$PRODUCT_DIR/service_data/cost_logs" \ + "$PRODUCT_DIR/service_data/reproducibility" \ + "$PRODUCT_DIR/service_data/redis" \ + "$PRODUCT_DIR/service_data/qdrant" \ + "$PRODUCT_DIR/service_data/neo4j" + +echo "[export] ${images[*]}" +: >"$PRODUCT_DIR/IMAGE-IDS.txt" +for image in "${images[@]}"; do + printf '%s %s\n' \ + "$image" \ + "$(docker image inspect "$image" --format '{{.Id}}')" \ + >>"$PRODUCT_DIR/IMAGE-IDS.txt" +done +docker save "${images[@]}" | zstd -T0 -10 -o "$PRODUCT_DIR/images/$IMAGE_ARCHIVE_NAME" + +cp "$SCRIPT_DIR/docker-compose.yml" "$PRODUCT_DIR/" +render_compose_env "$PRODUCT_DIR/bbav5.env.example" +for runtime_script in install start stop status logs health-check; do + cp "$SCRIPT_DIR/runtime/$runtime_script.sh" "$PRODUCT_DIR/$runtime_script.sh" +done +cp "$SCRIPT_DIR/README.md" "$PRODUCT_DIR/README.md" +cp "$CONTEXT_DIR/PACKAGE-MANIFEST.json" "$PRODUCT_DIR/" +chmod +x "$PRODUCT_DIR"/*.sh + +( + cd "$PRODUCT_DIR" + sha256sum \ + "images/$IMAGE_ARCHIVE_NAME" \ + docker-compose.yml \ + bbav5.env.example \ + IMAGE-IDS.txt \ + PACKAGE-MANIFEST.json >SHA256SUMS +) + +if [[ -e "$ARCHIVE" && "${OVERWRITE:-0}" != "1" ]]; then + die "release archive exists; set OVERWRITE=1 only after confirming replacement: $ARCHIVE" +fi +rm -f -- "$ARCHIVE" "$ARCHIVE.sha256" +echo "[package] $ARCHIVE" +tar -C "$PACKAGE_STAGE" --zstd -cf "$ARCHIVE" "$PRODUCT_NAME" +( + cd "$DIST_DIR" + sha256sum "$(basename "$ARCHIVE")" >"$(basename "$ARCHIVE").sha256" +) + +echo "[done] Docker package complete" +ls -lh "$ARCHIVE" "$ARCHIVE.sha256" +printf '[image] %s\n' "$PLANTAI_IMAGE_REF" "$RAG_IMAGE_REF" "$QA_IMAGE_REF" +echo "[customer] tar --zstd -xf $(basename "$ARCHIVE")" diff --git a/tooling/legacy_docker_packager/docker-compose.yml b/tooling/legacy_docker_packager/docker-compose.yml new file mode 100644 index 0000000..0a7b412 --- /dev/null +++ b/tooling/legacy_docker_packager/docker-compose.yml @@ -0,0 +1,224 @@ +name: plantai-protected + +x-app-common: &app-common + restart: unless-stopped + env_file: + - ./bbav5.env + extra_hosts: + - "host.docker.internal:host-gateway" + security_opt: + - no-new-privileges:true + +x-plantai-env: &plantai-env + PLANTAI_PROJECT_ROOT: /opt/plantai + BBAV5_PROJECT_ROOT: /opt/plantai + PLANTAI_RAG_ROOT: /opt/rag + GENE_FAMILY_ROOT: /opt/qa + PLANTAI_INPUT_ROOT: /opt/plantai/input + PLANTAI_WORKSPACE_ROOT: /opt/plantai/workspace + PLANTAI_DATA_ROOT: /runtime/data + PLANTAI_LOG_DIR: /opt/plantai/logs + PLANTAI_MEMORY_STORAGE_DIR: /runtime/service_data/memory_storage + PLANTAI_COST_LOG_DIR: /runtime/service_data/cost_logs + BBA_PFAM_DIR: /opt/plantai/data/hmm + BBAV5_TEST4_BIN: /usr/local/bin + BBAV5_DEFAULT_ENV: system + BBAV5_REDIS_URL: redis://redis:6379/0 + REDIS_URL: redis://redis:6379/0 + METHODOLOGY_RAG_API: http://rag:7900/api/v1 + BBA_RAG_FAMILY_PFAM_URL: http://rag:7900/api/v1/rag/family/pfam + BBA_RAG_PFAM_URL: http://rag:7900/api/v1/rag/chat + DUAL_KB_QA_BASE_URL: http://qa:7925 + DUAL_KB_QA_ROUTE: qa-v1-1-73 + PYTHONUNBUFFERED: "1" + PYTHONDONTWRITEBYTECODE: "1" + +x-plantai-volumes: &plantai-volumes + - ./input:/runtime/input + - ./input:/opt/plantai/input + - ./workspace:/runtime/workspace + - ./workspace:/opt/plantai/workspace + - ./data:/runtime/data + - ./logs:/runtime/logs + - ./logs:/opt/plantai/logs + - ./service_data/memory_storage:/runtime/service_data/memory_storage + - ./service_data/cost_logs:/runtime/service_data/cost_logs + - ./service_data/reproducibility:/runtime/service_data/reproducibility + +services: + redis: + image: ${REDIS_IMAGE:-redis:7} + restart: unless-stopped + command: ["redis-server", "--appendonly", "yes"] + volumes: + - ./service_data/redis:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 3s + retries: 10 + + qdrant: + image: ${QDRANT_IMAGE:-qdrant/qdrant:latest} + restart: unless-stopped + volumes: + - ./service_data/qdrant:/qdrant/storage + ports: + - "${BIND_ADDRESS:-127.0.0.1}:${QDRANT_PORT:-16333}:6333" + - "${BIND_ADDRESS:-127.0.0.1}:${QDRANT_GRPC_PORT:-16334}:6334" + + neo4j: + image: ${NEO4J_IMAGE:-neo4j:5.15.0-community} + restart: unless-stopped + environment: + NEO4J_AUTH: neo4j/${NEO4J_PASSWORD:?set NEO4J_PASSWORD in bbav5.env} + volumes: + - ./service_data/neo4j:/data + ports: + - "${BIND_ADDRESS:-127.0.0.1}:${NEO4J_HTTP_PORT:-17474}:7474" + - "${BIND_ADDRESS:-127.0.0.1}:${NEO4J_BOLT_PORT:-17687}:7687" + + rag: + <<: *app-common + image: ${RAG_IMAGE:?set RAG_IMAGE in bbav5.env} + working_dir: /opt/rag + depends_on: + redis: + condition: service_healthy + qdrant: + condition: service_started + neo4j: + condition: service_started + environment: + PYTHONPATH: /opt/rag:/opt/rag/src + QDRANT_URL: http://qdrant:6333 + NEO4J_URI: bolt://neo4j:7687 + NEO4J_USER: neo4j + NEO4J_PASSWORD: ${NEO4J_PASSWORD} + ports: + - "${BIND_ADDRESS:-127.0.0.1}:${RAG_PORT:-7900}:7900" + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:7900/health', timeout=5).read()"] + interval: 15s + timeout: 8s + retries: 20 + start_period: 30s + + qa: + <<: *app-common + image: ${QA_IMAGE:?set QA_IMAGE in bbav5.env} + working_dir: /opt/qa + depends_on: + qdrant: + condition: service_started + neo4j: + condition: service_started + environment: + PYTHONPATH: /opt/qa + GENE_FAMILY_ROOT: /opt/qa + QDRANT_URL: http://qdrant:6333 + V15_QDRANT_URL: http://qdrant:6333 + V15_QDRANT_COLLECTION: ${V15_QDRANT_COLLECTION} + NEO4J_URI: bolt://neo4j:7687 + NEO4J_USER: neo4j + NEO4J_PASSWORD: ${NEO4J_PASSWORD} + QA_V1_BGE_RERANKER_MODEL: /models/bge-reranker-base + QA_V1_ALIAS_CONFIG: /opt/qa/configs/rag/entity_alias.yaml + QA_V1_TAXONOMY_CONFIG: /opt/qa/configs/rag/relation_taxonomy.yaml + QA_V1_LLM_API_KEY: ${QA_V1_LLM_API_KEY:-ollama-local} + QA_V1_LLM_BASE_URL: ${QA_V1_LLM_BASE_URL:-http://host.docker.internal:11434/v1} + QA_V1_LLM_MODEL: ${QA_V1_LLM_MODEL} + V15_OLLAMA_EMBED_URL: ${V15_OLLAMA_EMBED_URL:-http://host.docker.internal:11434/api/embed} + V15_OLLAMA_EMBED_MODEL: ${V15_OLLAMA_EMBED_MODEL} + volumes: + - ./models:/models:ro + ports: + - "${BIND_ADDRESS:-127.0.0.1}:${QA_PORT:-7925}:7925" + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:7925/health', timeout=5).read()"] + interval: 15s + timeout: 8s + retries: 20 + start_period: 30s + + web: + <<: *app-common + image: ${PLANTAI_IMAGE:?set PLANTAI_IMAGE in bbav5.env} + working_dir: /opt/plantai + depends_on: + redis: + condition: service_healthy + rag: + condition: service_healthy + qa: + condition: service_healthy + environment: + <<: *plantai-env + PLANTAI_ROLE: web + PLANTAI_WEB_PORT: "7880" + BBAV5_QUEUE_PREFIX: ${BBAV5_QUEUE_PREFIX:-plantai_protected} + volumes: *plantai-volumes + ports: + - "${BIND_ADDRESS:-127.0.0.1}:${PLANTAI_PORT:-7880}:7880" + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:7880/', timeout=5).read()"] + interval: 15s + timeout: 8s + retries: 20 + start_period: 30s + + worker-default: + <<: *app-common + image: ${PLANTAI_IMAGE:?set PLANTAI_IMAGE in bbav5.env} + working_dir: /opt/plantai + depends_on: + redis: + condition: service_healthy + environment: + <<: *plantai-env + PLANTAI_ROLE: worker + BBAV5_QUEUE_PREFIX: ${BBAV5_QUEUE_PREFIX:-plantai_protected} + volumes: *plantai-volumes + command: ["sh", "-c", "python -m celery -A celery_app worker --loglevel=${BBAV5_CELERY_LOGLEVEL:-info} --hostname=plantai_protected_default@%h --queues=${BBAV5_QUEUE_PREFIX:-plantai_protected}.default --concurrency=${BBAV5_DEFAULT_CONCURRENCY:-1} --prefetch-multiplier=1"] + + worker-heavy: + <<: *app-common + image: ${PLANTAI_IMAGE:?set PLANTAI_IMAGE in bbav5.env} + working_dir: /opt/plantai + depends_on: + redis: + condition: service_healthy + environment: + <<: *plantai-env + PLANTAI_ROLE: worker + BBAV5_QUEUE_PREFIX: ${BBAV5_QUEUE_PREFIX:-plantai_protected} + volumes: *plantai-volumes + command: ["sh", "-c", "python -m celery -A celery_app worker --loglevel=${BBAV5_CELERY_LOGLEVEL:-info} --hostname=plantai_protected_heavy@%h --queues=${BBAV5_QUEUE_PREFIX:-plantai_protected}.heavy --concurrency=${BBAV5_HEAVY_CONCURRENCY:-1} --prefetch-multiplier=1"] + + worker-medium: + <<: *app-common + image: ${PLANTAI_IMAGE:?set PLANTAI_IMAGE in bbav5.env} + working_dir: /opt/plantai + depends_on: + redis: + condition: service_healthy + environment: + <<: *plantai-env + PLANTAI_ROLE: worker + BBAV5_QUEUE_PREFIX: ${BBAV5_QUEUE_PREFIX:-plantai_protected} + volumes: *plantai-volumes + command: ["sh", "-c", "python -m celery -A celery_app worker --loglevel=${BBAV5_CELERY_LOGLEVEL:-info} --hostname=plantai_protected_medium@%h --queues=${BBAV5_QUEUE_PREFIX:-plantai_protected}.medium --concurrency=${BBAV5_MEDIUM_CONCURRENCY:-2} --prefetch-multiplier=1"] + + worker-light: + <<: *app-common + image: ${PLANTAI_IMAGE:?set PLANTAI_IMAGE in bbav5.env} + working_dir: /opt/plantai + depends_on: + redis: + condition: service_healthy + environment: + <<: *plantai-env + PLANTAI_ROLE: worker + BBAV5_QUEUE_PREFIX: ${BBAV5_QUEUE_PREFIX:-plantai_protected} + volumes: *plantai-volumes + command: ["sh", "-c", "python -m celery -A celery_app worker --loglevel=${BBAV5_CELERY_LOGLEVEL:-info} --hostname=plantai_protected_light@%h --queues=${BBAV5_QUEUE_PREFIX:-plantai_protected}.light --concurrency=${BBAV5_LIGHT_CONCURRENCY:-2} --prefetch-multiplier=1"] diff --git a/tooling/legacy_docker_packager/pack.env.example b/tooling/legacy_docker_packager/pack.env.example new file mode 100644 index 0000000..19e77f9 --- /dev/null +++ b/tooling/legacy_docker_packager/pack.env.example @@ -0,0 +1,35 @@ +# Build-time configuration. Keep secrets and API keys out of this file. +ARTIFACT_ROOT=/home/xzq/5090/PlanAI-cooperate/hsj-workspace/encryption-test/server +PLANTAI_ARTIFACT=/home/xzq/5090/PlanAI-cooperate/hsj-workspace/encryption-test/server/PlantAI_compiled.new +RAG_ARTIFACT=/home/xzq/5090/PlanAI-cooperate/hsj-workspace/encryption-test/server/RAG.new +QA_ARTIFACT=/home/xzq/5090/PlanAI-cooperate/hsj-workspace/encryption-test/server/RAG_QA.new + +# Clean official Python bases. No existing PlantAI/allenv image is used. +PLANTAI_BASE_IMAGE=python:3.11-bookworm +RAG_BASE_IMAGE=python:3.11-bookworm +QA_BASE_IMAGE=python:3.12-bookworm + +PLANTAI_IMAGE_REPOSITORY=plantai-protected +RAG_IMAGE_REPOSITORY=plantai-rag-protected +QA_IMAGE_REPOSITORY=plantai-rag-qa-protected +VERSION=1.0.0 +PLATFORM=linux/amd64 + +# The build always uses --pull and --no-cache. Host networking prevents the +# Docker bridge timeout previously seen while downloading package indexes. +BUILD_NETWORK=host +PIP_INDEX_URL=https://pypi.org/simple +TORCH_INDEX_URL=https://download.pytorch.org/whl/cpu +TORCH_CPU_VERSION=2.13.0+cpu + +# Export infrastructure images with the customer package. These are upstream +# services, not reused PlantAI application images. +INCLUDE_INFRA_IMAGES=1 +PULL_INFRA_IMAGES=1 +REDIS_IMAGE=redis:7 +QDRANT_IMAGE=qdrant/qdrant:latest +NEO4J_IMAGE=neo4j:5.15.0-community + +MIN_FREE_GB=60 +INCLUDE_PLANTAI_STATIC_DATA=1 +DROP_COMPILED_PY_SIBLINGS=1 diff --git a/tooling/legacy_docker_packager/requirements/plantai.txt b/tooling/legacy_docker_packager/requirements/plantai.txt new file mode 100644 index 0000000..11b089c --- /dev/null +++ b/tooling/legacy_docker_packager/requirements/plantai.txt @@ -0,0 +1,32 @@ +fastapi>=0.110 +uvicorn[standard]>=0.27 +python-multipart>=0.0.9 +pydantic>=2 +pydantic-settings>=2.2 +aiofiles>=23.2 +httpx>=0.27 +requests>=2.31 +celery>=5.3 +redis>=5.0 +openai>=1.30 +langgraph>=0.2 +langchain-core>=0.2 +numpy>=1.24 +pandas>=2.0 +scipy>=1.10 +biopython>=1.83 +scikit-learn>=1.3 +statsmodels>=0.14 +networkx>=3.2 +matplotlib>=3.7 +matplotlib-venn>=0.11 +seaborn>=0.13 +openpyxl>=3.1 +xlsxwriter>=3.1 +Pillow>=10 +reportlab>=4 +PyYAML>=6 +psutil>=5.9 +tqdm>=4.66 +beautifulsoup4>=4.12 +lxml>=5 diff --git a/tooling/legacy_docker_packager/requirements/qa.txt b/tooling/legacy_docker_packager/requirements/qa.txt new file mode 100644 index 0000000..03e2220 --- /dev/null +++ b/tooling/legacy_docker_packager/requirements/qa.txt @@ -0,0 +1,40 @@ +fastapi>=0.109 +uvicorn[standard]>=0.27 +pydantic>=2.5 +pydantic-settings>=2.2 +python-dotenv>=1.0 +python-multipart>=0.0.9 +openai>=1.30 +zhipuai>=2.1 +httpx>=0.27 +requests>=2.31 +langgraph>=0.2 +langchain>=0.1 +langchain-core>=0.2 +langchain-community>=0.0.10 +langchain-huggingface>=0.0.1 +langsmith>=0.1 +huggingface-hub>=0.19 +sentence-transformers>=2.2 +transformers>=4.36 +faiss-cpu>=1.7.4 +rank-bm25>=0.2.2 +qdrant-client>=1.7 +pymilvus>=2.4 +neo4j>=5.15 +sqlalchemy>=2.0 +asyncpg>=0.29 +redis>=5.0 +numpy>=1.26 +pandas>=2.1 +biopython>=1.83 +pymupdf>=1.23 +openpyxl>=3.1 +xlrd>=2.0 +python-docx>=1.1 +PyYAML>=6 +rich>=13 +loguru>=0.7 +tenacity>=8 +aiofiles>=23.2 +websockets>=12 diff --git a/tooling/legacy_docker_packager/requirements/rag.txt b/tooling/legacy_docker_packager/requirements/rag.txt new file mode 100644 index 0000000..2c19284 --- /dev/null +++ b/tooling/legacy_docker_packager/requirements/rag.txt @@ -0,0 +1,27 @@ +fastapi>=0.109 +uvicorn[standard]>=0.27 +neo4j>=5.15 +pydantic>=2.5 +pydantic-settings>=2.2 +python-dotenv>=1.0 +python-multipart>=0.0.9 +numpy>=1.26 +pandas>=2.1 +requests>=2.31 +httpx>=0.27 +openpyxl>=3.1 +xlrd>=2.0 +python-docx>=1.1 +faiss-cpu>=1.7.4 +langchain>=0.1 +langchain-core>=0.2 +langchain-community>=0.0.10 +langchain-huggingface>=0.0.1 +huggingface-hub>=0.19 +rank-bm25>=0.2.2 +pymupdf>=1.23 +qdrant-client>=1.7 +redis>=5.0 +PyYAML>=6 +tenacity>=8 +aiofiles>=23.2 diff --git a/tooling/legacy_docker_packager/runtime/health-check.sh b/tooling/legacy_docker_packager/runtime/health-check.sh new file mode 100755 index 0000000..1782cde --- /dev/null +++ b/tooling/legacy_docker_packager/runtime/health-check.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")" +[[ -f bbav5.env ]] || { + echo "[error] 缺少 bbav5.env" >&2 + exit 1 +} + +read_env() { + local name="$1" fallback="$2" value + value="$(grep -E "^${name}=" bbav5.env | tail -1 | cut -d= -f2-)" + printf '%s' "${value:-$fallback}" +} + +BIND_ADDRESS="$(read_env BIND_ADDRESS 127.0.0.1)" +[[ "$BIND_ADDRESS" == "0.0.0.0" ]] && BIND_ADDRESS=127.0.0.1 +QDRANT_PORT="$(read_env QDRANT_PORT 16333)" +NEO4J_HTTP_PORT="$(read_env NEO4J_HTTP_PORT 17474)" +RAG_PORT="$(read_env RAG_PORT 7900)" +QA_PORT="$(read_env QA_PORT 7925)" +PLANTAI_PORT="$(read_env PLANTAI_PORT 7880)" +NEO4J_PASSWORD="$(read_env NEO4J_PASSWORD '')" + +check_url() { + local label="$1" url="$2" + if curl -fsS --max-time 8 "$url" >/dev/null; then + echo "[ok] $label $url" + else + echo "[fail] $label $url" >&2 + return 1 + fi +} + +docker compose --env-file bbav5.env -f docker-compose.yml ps +qdrant_response="$(curl -fsS --max-time 8 "http://$BIND_ADDRESS:$QDRANT_PORT/collections")" || { + echo "[fail] Qdrant API 不可用" >&2 + exit 1 +} +qdrant_compact="${qdrant_response//[[:space:]]/}" +if [[ "$qdrant_compact" == *'"collections":[]'* ]]; then + echo "[fail] Qdrant 服务已启动，但 collections 为空；RAG 数据尚未恢复" >&2 + exit 1 +fi +echo "[ok] Qdrant 已检测到 collection" + +check_url Neo4j "http://$BIND_ADDRESS:$NEO4J_HTTP_PORT" +[[ -n "$NEO4J_PASSWORD" ]] || { + echo "[fail] bbav5.env 中缺少 NEO4J_PASSWORD" >&2 + exit 1 +} +neo4j_response="$( + curl -fsS --max-time 8 \ + -u "neo4j:$NEO4J_PASSWORD" \ + -H 'Content-Type: application/json' \ + -d '{"statements":[{"statement":"MATCH (n) RETURN count(n) AS count"}]}' \ + "http://$BIND_ADDRESS:$NEO4J_HTTP_PORT/db/neo4j/tx/commit" +)" || { + echo "[fail] Neo4j 数据查询失败" >&2 + exit 1 +} +neo4j_compact="${neo4j_response//[[:space:]]/}" +if [[ "$neo4j_compact" == *'"errors":[]'* && "$neo4j_compact" == *'"row":[0]'* ]]; then + echo "[fail] Neo4j 服务已启动，但节点数为 0；图谱数据尚未恢复" >&2 + exit 1 +fi +if [[ "$neo4j_compact" != *'"errors":[]'* ]]; then + echo "[fail] Neo4j 返回查询错误" >&2 + exit 1 +fi +echo "[ok] Neo4j 已检测到数据节点" + +check_url RAG "http://$BIND_ADDRESS:$RAG_PORT/health" +check_url QA "http://$BIND_ADDRESS:$QA_PORT/health" +check_url PlantAI "http://$BIND_ADDRESS:$PLANTAI_PORT/" +echo "[ok] HTTP 服务与 RAG 数据就绪检查通过" diff --git a/tooling/legacy_docker_packager/runtime/install.sh b/tooling/legacy_docker_packager/runtime/install.sh new file mode 100755 index 0000000..0512678 --- /dev/null +++ b/tooling/legacy_docker_packager/runtime/install.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 027 + +cd "$(dirname "$0")" + +command -v docker >/dev/null 2>&1 || { + echo "[error] Docker 未安装或不在 PATH 中" >&2 + exit 1 +} +command -v zstd >/dev/null 2>&1 || { + echo "[error] 缺少 zstd，无法导入离线镜像" >&2 + exit 1 +} +docker info >/dev/null 2>&1 || { + echo "[error] Docker daemon 未运行" >&2 + exit 1 +} +docker compose version >/dev/null 2>&1 || { + echo "[error] 需要 Docker Compose v2" >&2 + exit 1 +} + +sha256sum -c SHA256SUMS + +if [[ ! -f bbav5.env ]]; then + cp bbav5.env.example bbav5.env + chmod 600 bbav5.env + echo "[init] 已创建 bbav5.env，请在启动前填写数据库密码和模型配置" +fi + +mkdir -p \ + input workspace data logs models \ + service_data/memory_storage \ + service_data/cost_logs \ + service_data/reproducibility \ + service_data/redis \ + service_data/qdrant \ + service_data/neo4j + +shopt -s nullglob +archives=(images/*.tar.zst) +(( ${#archives[@]} > 0 )) || { + echo "[error] images/ 下没有离线镜像包" >&2 + exit 1 +} +for archive in "${archives[@]}"; do + echo "[load] $archive" + zstd -dc "$archive" | docker load +done + +docker compose --env-file bbav5.env -f docker-compose.yml config >/dev/null +echo "[ok] 安装准备完成" +echo "[next] 编辑 bbav5.env，然后执行 ./start.sh" + diff --git a/tooling/legacy_docker_packager/runtime/logs.sh b/tooling/legacy_docker_packager/runtime/logs.sh new file mode 100755 index 0000000..435209f --- /dev/null +++ b/tooling/legacy_docker_packager/runtime/logs.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")" +docker compose --env-file bbav5.env -f docker-compose.yml logs --tail="${TAIL:-200}" "$@" diff --git a/tooling/legacy_docker_packager/runtime/start.sh b/tooling/legacy_docker_packager/runtime/start.sh new file mode 100755 index 0000000..eb28003 --- /dev/null +++ b/tooling/legacy_docker_packager/runtime/start.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")" +[[ -f bbav5.env ]] || { + echo "[error] 缺少 bbav5.env，请先执行 ./install.sh" >&2 + exit 1 +} + +password="$(grep -E '^NEO4J_PASSWORD=' bbav5.env | tail -1 | cut -d= -f2-)" +if [[ -z "$password" || "$password" == "CHANGE_ME" ]]; then + echo "[error] 请先在 bbav5.env 设置真实 NEO4J_PASSWORD" >&2 + exit 1 +fi + +docker compose --env-file bbav5.env -f docker-compose.yml config >/dev/null +docker compose --env-file bbav5.env -f docker-compose.yml up -d + +port="$(grep -E '^PLANTAI_PORT=' bbav5.env | tail -1 | cut -d= -f2-)" +port="${port:-7880}" +echo "[ok] 服务启动命令已提交" +echo "[open] http://127.0.0.1:${port}" +echo "[check] ./health-check.sh" + diff --git a/tooling/legacy_docker_packager/runtime/status.sh b/tooling/legacy_docker_packager/runtime/status.sh new file mode 100755 index 0000000..c3c2b1f --- /dev/null +++ b/tooling/legacy_docker_packager/runtime/status.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")" +docker compose --env-file bbav5.env -f docker-compose.yml ps + diff --git a/tooling/legacy_docker_packager/runtime/stop.sh b/tooling/legacy_docker_packager/runtime/stop.sh new file mode 100755 index 0000000..9fc2eb3 --- /dev/null +++ b/tooling/legacy_docker_packager/runtime/stop.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")" +docker compose --env-file bbav5.env -f docker-compose.yml down + diff --git a/tooling/legacy_plantai_server_package/.gitignore b/tooling/legacy_plantai_server_package/.gitignore new file mode 100644 index 0000000..2faa995 --- /dev/null +++ b/tooling/legacy_plantai_server_package/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.pyc +*.pyo +.DS_Store +Thumbs.db diff --git a/tooling/legacy_plantai_server_package/README.md b/tooling/legacy_plantai_server_package/README.md new file mode 100644 index 0000000..b948102 --- /dev/null +++ b/tooling/legacy_plantai_server_package/README.md @@ -0,0 +1,106 @@ +# PlantAI + RAG 服务端保护流程 + +本目录只放服务端构建、加密与授权工具。供应商私钥、私钥口令和版本主密钥不得复制到客户机。 + +## 新的阶段顺序 + +阶段 A 现在执行完整的代码硬化链： + +1. 从 `server.env` 读取 `PLANTAI_SOURCE` 和 `RAG_SOURCE`。 +2. Cython 生成 C 扩展。 +3. GCC 使用 `-fvisibility=hidden` 编译。 +4. `strip --strip-unneeded` 删除普通符号表和调试信息。 +5. ELF 工具把除 `PyInit_*` 外的已定义动态符号标记为 hidden，并逐个复核。 +6. 对完整目录计算摘要、写入 `.hardened-build.json`，再执行代表性导入/行为冒烟测试。 +7. 全部成功后，才把临时目录原子移动为 `PlantAI_compiled.new` 和 `RAG.new`。 + +阶段 C 会重新校验目录摘要、Python ABI、ELF 符号状态和冒烟测试。缺少构建标记、目录被修改、ABI 不一致或测试失败时都会停止，不会生成发行包。 + +## 0. 配置 + +只修改： + +```bash +cd /home/xzq/5090/PlanAI-cooperate/hsj-workspace/encryption-test/server/plantai_server_package +vi server.env +``` + +至少确认这些值： + +```bash +PLANTAI_SOURCE=/your/PlantAI/source +RAG_SOURCE=/your/RAG/source +PLANTAI_PYTHON=/path/to/python +VERSION=1.0.0 +``` + +`PLANTAI_PYTHON` 必须与客户运行环境的 Python 大版本、小版本和 ABI 一致，并已安装 Cython。 + +## A. 编译、隐藏符号、strip 和验证 + +```bash +cd /home/xzq/5090/PlanAI-cooperate/hsj-workspace/encryption-test/server/plantai_server_package +bash phase_a_compile_all.sh +``` + +验收： + +```bash +find ../PlantAI_compiled.new ../RAG.new -type f -name '*.c' +find ../PlantAI_compiled.new ../RAG.new -type f -name '*.so' | wc -l +python3 tools/audit_compiled_tree.py \ + --tree ../PlantAI_compiled.new --project PlantAI --require-marker +python3 tools/audit_compiled_tree.py \ + --tree ../RAG.new --project RAG --require-marker +``` + +第一条应无输出，两个审计命令应返回 `"ok": true`。阶段 A 为防止覆盖而拒绝已存在的 `.new` 目录；确认旧产物不再需要后，由操作者自行改名或移走。 + +## B. 初始化供应商密钥和版本主密钥 + +仅首次初始化供应商签名密钥： + +```bash +bash phase_b_init_supplier.sh +``` + +每个新 `VERSION` 都生成独立主密钥： + +```bash +bash phase_b_new_release_key.sh +``` + +## C. 构建联合发行包 + +阶段 C 默认读取 `server.env` 中的 `PLANTAI_COMPILED` 和 `RAG_COMPILED`；它们应指向阶段 A 的 `.new` 目录。 + +```bash +bash phase_c_build_dual_release.sh +``` + +脚本拒绝覆盖同版本的旧暂存目录或压缩包。确认旧文件后自行移走，再重新执行。成功后输出： + +```text +out/plantai-rag-release-.tar.gz +``` + +## D-F. 客户交付与授权 + +继续使用现有阶段脚本： + +```bash +bash phase_d_swtpm_test.sh # 仅软件 TPM 测试 +bash phase_e_issue_license.sh # 绑定客户设备并签发授权 +bash phase_f_acceptance.sh # 验收 +``` + +生产环境不要设置 `ALLOW_SOFTWARE_TPM_TEST=1`。代码的 `.so` 硬化提高逆向成本，但不是不可逆的保密边界；真正的机密仍应放在 TPM 解封后才出现的加密运行资产中。 + +## 常见失败 + +- `Output already exists`：为防覆盖而停止，人工检查并移走旧 `.new`。 +- `marker_missing`：输入不是由新阶段 A 生成，阶段 C 会拒绝。 +- `marker_mismatch`：阶段 A 后目录发生变化，必须重新构建并测试。 +- `python_abi_suffix_mismatch`：编译 Python 与当前校验/运行 Python ABI 不一致。 +- `visible_non_pyinit` 或 `symtab_present`：符号隐藏或 strip 未达到要求。 +- 冒烟测试失败：不要继续打包；先在源码或依赖环境中解决兼容问题。 diff --git a/tooling/legacy_plantai_server_package/bin/plantai_admin.py b/tooling/legacy_plantai_server_package/bin/plantai_admin.py new file mode 100755 index 0000000..c92b94c --- /dev/null +++ b/tooling/legacy_plantai_server_package/bin/plantai_admin.py @@ -0,0 +1,802 @@ +#!/usr/bin/env python3 +""" +PlantAI SERVER-SIDE admin tool. + +This file must NEVER be copied to a customer/client machine. It is the only +place in the whole system that can: + - hold / use the supplier Ed25519 PRIVATE key + - hold / use a plaintext asset master key (K_master) + - sign a license.json + - wrap K_master for a specific customer device + +The client only ever receives: a public key, a signed license.json, a +per-device wrapped asset key (asset.key.enc), and pre-encrypted asset blobs. +It never receives this script and never receives a private key or K_master. +""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import os +import secrets +import shutil +import subprocess +import tarfile +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey +from cryptography.hazmat.primitives.asymmetric import padding as _asym_padding +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.kdf.hkdf import HKDF + +DEFAULT_MODULES = [ + "identify", "phylogeny", "motif", "domain", "structure", "promoter", + "synteny", "interspecies_synteny", "karyotype", "combo", "ppi", + "expression", "pep", "rnaseq_qc", "rnaseq_align", "rnaseq_quant", + "rnaseq_de", "rnaseq_enrich", +] + + +# --------------------------------------------------------------------------- +# generic helpers +# --------------------------------------------------------------------------- +def now_iso() -> str: + return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds") + + +def canonical_bytes(data: Any) -> bytes: + return json.dumps(data, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def sha256_hex(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def write_json(path: Path, data: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +# --------------------------------------------------------------------------- +# signing key (private key lives ONLY on the server, in --private-key path) +# --------------------------------------------------------------------------- +def gen_keys(args: argparse.Namespace) -> None: + priv_path = Path(args.private_key) + pub_path = Path(args.public_key) + priv_path.parent.mkdir(parents=True, exist_ok=True) + if priv_path.exists() and not args.force: + raise SystemExit( + f"{priv_path} already exists. Refusing to overwrite an existing supplier " + f"signing key (this would invalidate every license/asset already issued " + f"with it). Pass --force only if you really intend to rotate the key." + ) + private_key = Ed25519PrivateKey.generate() + + # Resolve passphrase. None → unencrypted PEM (legacy / dev); + # bytes → PKCS#8 encrypted with the passphrase (production). + passphrase = _read_passphrase(args) + + if passphrase is not None: + encryption = serialization.BestAvailableEncryption(passphrase) + print("Encrypting private key with passphrase " + f"(source: {'env ' + getattr(args, 'key_passphrase_env', '') if getattr(args, 'key_passphrase_env', '') else 'file ' + getattr(args, 'key_passphrase_file', '')})") + else: + encryption = serialization.NoEncryption() + print("WARNING: Generating an unencrypted private key. " + "For production, use --key-passphrase-env or --key-passphrase-file " + "to encrypt the key at rest.") + + priv_path.write_bytes(private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=encryption, + )) + priv_path.chmod(0o600) + pub_path.write_bytes(private_key.public_key().public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + )) + print(f"Generated supplier keypair.\n private: {priv_path} (keep on server only, mode 600)\n public: {pub_path} (ship this to clients)") + + +def load_private(path: Path, password: bytes | None = None) -> Ed25519PrivateKey: + """Load the supplier Ed25519 private key. + + Tries `password` first if provided, then falls back to password=None + for backward compatibility with unencrypted keys. + """ + raw = path.read_bytes() + if password is not None: + try: + return serialization.load_pem_private_key(raw, password=password) + except (ValueError, TypeError, Exception): + pass + return serialization.load_pem_private_key(raw, password=None) + + +def _read_passphrase(args: argparse.Namespace) -> bytes | None: + """Resolve a passphrase from --key-passphrase-env or --key-passphrase-file. + + Returns None when neither is configured (meaning the key is unencrypted). + Exits with a clear error when the env var or file is configured but + missing / unreadable. + """ + env_var = getattr(args, "key_passphrase_env", None) or "" + file_path = getattr(args, "key_passphrase_file", None) or "" + + if env_var and file_path: + raise SystemExit( + "Specify --key-passphrase-env OR --key-passphrase-file, not both." + ) + if env_var: + val = os.environ.get(env_var, "") + if not val: + raise SystemExit( + f"Environment variable {env_var} is not set or empty. " + f"Set it to the private key passphrase." + ) + return val.encode("utf-8") + if file_path: + p = Path(file_path) + if not p.exists(): + raise SystemExit(f"Passphrase file does not exist: {p}") + if p.stat().st_mode & 0o077: + raise SystemExit( + f"Passphrase file {p} has overly permissive mode " + f"(should be 0600). Fix: chmod 600 {p}" + ) + return p.read_bytes().rstrip(b"\n\r") + return None + + +def sign_payload(private_key_path: Path, payload: dict[str, Any], + password: bytes | None = None) -> str: + unsigned = dict(payload) + unsigned.pop("signature", None) + signature = load_private(private_key_path, password).sign(canonical_bytes(unsigned)) + return base64.b64encode(signature).decode("ascii") + + +# --------------------------------------------------------------------------- +# master asset key (K_master) -- one per release VERSION, shared by all +# customers of that version, plaintext form NEVER leaves the server. +# --------------------------------------------------------------------------- +def gen_master_key(args: argparse.Namespace) -> None: + path = Path(args.output) + if path.exists() and not args.force: + raise SystemExit( + f"{path} already exists. Refusing to overwrite (this would break decrypt " + f"for every asset already built with it). Pass --force to rotate." + ) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(secrets.token_bytes(32)) + path.chmod(0o600) + print(f"Generated master asset key: {path} (server-only, mode 600)") + + +def derive_wrap_key(anchor_hash: str, salt: bytes) -> bytes: + return HKDF( + algorithm=hashes.SHA256(), + length=32, + salt=salt, + info=b"PlantAI K_master wrap v2", + ).derive(anchor_hash.encode("ascii")) + + +def _parse_tpms_attest_name(attestation_blob: bytes) -> bytes | None: + """Parse a TPMS_ATTEST (TPM_ST_ATTEST_CERTIFY) blob and extract the + certified object's Name (attested.certify.name). + + Every offset read is bounds-checked; the whole parse is wrapped so a + malformed or malicious blob can never crash the server — it just + returns None. + + TPM wire format (all fields big-endian): + UINT32 magic; // 4 bytes, must be 0xFF544347 + UINT16 type; // 2 bytes, must be 0x8017 + TPM2B_NAME qualifiedSigner; // UINT16 len + bytes + TPM2B_DATA extraData; // UINT16 len + bytes + TPMS_CLOCK_INFO clockInfo; // 20 bytes (8+4+4+4) + UINT64 firmwareVersion; // 8 bytes + TPMS_CERTIFY_INFO { // (only when type==0x8017) + TPM2B_NAME name; // UINT16 len + bytes <-- we want this + TPM2B_NAME qualifiedName; + } + """ + import struct + + def _safe_read_u16(data: bytes, off: int) -> tuple[int, int] | None: + if off + 2 > len(data): + return None + return struct.unpack(">H", data[off:off + 2])[0], off + 2 + + def _safe_skip_tpm2b(data: bytes, off: int) -> int | None: + r = _safe_read_u16(data, off) + if r is None: + return None + length, off = r + if off + length > len(data): + return None + return off + length + + try: + if len(attestation_blob) < 10: + return None + magic = struct.unpack(">I", attestation_blob[0:4])[0] + if magic != 0xFF544347: + return None + att_type = struct.unpack(">H", attestation_blob[4:6])[0] + if att_type != 0x8017: # TPM_ST_ATTEST_CERTIFY + return None + + offset = _safe_skip_tpm2b(attestation_blob, 6) # qualifiedSigner + if offset is None: + return None + offset = _safe_skip_tpm2b(attestation_blob, offset) # extraData + if offset is None: + return None + offset += 20 # clockInfo (UINT64+UINT32+UINT32+UINT32) + offset += 8 # firmwareVersion + if offset > len(attestation_blob): + return None + + # attested.certify.name + r = _safe_read_u16(attestation_blob, offset) + if r is None: + return None + name_len, offset = r + if offset + name_len > len(attestation_blob): + return None + return attestation_blob[offset:offset + name_len] + except Exception: + return None + + +def _parse_tpmt_public_rsa_numbers(tpmt_public: bytes) -> tuple[int, int] | None: + """Parse RSA modulus (n) and public exponent (e) from a marshaled + TPMT_PUBLIC structure (TPM_ALG_RSA). + + TPMT_PUBLIC layout for RSA keys: + TPMI_ALG_PUBLIC type; // UINT16 (0x0001 = RSA) + TPMI_ALG_HASH nameAlg; // UINT16 + TPMA_OBJECT objectAttributes; // UINT32 + TPM2B_DIGEST authPolicy; // UINT16 len + bytes + TPMS_RSA_PARMS parameters: + TPMT_SYM_DEF_OBJECT symmetric; // alg(2); + keyBits(2)+mode(2) if non-NULL + TPMT_RSA_SCHEME scheme; // alg(2); + hashAlg(2) if non-NULL + TPMI_RSA_KEY_BITS keyBits; // UINT16 + UINT32 exponent; // 0 = default 65537 + TPMS_RSA_UNIQUE unique: + TPM2B_PUBLIC_KEY_RSA n; // UINT16 len + bytes + + Returns (n, e) as Python ints, or None on parse failure. + """ + import struct + + def _safe_read_u16(data: bytes, off: int) -> tuple[int, int] | None: + if off + 2 > len(data): + return None + return struct.unpack(">H", data[off:off + 2])[0], off + 2 + + try: + if len(tpmt_public) < 10: + return None + + # type (must be TPM_ALG_RSA = 0x0001) + alg_type = struct.unpack(">H", tpmt_public[0:2])[0] + if alg_type != 0x0001: + return None + + # Skip: nameAlg (2) + objectAttributes (4) + authPolicy (2+len) + offset = 8 + r = _safe_read_u16(tpmt_public, offset) + if r is None: + return None + auth_len, offset = r + offset += auth_len + if offset > len(tpmt_public): + return None + + # symmetric: TPMT_SYM_DEF_OBJECT — alg(2), + keyBits(2)+mode(2) when non-NULL + r = _safe_read_u16(tpmt_public, offset) + if r is None: + return None + sym_alg, offset = r + if sym_alg != 0x0010: + if offset + 4 > len(tpmt_public): + return None + offset += 4 + + # scheme: TPMT_RSA_SCHEME — alg(2), + hashAlg(2) when non-NULL + r = _safe_read_u16(tpmt_public, offset) + if r is None: + return None + scheme, offset = r + if scheme != 0x0010: + if offset + 2 > len(tpmt_public): + return None + offset += 2 + + # keyBits (2) + exponent (4) + if offset + 6 > len(tpmt_public): + return None + key_bits = struct.unpack(">H", tpmt_public[offset:offset + 2])[0] + offset += 2 + exponent_raw = struct.unpack(">I", tpmt_public[offset:offset + 4])[0] + exponent = exponent_raw if exponent_raw != 0 else 65537 + offset += 4 + + # RSA modulus n: TPM2B_PUBLIC_KEY_RSA = UINT16 size + bytes + r = _safe_read_u16(tpmt_public, offset) + if r is None: + return None + n_size, offset = r + if offset + n_size > len(tpmt_public): + return None + n_bytes = tpmt_public[offset:offset + n_size] + n = int.from_bytes(n_bytes, "big") + + return (n, exponent) + except Exception: + return None + + +def verify_tpm_attestation(tpm_info: dict[str, Any]) -> tuple[bool, str]: + """Cryptographically verify the TPM attestation that proves the wrapping + key is TPM-resident. + + The client runs "tpm2_certify" during tpm2-setup: the TPM signs a + TPMS_ATTEST structure with a temporary certifying key. We verify: + 1. The signature is valid for the certifying key's public key. + 2. The attestation blob has the correct magic and type bytes. + 3. The TPM Name embedded in the attestation matches the Name computed + from the wrapping key's raw public area (TPMT_PUBLIC). + 4. The RSA modulus and exponent extracted from TPMT_PUBLIC match + those in public_key_pem. This is the critical cross-check that + prevents an attacker from submitting a real TPM attestation for + key A while asking us to wrap K_master with attacker-controlled + software key B. + + NOTE: Without a trusted EK/AK chain, a software TPM emulator could + still forge the entire attestation. For production, either verify the + TPM's EK certificate chain or provision TPM keys in a trusted factory + environment. + + Returns (is_valid, reason). + """ + attestation = tpm_info.get("attestation") + if not attestation: + return False, "No attestation data in fingerprint — the key cannot be proven TPM-resident" + + certifying_key_pem = attestation.get("certifying_key_pem") + if not certifying_key_pem: + return False, "Attestation is missing certifying_key_pem" + + attestation_blob_b64 = attestation.get("attestation_blob_b64") + signature_b64 = attestation.get("attestation_signature_b64") + if not attestation_blob_b64 or not signature_b64: + return False, "Attestation is missing blob or signature" + + try: + attestation_blob = base64.b64decode(attestation_blob_b64) + signature = base64.b64decode(signature_b64) + + certifying_pubkey = serialization.load_pem_public_key( + certifying_key_pem.encode() + ) + certifying_pubkey.verify( + signature, + attestation_blob, + _asym_padding.PKCS1v15(), + hashes.SHA256(), + ) + except InvalidSignature: + return False, "Attestation signature verification FAILED — the attestation blob does not match the certifying key" + except Exception as e: + return False, f"Attestation verification error: {e}" + + # Basic structural check: TPMS_ATTEST starts with magic (0xFF544347) + # and a 2-byte type field (TPM_ST_ATTEST_CERTIFY = 0x8017). + if len(attestation_blob) < 10: + return False, "Attestation blob is too short to be valid TPMS_ATTEST" + + magic = int.from_bytes(attestation_blob[0:4], "big") + if magic != 0xFF544347: + return False, f"Attestation magic is wrong (got 0x{magic:08X}, expected 0xFF544347)" + + # Check for human-readable attestation data + attest_json = attestation.get("attestation_json") + if attest_json: + att_type = attest_json.get("type", "") + if att_type != "TPM_ST_ATTEST_CERTIFY": + return False, f"Attestation type is {att_type}, expected TPM_ST_ATTEST_CERTIFY" + + # ---- Name binding: verify the attestation is about the submitted key ---- + # TPM Name = nameAlg || H(nameAlg, marshaled TPMT_PUBLIC). + # SHA-256 algorithm ID is 0x000B (TPM_ALG_SHA256). + _TPM_ALG_SHA256 = b"\x00\x0b" + tpmt_public_b64 = attestation.get("wrapkey_tpmt_public_b64") + if not tpmt_public_b64: + return False, "Attestation is missing wrapkey_tpmt_public_b64 — cannot bind attestation to the wrapping key" + + tpmt_public = base64.b64decode(tpmt_public_b64) + expected_name = _TPM_ALG_SHA256 + hashlib.sha256(tpmt_public).digest() + + attest_name = _parse_tpms_attest_name(attestation_blob) + if attest_name is None or attest_name != expected_name: + idx = attestation_blob.find(expected_name) + if idx >= 0: + attest_name = expected_name + if attest_name is None: + return False, "Could not parse attested Name from TPMS_ATTEST blob" + if attest_name != expected_name: + return False, ( + "TPM Name mismatch — the attestation was issued for a DIFFERENT key " + "than the one represented by wrapkey_tpmt_public_b64.\n" + f" Attested Name: {attest_name.hex()}\n" + f" Expected Name: {expected_name.hex()}" + ) + + # ---- RSA key cross-verification: TPMT_PUBLIC vs public_key_pem ---- + # This is the critical check that prevents the attack where an attacker + # submits a real TPM attestation + TPMT_PUBLIC from key A, but asks us + # to wrap K_master with a different software-generated RSA key B. + # + # We parse the RSA modulus and exponent from both TPMT_PUBLIC and + # public_key_pem, then require them to be identical. + public_key_pem = tpm_info.get("public_key_pem") + if not public_key_pem: + return False, "TPM info is missing public_key_pem" + + # Extract (n, e) from the PEM public key + try: + pem_pubkey = serialization.load_pem_public_key(public_key_pem.encode()) + pem_numbers = pem_pubkey.public_numbers() + except Exception as e: + return False, f"Could not parse public_key_pem: {e}" + + # Extract (n, e) from TPMT_PUBLIC + tpmt_numbers = _parse_tpmt_public_rsa_numbers(tpmt_public) + if tpmt_numbers is None: + return False, "Could not parse TPMT_PUBLIC — the blob may be malformed" + tpmt_n, tpmt_e = tpmt_numbers + + if pem_numbers.n != tpmt_n or pem_numbers.e != tpmt_e: + return False, ( + "RSA key mismatch — public_key_pem and wrapkey_tpmt_public_b64 " + "represent DIFFERENT RSA keys.\n" + f" PEM modulus: {pem_numbers.n}\n" + f" TPMT modulus: {tpmt_n}\n" + "The attestation proves a TPM key exists, but the key submitted " + "for wrapping (public_key_pem) is a different key. This is a " + "strong indicator of tampering — the client may be trying to " + "bypass hardware binding." + ) + + return True, "Attestation verified — wrapping key is cryptographically proven TPM-resident" + + +def wrap_key_with_tpm(tpm_pubkey_pem: str, master_key: bytes, + device_id: str) -> dict[str, Any]: + """Wrap K_master with the client's TPM public key (RSA-2048 OAEP-SHA256). + + Only the TPM chip that holds the corresponding non-exportable private key + can unwrap this. This provides real hardware binding — unlike the legacy + HKDF approach where all the key material is stored in the same file. + """ + pubkey = serialization.load_pem_public_key(tpm_pubkey_pem.encode()) + + ciphertext = pubkey.encrypt( + master_key, + _asym_padding.OAEP( + mgf=_asym_padding.MGF1(algorithm=hashes.SHA256()), + algorithm=hashes.SHA256(), + label=None, + ), + ) + + return { + "version": 3, + "algorithm": "RSA-2048-OAEP-SHA256", + "wrap_mode": "tpm-rsa-oaep-v1", + "device_id": device_id, + "tpm_public_key_sha256": sha256_hex(tpm_pubkey_pem.encode()), + "wrapped_key_b64": base64.b64encode(ciphertext).decode("ascii"), + "issued_at": now_iso(), + } + + +def wrap_key_for_device(args: argparse.Namespace) -> None: + """Wrap K_master so only the device that produced this fingerprint can + unwrap it. + + If the fingerprint includes a TPM public key (v3), uses RSA-OAEP wrapping + that binds K_master to the specific TPM chip. Without TPM, falls back to + the legacy HKDF-based wrapping which provides NO real hardware binding. + """ + master_key = Path(args.master_key).read_bytes() + fp = load_json(Path(args.fingerprint)) + + tpm_info = fp.get("tpm_public_key") + + if tpm_info: + # ---- v3: TPM-based wrapping (hardware-bound) ---- + print(f"TPM public key found in fingerprint. TPM key SHA-256: {tpm_info['public_key_sha256']}") + + # Cryptographically verify the TPM attestation that proves the key + # is truly hardware-resident (not a software-generated fake). + attest_ok, attest_reason = verify_tpm_attestation(tpm_info) + if attest_ok: + print(f" TPM attestation: VERIFIED — {attest_reason}") + else: + print(f" TPM attestation: WARNING — {attest_reason}") + print(" The wrapping key COULD be a software-generated RSA key") + print(" rather than a genuine non-exportable TPM key.") + print(" Ask the client to re-run 'plantai_client.py tpm2-setup'") + print(" to generate a proper attestation.") + if not getattr(args, "allow_unverified_tpm_key", False): + print(" Refusing to wrap. Re-run with --allow-unverified-tpm-key") + print(" to accept this risk, or have the client fix their TPM setup.") + raise SystemExit( + "TPM attestation verification failed and " + "--allow-unverified-tpm-key was not set." + ) + + print("Using TPM-based (RSA-2048-OAEP) wrapping.") + envelope = wrap_key_with_tpm( + tpm_info["public_key_pem"], + master_key, + fp["device_id"], + ) + elif getattr(args, "allow_legacy_wrap", False): + # ---- v2: Legacy HKDF-based wrapping (WEAK, explicit opt-in) ---- + print( + "WARNING: Using legacy HKDF wrapping (v2). " + "This provides NO hardware binding — anyone with the client " + "package can derive the unwrap key." + ) + anchor_hash = fp["core_id_hash"] + salt = secrets.token_bytes(16) + nonce = secrets.token_bytes(12) + wrap_key = derive_wrap_key(anchor_hash, salt) + ciphertext = AESGCM(wrap_key).encrypt( + nonce, master_key, anchor_hash.encode("ascii") + ) + envelope = { + "version": 2, + "algorithm": "AES-256-GCM", + "wrap_mode": "core-id-hkdf-aesgcm-v2", + "core_id_hash": anchor_hash, + "core_id_anchor": fp.get("core_id_anchor", ""), + "device_id": fp["device_id"], + "salt": base64.b64encode(salt).decode("ascii"), + "nonce": base64.b64encode(nonce).decode("ascii"), + "ciphertext": base64.b64encode(ciphertext).decode("ascii"), + "issued_at": now_iso(), + } + else: + raise SystemExit( + "No TPM public key in fingerprint — refusing to generate a v2 " + "(legacy HKDF) envelope by default.\n\n" + "v2 provides NO real hardware binding: anyone with the client " + "package can derive the unwrap key from data stored inside " + "asset.key.enc.\n\n" + "To proceed anyway (e.g. for a test VM without vTPM), re-run " + "with --allow-legacy-wrap. For production, the client must run " + "'plantai_client.py tpm2-setup' on a machine with TPM 2.0 and " + "re-submit their fingerprint." + ) + + write_json(Path(args.output), envelope) + print(f"Wrapped K_master for device_id={fp['device_id']} -> {args.output}") + + +# --------------------------------------------------------------------------- +# license issuance +# --------------------------------------------------------------------------- +def issue_license(args: argparse.Namespace) -> None: + fp = load_json(Path(args.fingerprint)) + + # Refuse to issue a license for a device with only a weak anchor + # (hostname) unless the operator explicitly acknowledges the risk. + if fp.get("core_id_anchor") == "hostname_weak" and not getattr(args, "allow_weak_anchor", False): + raise SystemExit( + "Device fingerprint has only a hostname_weak anchor. " + "This machine has no DMI product UUID, board serial, or disk " + "serial — the identity is trivial to spoof and does NOT provide " + "hardware binding. " + "If you are sure this is acceptable (e.g., a VM for testing), " + "re-run with --allow-weak-anchor." + ) + + license_data = { + "license_version": "2.0", + "product": "PlantAI", + "edition": args.edition, + "license_type": "permanent_local", + "customer_id": args.customer_id, + "customer_name": args.customer_name, + "device_id": fp["device_id"], + "serial_no": args.serial_no, + "issued_at": now_iso(), + "expires_at": None, + "modules": DEFAULT_MODULES, + "hardware_policy": { + "min_match_score": int(args.min_match_score), + "allow_warning_score": int(args.allow_warning_score), + "fingerprint_snapshot": fp["fingerprint_material"], + "core_id_hash": fp["core_id_hash"], + "core_id_anchor": fp.get("core_id_anchor", ""), + }, + "asset_policy": { + "asset_key_id": args.asset_key_id, + "release_version": args.release_version, + "min_wrap_version": 3 if fp.get("tpm_public_key") else 2, + "allowed_assets": args.allowed_assets, + }, + } + + # Bind the asset key envelope hash into the license so the client can + # verify that license.json and asset.key.enc were issued together. + if getattr(args, "asset_key_envelope", None): + envelope_path = Path(args.asset_key_envelope) + license_data["asset_policy"]["asset_key_sha256"] = sha256_hex(envelope_path.read_bytes()) + print(f"License bound to asset.key.enc hash: {license_data['asset_policy']['asset_key_sha256']}") + + # Bind TPM public key hash so the client can verify the TPM key hasn't + # been swapped between license issuance and runtime. + tpm_info = fp.get("tpm_public_key") + if tpm_info: + license_data["asset_policy"]["tpm_public_key_sha256"] = tpm_info["public_key_sha256"] + license_data["signature"] = sign_payload(Path(args.private_key), license_data, + _read_passphrase(args)) + write_json(Path(args.output), license_data) + print(f"Issued license for device_id={fp['device_id']} customer_id={args.customer_id} -> {args.output}") + + +# --------------------------------------------------------------------------- +# build-time asset encryption (uses K_master, has NOTHING to do with any +# individual customer's hardware -- runs once per release version) +# --------------------------------------------------------------------------- +def make_tar_zst(source: Path) -> bytes: + with tempfile.TemporaryDirectory() as tmp: + tar_path = Path(tmp) / "asset.tar" + with tarfile.open(tar_path, "w") as tf: + tf.add(source, arcname=source.name) + if shutil.which("zstd"): + out_path = Path(tmp) / "asset.tar.zst" + subprocess.check_call(["zstd", "-q", "-19", "-f", str(tar_path), "-o", str(out_path)]) + return out_path.read_bytes() + return tar_path.read_bytes() + + +def build_encrypt_asset(args: argparse.Namespace) -> None: + source = Path(args.source) + name = args.name or source.name + master_key = Path(args.master_key).read_bytes() + out_dir = Path(args.output_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + plaintext = make_tar_zst(source) + nonce = secrets.token_bytes(12) + aad = name.encode("utf-8") + ciphertext = AESGCM(master_key).encrypt(nonce, plaintext, aad) + enc_path = out_dir / f"{name}.enc" + enc_path.write_bytes(base64.b64encode(nonce) + b"\n" + ciphertext) + + manifest = { + "asset_package_version": "2.0", + "asset_name": name, + "asset_version": args.asset_version, + "release_version": args.release_version, + "algorithm": "AES-256-GCM", + "compression": "tar.zst" if shutil.which("zstd") else "tar", + "content_hash": sha256_hex(plaintext), + "cipher_hash": sha256_hex(ciphertext), + "created_at": now_iso(), + } + manifest["signature"] = sign_payload(Path(args.private_key), manifest, + _read_passphrase(args)) + write_json(out_dir / f"{name}.manifest.json", manifest) + print(f"Encrypted asset '{name}' with master key -> {enc_path}") + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- +def _add_key_passphrase_args(parser: argparse.ArgumentParser) -> None: + """Add --key-passphrase-env / --key-passphrase-file options to a subparser.""" + group = parser.add_argument_group("private key passphrase") + group.add_argument( + "--key-passphrase-env", default="", + help="Name of an environment variable holding the private key passphrase.", + ) + group.add_argument( + "--key-passphrase-file", default="", + help="Path to a file (mode 0600) containing the private key passphrase.", + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description="PlantAI server-side admin tool (never ship to clients).") + sub = parser.add_subparsers(dest="cmd", required=True) + + p = sub.add_parser("gen-keys", help="Generate the ONE supplier signing keypair (do this once).") + p.add_argument("--private-key", required=True) + p.add_argument("--public-key", required=True) + p.add_argument("--force", action="store_true") + _add_key_passphrase_args(p) + + p = sub.add_parser("gen-master-key", help="Generate a K_master for a release version (do this once per version).") + p.add_argument("--output", required=True) + p.add_argument("--force", action="store_true") + + p = sub.add_parser("build-encrypt-asset", help="Encrypt models/ or prompts/ with K_master at build time.") + p.add_argument("--source", required=True) + p.add_argument("--master-key", required=True) + p.add_argument("--private-key", required=True) + p.add_argument("--output-dir", required=True) + p.add_argument("--asset-version", default=datetime.now().strftime("%Y.%m.%d")) + p.add_argument("--release-version", required=True) + p.add_argument("--name", default="") + _add_key_passphrase_args(p) + + p = sub.add_parser("issue-license", help="Sign a license for one customer device (needs their fingerprint file).") + p.add_argument("--fingerprint", required=True, help="device_fingerprint.json received from the client") + p.add_argument("--private-key", required=True) + p.add_argument("--output", required=True) + p.add_argument("--customer-id", required=True) + p.add_argument("--customer-name", required=True) + p.add_argument("--serial-no", required=True) + p.add_argument("--edition", default="standard") + p.add_argument("--asset-key-id", default="ASSET-KEY-0001") + p.add_argument("--release-version", required=True) + p.add_argument("--min-match-score", default="60") + p.add_argument("--allow-warning-score", default="85") + p.add_argument("--allow-weak-anchor", action="store_true", help="Allow issuing a license to a device whose only identity anchor is hostname (trivial to spoof).") + p.add_argument("--asset-key-envelope", default=None, + help="Path to an already-generated asset.key.enc. If provided, its " + "SHA-256 is baked into the license so the client can verify " + "license and asset key were issued together.") + p.add_argument("--allowed-assets", nargs="*", default=["models", "prompts"], + help="Asset package names this license permits (default: models prompts).") + _add_key_passphrase_args(p) + + p = sub.add_parser("wrap-key-for-device", help="Wrap K_master for one customer device (needs their fingerprint file).") + p.add_argument("--fingerprint", required=True) + p.add_argument("--master-key", required=True) + p.add_argument("--output", required=True) + p.add_argument("--allow-legacy-wrap", action="store_true", + help="Allow v2 (HKDF, file-level) wrapping when NO TPM key is present " + "in the fingerprint. DANGEROUS — anyone with the client package " + "can derive the unwrap key. Only for VMs without vTPM.") + p.add_argument("--allow-unverified-tpm-key", action="store_true", + help="Accept a TPM public key whose attestation could not be " + "cryptographically verified. The wrapping will still use v3 " + "(RSA-OAEP), but the server cannot prove the key is truly " + "hardware-resident. Only for testing / development.") + + args = parser.parse_args() + { + "gen-keys": gen_keys, + "gen-master-key": gen_master_key, + "build-encrypt-asset": build_encrypt_asset, + "issue-license": issue_license, + "wrap-key-for-device": wrap_key_for_device, + }[args.cmd](args) + + +if __name__ == "__main__": + main() diff --git a/tooling/legacy_plantai_server_package/compile_plantai.sh b/tooling/legacy_plantai_server_package/compile_plantai.sh new file mode 100755 index 0000000..8ab18b0 --- /dev/null +++ b/tooling/legacy_plantai_server_package/compile_plantai.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +# Compile and harden a PlantAI source tree into Cython extension modules. +set -euo pipefail +umask 027 + +SOURCE_ROOT="${1:?Usage: $0 [python]}" +OUTPUT_DIR="${2:?Usage: $0 [python]}" +PYTHON="${3:-python3}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VISIBILITY_TOOL="${ELF_VISIBILITY_TOOL:-$SCRIPT_DIR/tools/elf_visibility.py}" + +ENTRY_SCRIPTS=( + "celery_app.py" + "tasks.py" + "check_cost.py" + "enable_persistent_memory.py" + "example_task_with_wechat.py" + "get_wechat_id.py" + "notify_integration.py" + "pipeline_backend_patch.py" + "session_storage.py" + "update_prompts.py" + "validate_js.py" + "wechat_bot.py" + "wechat_integration.py" + "wechat_service.py" + "wechat_task_bridge.py" + "test_langgraph_state.py" + "test_router.py" +) +CLI_ENTRY_SCRIPTS=( + "run_identify.py" + "run_phylogeny.py" +) + +SOURCE_ROOT="$(realpath "$SOURCE_ROOT")" +OUTPUT_DIR="$(realpath -m "$OUTPUT_DIR")" + +[[ -d "$SOURCE_ROOT" ]] || { echo "PlantAI source directory missing: $SOURCE_ROOT" >&2; exit 2; } +[[ -x "$PYTHON" ]] || { echo "Python is not executable: $PYTHON" >&2; exit 2; } +[[ -f "$VISIBILITY_TOOL" ]] || { echo "ELF visibility tool missing: $VISIBILITY_TOOL" >&2; exit 2; } +command -v gcc >/dev/null +command -v strip >/dev/null +command -v rsync >/dev/null + +case "$OUTPUT_DIR" in + "/"|"$(realpath -m "$HOME")"|"$SOURCE_ROOT") + echo "Unsafe output directory: $OUTPUT_DIR" >&2 + exit 2 + ;; +esac +[[ ! -e "$OUTPUT_DIR" ]] || { + echo "Output already exists; refusing to overwrite: $OUTPUT_DIR" >&2 + exit 2 +} + +mkdir -p "$OUTPUT_DIR" + +WHITELIST_EXT=( + "*.py" "*.yaml" "*.yml" "*.json" "*.md" "*.html" "*.js" "*.css" + "*.R" "*.sh" "*.hmm" "*.txt" "*.pem" "*.tsv" "*.csv" "*.db" + "/apps/web/vue3/static/*.png" +) +BLACKLIST=( + "__pycache__/" ".git/" ".codegraph/" "*.pyc" ".bak" "*.bak*" + "_backup_*" "vue3copy*" "workspace/" "/data/" "input/" "cost_logs/" + "memory_storage/" "rag_kb_store/" "*logs/" "temp/" "test*" "项目记录/" + "secrets/" "*.key" "*private*.pem" "*secret*.pem" +) + +RSYNC_RULES=() +for pattern in "${BLACKLIST[@]}"; do RSYNC_RULES+=(--exclude="$pattern"); done +for pattern in "${WHITELIST_EXT[@]}"; do RSYNC_RULES+=(--include="$pattern"); done +RSYNC_RULES+=(--include='*/' --exclude='*') + +echo "[PlantAI 1/5] Copying allowlisted runtime files" +for directory in apps gfa_core config scripts; do + if [[ -d "$SOURCE_ROOT/$directory" ]]; then + rsync -a -m "${RSYNC_RULES[@]}" \ + "$SOURCE_ROOT/$directory/" "$OUTPUT_DIR/$directory/" + fi +done +for relative in "${ENTRY_SCRIPTS[@]}"; do + if [[ -f "$SOURCE_ROOT/$relative" ]]; then + install -D -m 0750 "$SOURCE_ROOT/$relative" "$OUTPUT_DIR/$relative" + fi +done +for relative in "${CLI_ENTRY_SCRIPTS[@]}"; do + if [[ -f "$SOURCE_ROOT/apps/cli/$relative" ]]; then + install -D -m 0750 \ + "$SOURCE_ROOT/apps/cli/$relative" "$OUTPUT_DIR/apps/cli/$relative" + fi +done + +echo "[PlantAI 2/5] Discovering Python modules" +EXCLUDE_ARGS=() +for name in "${ENTRY_SCRIPTS[@]}"; do EXCLUDE_ARGS+=(-not -name "$name"); done +for name in "${CLI_ENTRY_SCRIPTS[@]}"; do EXCLUDE_ARGS+=(-not -name "$name"); done +EXCLUDE_ARGS+=(-not -name '__init__.py') +EXCLUDE_ARGS+=(-not -path "*/apps/web/vue3/main.py") +EXCLUDE_ARGS+=(-not -name "*[-]*.py") + +PY_FILES=() +while IFS= read -r -d '' path; do PY_FILES+=("$path"); done \ + < <(find "$OUTPUT_DIR" -type f -name '*.py' "${EXCLUDE_ARGS[@]}" -print0) +TOTAL="${#PY_FILES[@]}" +(( TOTAL > 0 )) || { echo "No PlantAI modules selected for compilation" >&2; exit 3; } + +INCLUDE_PY="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("INCLUDEPY") or "")')" +EXT_SUFFIX="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX") or ".so")')" +[[ -n "$INCLUDE_PY" ]] || { echo "Python INCLUDEPY is empty" >&2; exit 3; } + +echo "[PlantAI 3/5] Cython + GCC hidden visibility + strip ($TOTAL modules)" +COUNT=0 +for py_file in "${PY_FILES[@]}"; do + COUNT=$((COUNT + 1)) + directory="$(dirname "$py_file")" + module="$(basename "$py_file" .py)" + c_file="$directory/$module.c" + so_file="$directory/$module$EXT_SUFFIX" + echo " [$COUNT/$TOTAL] ${py_file#"$OUTPUT_DIR"/}" + + "$PYTHON" -m cython -3 \ + --directive always_allow_keywords=true,annotation_typing=False \ + --no-docstrings \ + "$py_file" -o "$c_file" + + gcc -shared -pthread -fPIC -O2 -fvisibility=hidden \ + -I"$INCLUDE_PY" \ + -o "$so_file" "$c_file" + + strip --strip-unneeded "$so_file" + "$PYTHON" -B "$VISIBILITY_TOOL" \ + --hide-non-pyinit --check "$so_file" >/dev/null + + rm -f -- "$c_file" "$py_file" +done + +echo "[PlantAI 3.5/5] Generating .py stubs for compiled modules" +"$PYTHON" -B "$SCRIPT_DIR/tools/generate_stubs.py" "$OUTPUT_DIR" + +echo "[PlantAI 4/5] Restoring package markers" +while IFS= read -r -d '' directory; do + if find "$directory" -maxdepth 1 -type f \( -name '*.so' -o -name '*.py' \) -print -quit | grep -q .; then + [[ -f "$directory/__init__.py" ]] || touch "$directory/__init__.py" + fi +done < <(find "$OUTPUT_DIR" -type d -print0) + +echo "[PlantAI 5/5] Final checks" +SO_COUNT="$(find "$OUTPUT_DIR" -type f -name '*.so' | wc -l)" +[[ "$SO_COUNT" -eq "$TOTAL" ]] || { + echo "PlantAI count mismatch: selected=$TOTAL produced=$SO_COUNT" >&2 + exit 4 +} +if find "$OUTPUT_DIR" -type f -name '*.c' -print -quit | grep -q .; then + echo "Generated C source remains in output" >&2 + exit 4 +fi + +echo "PlantAI compile+harden complete: modules=$SO_COUNT output=$OUTPUT_DIR" diff --git a/tooling/legacy_plantai_server_package/compile_qa.sh b/tooling/legacy_plantai_server_package/compile_qa.sh new file mode 100755 index 0000000..46290c2 --- /dev/null +++ b/tooling/legacy_plantai_server_package/compile_qa.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# Compile the QA V1.1.73 runtime surface into stripped Cython extensions. +set -euo pipefail +umask 027 + +SOURCE_ROOT="${1:?Usage: $0 [python]}" +OUTPUT_DIR="${2:?Usage: $0 [python]}" +PYTHON="${3:-python3}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VISIBILITY_TOOL="${ELF_VISIBILITY_TOOL:-$SCRIPT_DIR/tools/elf_visibility.py}" +ENTRY_SCRIPT="scripts/qa_v1_1_73_server.py" + +SOURCE_ROOT="$(realpath "$SOURCE_ROOT")" +OUTPUT_DIR="$(realpath -m "$OUTPUT_DIR")" + +[[ -d "$SOURCE_ROOT" ]] || { echo "QA source directory missing: $SOURCE_ROOT" >&2; exit 2; } +[[ -x "$PYTHON" ]] || { echo "Python is not executable: $PYTHON" >&2; exit 2; } +[[ -f "$VISIBILITY_TOOL" ]] || { echo "ELF visibility tool missing: $VISIBILITY_TOOL" >&2; exit 2; } +for command_name in gcc strip rsync; do command -v "$command_name" >/dev/null; done + +case "$OUTPUT_DIR" in + "/"|"$(realpath -m "$HOME")"|"$SOURCE_ROOT") + echo "Unsafe output directory: $OUTPUT_DIR" >&2 + exit 2 + ;; +esac +[[ ! -e "$OUTPUT_DIR" ]] || { + echo "Output already exists; refusing to overwrite: $OUTPUT_DIR" >&2 + exit 2 +} + +mkdir -p \ + "$OUTPUT_DIR/scripts" \ + "$OUTPUT_DIR/src/rag/qa_v1_1_73" \ + "$OUTPUT_DIR/src/rag/retrieval" \ + "$OUTPUT_DIR/src/rag/dual_kb_taxonomy_rag" \ + "$OUTPUT_DIR/configs/rag" + +echo "[QA 1/6] Copying the audited runtime surface" +install -m 0750 "$SOURCE_ROOT/$ENTRY_SCRIPT" "$OUTPUT_DIR/$ENTRY_SCRIPT" +rsync -a --include='*.py' --exclude='*' \ + "$SOURCE_ROOT/src/rag/qa_v1_1_73/" "$OUTPUT_DIR/src/rag/qa_v1_1_73/" +rsync -a --include='*.py' --exclude='*' \ + "$SOURCE_ROOT/src/rag/retrieval/" "$OUTPUT_DIR/src/rag/retrieval/" +install -m 0640 \ + "$SOURCE_ROOT/src/rag/dual_kb_taxonomy_rag/entity_alias.py" \ + "$OUTPUT_DIR/src/rag/dual_kb_taxonomy_rag/entity_alias.py" +for relative in \ + configs/rag/entity_alias.yaml \ + configs/rag/relation_taxonomy.yaml \ + pyproject.toml \ + requirements.txt \ + requirements-rag.txt +do + if [[ -f "$SOURCE_ROOT/$relative" ]]; then + install -D -m 0640 "$SOURCE_ROOT/$relative" "$OUTPUT_DIR/$relative" + fi +done + +# Namespace/package markers are intentionally left as minimal Python files. +touch \ + "$OUTPUT_DIR/scripts/__init__.py" \ + "$OUTPUT_DIR/src/__init__.py" \ + "$OUTPUT_DIR/src/rag/__init__.py" \ + "$OUTPUT_DIR/src/rag/dual_kb_taxonomy_rag/__init__.py" + +echo "[QA 2/6] Discovering Python modules" +PY_FILES=() +while IFS= read -r -d '' path; do PY_FILES+=("$path"); done \ + < <(find "$OUTPUT_DIR" -type f -name '*.py' \ + -not -path "*/$ENTRY_SCRIPT" \ + -not -name '__init__.py' -print0) +TOTAL="${#PY_FILES[@]}" +(( TOTAL > 0 )) || { echo "No QA modules selected for compilation" >&2; exit 3; } + +INCLUDE_PY="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("INCLUDEPY") or "")')" +EXT_SUFFIX="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX") or ".so")')" +[[ -n "$INCLUDE_PY" ]] || { echo "Python INCLUDEPY is empty" >&2; exit 3; } + +echo "[QA 3/6] Cython preflight ($TOTAL modules)" +COUNT=0 +CYTHON_FAILURES=0 +for py_file in "${PY_FILES[@]}"; do + COUNT=$((COUNT + 1)) + directory="$(dirname "$py_file")" + module="$(basename "$py_file" .py)" + c_file="$directory/$module.c" + echo " [$COUNT/$TOTAL] ${py_file#"$OUTPUT_DIR"/}" + if ! "$PYTHON" -m cython -3 \ + --directive always_allow_keywords=true,annotation_typing=False \ + --no-docstrings "$py_file" -o "$c_file"; then + CYTHON_FAILURES=$((CYTHON_FAILURES + 1)) + fi +done +(( CYTHON_FAILURES == 0 )) || { + echo "QA Cython preflight failed: modules=$CYTHON_FAILURES" >&2 + exit 3 +} + +echo "[QA 4/6] GCC hidden visibility + strip ($TOTAL modules)" +COUNT=0 +for py_file in "${PY_FILES[@]}"; do + COUNT=$((COUNT + 1)) + directory="$(dirname "$py_file")" + module="$(basename "$py_file" .py)" + c_file="$directory/$module.c" + so_file="$directory/$module$EXT_SUFFIX" + echo " [$COUNT/$TOTAL] ${py_file#"$OUTPUT_DIR"/}" + gcc -shared -pthread -fPIC -O2 -fvisibility=hidden \ + -I"$INCLUDE_PY" -o "$so_file" "$c_file" + strip --strip-unneeded "$so_file" + "$PYTHON" -B "$VISIBILITY_TOOL" \ + --hide-non-pyinit --check "$so_file" >/dev/null + rm -f -- "$c_file" "$py_file" +done + +echo "[QA 5/6] Restoring package markers" +while IFS= read -r -d '' directory; do + if find "$directory" -maxdepth 1 -type f \( -name '*.so' -o -name '*.py' \) -print -quit | grep -q .; then + [[ -f "$directory/__init__.py" ]] || touch "$directory/__init__.py" + fi +done < <(find "$OUTPUT_DIR" -type d -print0) + +echo "[QA 6/6] Final checks" +SO_COUNT="$(find "$OUTPUT_DIR" -type f -name '*.so' | wc -l)" +[[ "$SO_COUNT" -eq "$TOTAL" ]] || { + echo "QA count mismatch: selected=$TOTAL produced=$SO_COUNT" >&2 + exit 4 +} +if find "$OUTPUT_DIR" -type f -name '*.c' -print -quit | grep -q .; then + echo "Generated C source remains in output" >&2 + exit 4 +fi + +echo "QA compile+harden complete: modules=$SO_COUNT output=$OUTPUT_DIR" diff --git a/tooling/legacy_plantai_server_package/compile_rag.sh b/tooling/legacy_plantai_server_package/compile_rag.sh new file mode 100755 index 0000000..899b3d9 --- /dev/null +++ b/tooling/legacy_plantai_server_package/compile_rag.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +# Compile and harden a RAG source tree into Cython extension modules. +set -euo pipefail +umask 027 + +SOURCE_ROOT="${1:?Usage: $0 [python]}" +OUTPUT_DIR="${2:?Usage: $0 [python]}" +PYTHON="${3:-python3}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VISIBILITY_TOOL="${ELF_VISIBILITY_TOOL:-$SCRIPT_DIR/tools/elf_visibility.py}" + +ENTRY_SCRIPTS=( + "api/main.py" + "api/app.py" +) +API_KEEP_PY=( +) + +SOURCE_ROOT="$(realpath "$SOURCE_ROOT")" +OUTPUT_DIR="$(realpath -m "$OUTPUT_DIR")" + +[[ -d "$SOURCE_ROOT" ]] || { echo "RAG source directory missing: $SOURCE_ROOT" >&2; exit 2; } +[[ -x "$PYTHON" ]] || { echo "Python is not executable: $PYTHON" >&2; exit 2; } +[[ -f "$VISIBILITY_TOOL" ]] || { echo "ELF visibility tool missing: $VISIBILITY_TOOL" >&2; exit 2; } +command -v gcc >/dev/null +command -v strip >/dev/null +command -v rsync >/dev/null + +case "$OUTPUT_DIR" in + "/"|"$(realpath -m "$HOME")"|"$SOURCE_ROOT") + echo "Unsafe output directory: $OUTPUT_DIR" >&2 + exit 2 + ;; +esac +[[ ! -e "$OUTPUT_DIR" ]] || { + echo "Output already exists; refusing to overwrite: $OUTPUT_DIR" >&2 + exit 2 +} + +mkdir -p "$OUTPUT_DIR" + +WHITELIST_EXT=( + "*.py" "*.yaml" "*.yml" "*.json" "*.md" "*.html" "*.js" "*.css" + "*.sh" "*.txt" "*.pem" "*.csv" "*.tsv" "*.db" "*.sql" "*.toml" "*.lock" +) +BLACKLIST=( + "__pycache__/" ".git/" ".codegraph/" "*.pyc" ".bak" "*.bak*" + "node_modules/" "workspace/" "/data/" "input/" "output/" "outputs/" + "v15_output/" "neo4j/" "service_data/" "logs/" "temp/" "test*" + "*_test.py" "docs/" "evaluation/" "v14_learning/" "rag_kb_store/" + "protected_release/" "runtime_scripts/" "enviroment/" "frontend/" + "secrets/" "*.key" "*private*.pem" "*secret*.pem" + "rag/extractors/universal_gene_family_extractor.py" + "rag/hybrid_retriever_backup.py" + "rag/temporal/paper_metadata_extractor.py" +) + +RSYNC_RULES=() +for pattern in "${BLACKLIST[@]}"; do RSYNC_RULES+=(--exclude="$pattern"); done +for pattern in "${WHITELIST_EXT[@]}"; do RSYNC_RULES+=(--include="$pattern"); done +RSYNC_RULES+=(--include='*/' --exclude='*') + +echo "[RAG 1/5] Copying allowlisted runtime files" +for directory in api src config configs scripts tools prompts; do + if [[ -d "$SOURCE_ROOT/$directory" ]]; then + rsync -a -m "${RSYNC_RULES[@]}" \ + "$SOURCE_ROOT/$directory/" "$OUTPUT_DIR/$directory/" + fi +done +for relative in \ + pyproject.toml poetry.lock requirements.txt requirements-rag.txt \ + .env.example rag.env.example +do + if [[ -f "$SOURCE_ROOT/$relative" ]]; then + install -D -m 0640 "$SOURCE_ROOT/$relative" "$OUTPUT_DIR/$relative" + fi +done +for relative in "${ENTRY_SCRIPTS[@]}"; do + if [[ -f "$SOURCE_ROOT/$relative" ]]; then + install -D -m 0750 "$SOURCE_ROOT/$relative" "$OUTPUT_DIR/$relative" + fi +done + +echo "[RAG 2/5] Discovering Python modules" +EXCLUDE_ARGS=() +for relative in "${ENTRY_SCRIPTS[@]}"; do EXCLUDE_ARGS+=(-not -path "*/$relative"); done +for relative in "${API_KEEP_PY[@]}"; do EXCLUDE_ARGS+=(-not -path "*/$relative"); done +EXCLUDE_ARGS+=(-not -name '__init__.py') + +PY_FILES=() +while IFS= read -r -d '' path; do PY_FILES+=("$path"); done \ + < <(find "$OUTPUT_DIR" -type f -name '*.py' "${EXCLUDE_ARGS[@]}" -print0) +TOTAL="${#PY_FILES[@]}" +(( TOTAL > 0 )) || { echo "No RAG modules selected for compilation" >&2; exit 3; } + +INCLUDE_PY="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("INCLUDEPY") or "")')" +EXT_SUFFIX="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX") or ".so")')" +[[ -n "$INCLUDE_PY" ]] || { echo "Python INCLUDEPY is empty" >&2; exit 3; } + +echo "[RAG 3/6] Cython preflight ($TOTAL modules)" +COUNT=0 +CYTHON_FAILURES=0 +for py_file in "${PY_FILES[@]}"; do + COUNT=$((COUNT + 1)) + directory="$(dirname "$py_file")" + module="$(basename "$py_file" .py)" + c_file="$directory/$module.c" + echo " [$COUNT/$TOTAL] ${py_file#"$OUTPUT_DIR"/}" + + if ! "$PYTHON" -m cython -3 \ + --directive always_allow_keywords=true,annotation_typing=False \ + --no-docstrings \ + "$py_file" -o "$c_file"; then + CYTHON_FAILURES=$((CYTHON_FAILURES + 1)) + fi +done +(( CYTHON_FAILURES == 0 )) || { + echo "RAG Cython preflight failed: modules=$CYTHON_FAILURES" >&2 + exit 3 +} + +echo "[RAG 4/6] GCC hidden visibility + strip ($TOTAL modules)" +COUNT=0 +for py_file in "${PY_FILES[@]}"; do + COUNT=$((COUNT + 1)) + directory="$(dirname "$py_file")" + module="$(basename "$py_file" .py)" + c_file="$directory/$module.c" + so_file="$directory/$module$EXT_SUFFIX" + echo " [$COUNT/$TOTAL] ${py_file#"$OUTPUT_DIR"/}" + + [[ -f "$c_file" ]] || { + echo "Missing preflight C source: $c_file" >&2 + exit 3 + } + gcc -shared -pthread -fPIC -O2 -fvisibility=hidden \ + -I"$INCLUDE_PY" \ + -o "$so_file" "$c_file" + + strip --strip-unneeded "$so_file" + "$PYTHON" -B "$VISIBILITY_TOOL" \ + --hide-non-pyinit --check "$so_file" >/dev/null + + rm -f -- "$c_file" "$py_file" +done + +echo "[RAG 5/6] Restoring package markers" +while IFS= read -r -d '' directory; do + if find "$directory" -maxdepth 1 -type f \( -name '*.so' -o -name '*.py' \) -print -quit | grep -q .; then + [[ -f "$directory/__init__.py" ]] || touch "$directory/__init__.py" + fi +done < <(find "$OUTPUT_DIR" -type d -print0) + +echo "[RAG 6/6] Final checks" +SO_COUNT="$(find "$OUTPUT_DIR" -type f -name '*.so' | wc -l)" +[[ "$SO_COUNT" -eq "$TOTAL" ]] || { + echo "RAG count mismatch: selected=$TOTAL produced=$SO_COUNT" >&2 + exit 4 +} +if find "$OUTPUT_DIR" -type f -name '*.c' -print -quit | grep -q .; then + echo "Generated C source remains in output" >&2 + exit 4 +fi + +echo "RAG compile+harden complete: modules=$SO_COUNT output=$OUTPUT_DIR" diff --git a/tooling/legacy_plantai_server_package/issue_client.sh b/tooling/legacy_plantai_server_package/issue_client.sh new file mode 100755 index 0000000..0702888 --- /dev/null +++ b/tooling/legacy_plantai_server_package/issue_client.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Run this on your server, once per customer device, AFTER that customer has +# sent you their device_fingerprint.json (produced by the client package's +# install_plantai.sh fingerprint step). +# +# Usage: +# ./issue_client.sh \ +# --fingerprint /path/to/device_fingerprint.json \ +# --release-version 1.2.0 \ +# --customer-id CUS-0001 \ +# --customer-name "Some Customer Co" \ +# --serial-no PlantAI-0001 +# +# Produces: out/provision//license.json and asset.key.enc, +# packaged into out/provision/plantai-provision-.tar.gz +# which is the ONLY thing you send back to that customer. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +for cfg in "$SCRIPT_DIR/server.env"; do + [ -f "$cfg" ] && . "$cfg" +done + +SECRETS_DIR="${SECRETS_DIR:-$SCRIPT_DIR/secrets}" +ADMIN="$SCRIPT_DIR/bin/plantai_admin.py" +PYTHON="${PLANTAI_PYTHON:-python3}" + +KEY_PASSPHRASE_ARGS=() +if [ -n "${SUPPLIER_KEY_PASSPHRASE_ENV:-}" ] && [ -n "${SUPPLIER_KEY_PASSPHRASE_FILE:-}" ]; then + echo "Set only one of SUPPLIER_KEY_PASSPHRASE_ENV or SUPPLIER_KEY_PASSPHRASE_FILE" >&2 + exit 1 +elif [ -n "${SUPPLIER_KEY_PASSPHRASE_ENV:-}" ]; then + KEY_PASSPHRASE_ARGS=(--key-passphrase-env "$SUPPLIER_KEY_PASSPHRASE_ENV") +elif [ -n "${SUPPLIER_KEY_PASSPHRASE_FILE:-}" ]; then + KEY_PASSPHRASE_ARGS=(--key-passphrase-file "$SUPPLIER_KEY_PASSPHRASE_FILE") +fi + +FINGERPRINT="" +RELEASE_VERSION="" +CUSTOMER_ID="" +CUSTOMER_NAME="" +SERIAL_NO="" +EDITION="${EDITION:-standard}" +MIN_MATCH_SCORE="${MIN_MATCH_SCORE:-60}" +ALLOW_WARNING_SCORE="${ALLOW_WARNING_SCORE:-85}" +ALLOWED_ASSETS="${ALLOWED_ASSETS:-runtime-assets}" +ALLOW_UNVERIFIED_TPM_KEY="" + +while [ $# -gt 0 ]; do + case "$1" in + --fingerprint) FINGERPRINT="$2"; shift 2 ;; + --release-version) RELEASE_VERSION="$2"; shift 2 ;; + --customer-id) CUSTOMER_ID="$2"; shift 2 ;; + --customer-name) CUSTOMER_NAME="$2"; shift 2 ;; + --serial-no) SERIAL_NO="$2"; shift 2 ;; + --edition) EDITION="$2"; shift 2 ;; + --min-match-score) MIN_MATCH_SCORE="$2"; shift 2 ;; + --allow-warning-score) ALLOW_WARNING_SCORE="$2"; shift 2 ;; + --allowed-assets) ALLOWED_ASSETS="$2"; shift 2 ;; + --allow-unverified-tpm-key) ALLOW_UNVERIFIED_TPM_KEY="--allow-unverified-tpm-key"; shift ;; + *) echo "Unknown arg: $1" >&2; exit 1 ;; + esac +done + +for req in FINGERPRINT RELEASE_VERSION CUSTOMER_ID CUSTOMER_NAME SERIAL_NO; do + if [ -z "${!req}" ]; then + echo "Missing required --${req,,}" >&2 + exit 1 + fi +done + +MASTER_KEY="$SECRETS_DIR/master_keys/$RELEASE_VERSION.key" +if [ ! -f "$MASTER_KEY" ]; then + echo "No master key found for release $RELEASE_VERSION at $MASTER_KEY -- run build_release.sh first." >&2 + exit 1 +fi + +DEVICE_ID="$("$PYTHON" -c "import json,sys; print(json.load(open(sys.argv[1]))['device_id'])" "$FINGERPRINT")" +OUT_DIR="$SCRIPT_DIR/out/provision/$DEVICE_ID" +mkdir -p "$OUT_DIR" + +echo "[1/3] Log device registration" +mkdir -p "$SCRIPT_DIR/out/registry" +"$PYTHON" - "$FINGERPRINT" "$CUSTOMER_ID" "$DEVICE_ID" "$RELEASE_VERSION" >> "$SCRIPT_DIR/out/registry/devices.jsonl" <<'PYEOF' +import json, sys, datetime +fp_path, customer_id, device_id, release_version = sys.argv[1:5] +fp = json.load(open(fp_path)) +print(json.dumps({ + "at": datetime.datetime.now().astimezone().isoformat(timespec="seconds"), + "device_id": device_id, + "customer_id": customer_id, + "release_version": release_version, + "core_id_anchor": fp.get("core_id_anchor"), + "core_id_hash": fp.get("core_id_hash"), +}, ensure_ascii=False)) +PYEOF + +# IMPORTANT: wrap-key-for-device runs FIRST, then issue-license with +# --asset-key-envelope so the license contains a cryptographic binding +# (asset_key_sha256) to the exact asset.key.enc. The client will +# verify this hash at runtime to detect tampering or mismatched files. +echo "[2/3] Wrap this release's K_master for this device" +"$PYTHON" "$ADMIN" wrap-key-for-device \ + --fingerprint "$FINGERPRINT" \ + --master-key "$MASTER_KEY" \ + --output "$OUT_DIR/asset.key.enc" \ + $ALLOW_UNVERIFIED_TPM_KEY + +echo "[3/3] Issue signed license (bound to asset.key.enc hash)" +"$PYTHON" "$ADMIN" issue-license \ + --fingerprint "$FINGERPRINT" \ + --private-key "$SECRETS_DIR/supplier_private_key.pem" \ + --output "$OUT_DIR/license.json" \ + --customer-id "$CUSTOMER_ID" \ + --customer-name "$CUSTOMER_NAME" \ + --serial-no "$SERIAL_NO" \ + --edition "$EDITION" \ + --release-version "$RELEASE_VERSION" \ + --min-match-score "$MIN_MATCH_SCORE" \ + --allow-warning-score "$ALLOW_WARNING_SCORE" \ + --allowed-assets $ALLOWED_ASSETS \ + --asset-key-envelope "$OUT_DIR/asset.key.enc" \ + "${KEY_PASSPHRASE_ARGS[@]}" + +tar -C "$OUT_DIR" -czf "$SCRIPT_DIR/out/provision/plantai-provision-$DEVICE_ID.tar.gz" license.json asset.key.enc + +echo "Done." +echo "Send this file back to the customer: $SCRIPT_DIR/out/provision/plantai-provision-$DEVICE_ID.tar.gz" +echo "It contains only license.json + asset.key.enc -- no keys, no master key." diff --git a/tooling/legacy_plantai_server_package/phase_a_compile_all.sh b/tooling/legacy_plantai_server_package/phase_a_compile_all.sh new file mode 100755 index 0000000..ad122af --- /dev/null +++ b/tooling/legacy_plantai_server_package/phase_a_compile_all.sh @@ -0,0 +1,260 @@ +#!/usr/bin/env bash +# 阶段 A：从 server.env 指定的源码生成全新的、已硬化并验证的 .new 目录。 +set -euo pipefail +umask 027 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${SERVER_ENV_FILE:-$SCRIPT_DIR/server.env}" +[[ -f "$ENV_FILE" ]] || { echo "缺少配置文件: $ENV_FILE" >&2; exit 1; } +# shellcheck source=/dev/null +. "$ENV_FILE" + +: "${PLANTAI_SOURCE:?请在 server.env 中设置 PLANTAI_SOURCE}" +: "${RAG_SOURCE:?请在 server.env 中设置 RAG_SOURCE}" +: "${QA_SOURCE:?请在 server.env 中设置 QA_SOURCE}" + +TEST_ROOT="${TEST_ROOT:-$(cd "$SCRIPT_DIR/../.." && pwd)}" +SERVER_ROOT="${SERVER_ROOT:-$TEST_ROOT/server}" +PYTHON="${PLANTAI_PYTHON:-python3}" +QA_PYTHON="${QA_PYTHON:-python3}" +PLANTAI_OUTPUT="${PLANTAI_OUTPUT:-$SERVER_ROOT/PlantAI_compiled.new}" +RAG_OUTPUT="${RAG_OUTPUT:-$SERVER_ROOT/RAG.new}" +QA_OUTPUT="${QA_OUTPUT:-$SERVER_ROOT/RAG_QA.new}" +VERSION="${VERSION:-unversioned}" +AUDIT_DIR="${PHASE_A_AUDIT_DIR:-$SCRIPT_DIR/out/build-audit/$VERSION}" +AUDITOR="$SCRIPT_DIR/tools/audit_compiled_tree.py" +SMOKE="$SCRIPT_DIR/tools/smoke_dual_build.py" + +for command_name in realpath find gcc strip rsync; do + command -v "$command_name" >/dev/null || { + echo "缺少阶段 A 工具: $command_name" >&2 + exit 1 + } +done +for path in "$PLANTAI_SOURCE" "$RAG_SOURCE" "$QA_SOURCE"; do + [[ -d "$path" ]] || { echo "源码目录不存在: $path" >&2; exit 1; } +done +[[ -x "$PYTHON" ]] || { echo "Python 不可执行: $PYTHON" >&2; exit 1; } +[[ -x "$QA_PYTHON" ]] || { echo "QA Python 不可执行: $QA_PYTHON" >&2; exit 1; } +"$PYTHON" -c 'import Cython' >/dev/null || { + echo "当前 Python 环境未安装 Cython: $PYTHON" >&2 + exit 1 +} +"$QA_PYTHON" -c 'import Cython' >/dev/null || { + echo "QA Python 环境未安装 Cython: $QA_PYTHON" >&2 + exit 1 +} +for path in "$AUDITOR" "$SMOKE" "$SCRIPT_DIR/tools/elf_visibility.py" "$SCRIPT_DIR/compile_qa.sh"; do + [[ -f "$path" ]] || { echo "缺少阶段 A 脚本: $path" >&2; exit 1; } +done + +SERVER_ROOT="$(realpath "$SERVER_ROOT")" +PLANTAI_OUTPUT="$(realpath -m "$PLANTAI_OUTPUT")" +RAG_OUTPUT="$(realpath -m "$RAG_OUTPUT")" +QA_OUTPUT="$(realpath -m "$QA_OUTPUT")" +for output in "$PLANTAI_OUTPUT" "$RAG_OUTPUT" "$QA_OUTPUT"; do + case "$output" in + "$SERVER_ROOT"/*) ;; + *) echo "阶段 A 输出必须位于 SERVER_ROOT 下: $output" >&2; exit 1 ;; + esac +done +[[ "$PLANTAI_OUTPUT" != "$RAG_OUTPUT" && "$PLANTAI_OUTPUT" != "$QA_OUTPUT" && "$RAG_OUTPUT" != "$QA_OUTPUT" ]] || { + echo "PlantAI、RAG 与 QA 输出目录不能相同" >&2 + exit 1 +} + +PLANTAI_STAGE="$SERVER_ROOT/.PlantAI_compiled.new.staging.$$" +RAG_STAGE="$SERVER_ROOT/.RAG.new.staging.$$" +QA_STAGE="$SERVER_ROOT/.RAG_QA.new.staging.$$" +for stage in "$PLANTAI_STAGE" "$RAG_STAGE" "$QA_STAGE"; do + [[ ! -e "$stage" ]] || { echo "临时目录已存在: $stage" >&2; exit 1; } +done + +cleanup() { + for stage in "$PLANTAI_STAGE" "$RAG_STAGE" "$QA_STAGE"; do + case "$stage" in + "$SERVER_ROOT"/.*.staging."$$") rm -rf -- "$stage" ;; + esac + done +} +trap cleanup EXIT + +mkdir -p "$AUDIT_DIR" + +sync_plain_asset_tree() { + local source_root="$1" + local output_root="$2" + local asset_name="$3" + local project_name="$4" + local source_dir="$source_root/$asset_name" + local output_dir="$output_root/$asset_name" + + if [[ ! -d "$source_dir" ]]; then + echo " $project_name 无 $asset_name 目录，跳过" + return + fi + if find "$source_dir" -type f -name '*.py' -print -quit | grep -q .; then + echo "$project_name/$asset_name 含 Python 源码，拒绝作为明文运行资产复制: $source_dir" >&2 + exit 1 + fi + + mkdir -p "$output_dir" + rsync -a --delete "$source_dir/" "$output_dir/" + echo " 已同步 $project_name/$asset_name: $source_dir -> $output_dir" +} + +sync_plantai_runtime_assets() { + local output_root="$1" + local pfam_source="${BBA_PFAM_DIR:-$PLANTAI_SOURCE/data/hmm}" + local pfam_output="$output_root/data/hmm" + sync_plain_asset_tree "$PLANTAI_SOURCE" "$output_root" models PlantAI + sync_plain_asset_tree "$PLANTAI_SOURCE" "$output_root" prompts PlantAI + # 内置 Pfam 映射表（系统配置，非用户数据） + if [[ -d "$PLANTAI_SOURCE/data/rag_db" ]]; then + mkdir -p "$output_root/data/rag_db" + rsync -a "$PLANTAI_SOURCE/data/rag_db/" "$output_root/data/rag_db/" + echo " 已同步 data/rag_db" + fi + [[ -f "$pfam_source/Pfam-A.hmm" ]] || { + echo "缺少 PlantAI Pfam-A.hmm: $pfam_source/Pfam-A.hmm" >&2 + exit 1 + } + mkdir -p "$pfam_output" + rsync -a --delete "$pfam_source/" "$pfam_output/" + echo " 已同步 PlantAI Pfam-A: $pfam_source -> $pfam_output" +} + +sync_rag_runtime_assets() { + # RAG prompts 已由 compile_rag.sh 复制；其中的 Python prompt 会编译为 .so， + # 这里不能再次从源码覆盖，否则会把明文 .py 放回硬化产物。 + sync_plain_asset_tree "$RAG_SOURCE" "$1" models RAG +} + +if [[ -e "$PLANTAI_OUTPUT" ]]; then + [[ -d "$PLANTAI_OUTPUT" ]] || { + echo "PlantAI 产物路径已存在但不是目录: $PLANTAI_OUTPUT" >&2 + exit 1 + } + echo "[A/6] 验证并复用已完成的 PlantAI 产物" + "$PYTHON" -B "$AUDITOR" \ + --tree "$PLANTAI_OUTPUT" --project PlantAI --require-marker \ + | tee "$AUDIT_DIR/plantai-before-assets-elf.json" + sync_plantai_runtime_assets "$PLANTAI_OUTPUT" + "$PYTHON" -B "$AUDITOR" \ + --tree "$PLANTAI_OUTPUT" --project PlantAI --write-marker \ + | tee "$AUDIT_DIR/plantai-reuse-elf.json" + "$PYTHON" -B "$SMOKE" \ + --plantai "$PLANTAI_OUTPUT" \ + --python "$PYTHON" \ + --output "$AUDIT_DIR/plantai-smoke.json" + "$PYTHON" -B "$AUDITOR" \ + --tree "$PLANTAI_OUTPUT" --project PlantAI --require-marker >/dev/null +else + echo "[A/6] 编译并硬化 PlantAI" + bash "$SCRIPT_DIR/compile_plantai.sh" "$PLANTAI_SOURCE" "$PLANTAI_STAGE" "$PYTHON" + sync_plantai_runtime_assets "$PLANTAI_STAGE" + + echo "[A/6] 审计并验证 PlantAI" + "$PYTHON" -B "$AUDITOR" \ + --tree "$PLANTAI_STAGE" --project PlantAI --write-marker \ + | tee "$AUDIT_DIR/plantai-elf.json" + "$PYTHON" -B "$SMOKE" \ + --plantai "$PLANTAI_STAGE" \ + --python "$PYTHON" \ + --output "$AUDIT_DIR/plantai-smoke.json" + "$PYTHON" -B "$AUDITOR" \ + --tree "$PLANTAI_STAGE" --project PlantAI --require-marker >/dev/null + + echo "[A/6] 原子发布 PlantAI；后续 RAG 失败时可直接复用" + mv -- "$PLANTAI_STAGE" "$PLANTAI_OUTPUT" +fi + +if [[ -e "$RAG_OUTPUT" ]]; then + [[ -d "$RAG_OUTPUT" ]] || { + echo "RAG 产物路径已存在但不是目录: $RAG_OUTPUT" >&2 + exit 1 + } + echo "[A/6] 验证并复用已完成的 RAG 产物" + "$PYTHON" -B "$AUDITOR" \ + --tree "$RAG_OUTPUT" --project RAG --require-marker \ + | tee "$AUDIT_DIR/rag-before-assets-elf.json" + sync_rag_runtime_assets "$RAG_OUTPUT" + "$PYTHON" -B "$AUDITOR" \ + --tree "$RAG_OUTPUT" --project RAG --write-marker \ + | tee "$AUDIT_DIR/rag-reuse-elf.json" + "$PYTHON" -B "$SMOKE" \ + --rag "$RAG_OUTPUT" \ + --python "$PYTHON" \ + --output "$AUDIT_DIR/rag-smoke.json" + "$PYTHON" -B "$AUDITOR" \ + --tree "$RAG_OUTPUT" --project RAG --require-marker >/dev/null +else + echo "[A/6] 编译并硬化 RAG" + bash "$SCRIPT_DIR/compile_rag.sh" "$RAG_SOURCE" "$RAG_STAGE" "$PYTHON" + sync_rag_runtime_assets "$RAG_STAGE" + + echo "[A/6] 审计并验证 RAG" + "$PYTHON" -B "$AUDITOR" \ + --tree "$RAG_STAGE" --project RAG --write-marker \ + | tee "$AUDIT_DIR/rag-elf.json" + "$PYTHON" -B "$SMOKE" \ + --rag "$RAG_STAGE" \ + --python "$PYTHON" \ + --output "$AUDIT_DIR/rag-smoke.json" + "$PYTHON" -B "$AUDITOR" \ + --tree "$RAG_STAGE" --project RAG --require-marker >/dev/null + + echo "[A/6] 原子发布 RAG" + mv -- "$RAG_STAGE" "$RAG_OUTPUT" +fi + +if [[ -e "$QA_OUTPUT" ]]; then + [[ -d "$QA_OUTPUT" ]] || { + echo "QA 产物路径已存在但不是目录: $QA_OUTPUT" >&2 + exit 1 + } + echo "[A/6] 验证并复用已完成的 QA 产物" + "$QA_PYTHON" -B "$AUDITOR" \ + --tree "$QA_OUTPUT" --project QA --require-marker \ + | tee "$AUDIT_DIR/qa-reuse-elf.json" + "$QA_PYTHON" -B "$SMOKE" \ + --qa "$QA_OUTPUT" \ + --qa-python "$QA_PYTHON" \ + --output "$AUDIT_DIR/qa-smoke.json" +else + echo "[A/6] 编译并硬化 QA" + bash "$SCRIPT_DIR/compile_qa.sh" "$QA_SOURCE" "$QA_STAGE" "$QA_PYTHON" + + echo "[A/6] 审计并验证 QA" + "$QA_PYTHON" -B "$AUDITOR" \ + --tree "$QA_STAGE" --project QA --write-marker \ + | tee "$AUDIT_DIR/qa-elf.json" + "$QA_PYTHON" -B "$SMOKE" \ + --qa "$QA_STAGE" \ + --qa-python "$QA_PYTHON" \ + --output "$AUDIT_DIR/qa-smoke.json" + "$QA_PYTHON" -B "$AUDITOR" \ + --tree "$QA_STAGE" --project QA --require-marker >/dev/null + + echo "[A/6] 原子发布 QA" + mv -- "$QA_STAGE" "$QA_OUTPUT" +fi + +echo "[A/6] 运行最终三项目代表性导入与行为冒烟测试" +"$PYTHON" -B "$SMOKE" \ + --plantai "$PLANTAI_OUTPUT" \ + --rag "$RAG_OUTPUT" \ + --qa "$QA_OUTPUT" \ + --python "$PYTHON" \ + --qa-python "$QA_PYTHON" \ + --output "$AUDIT_DIR/smoke.json" + +plantai_so="$(find "$PLANTAI_OUTPUT" -type f -name '*.so' | wc -l)" +rag_so="$(find "$RAG_OUTPUT" -type f -name '*.so' | wc -l)" +qa_so="$(find "$QA_OUTPUT" -type f -name '*.so' | wc -l)" +echo "阶段 A 完成：PlantAI .so=$plantai_so，RAG .so=$rag_so，QA .so=$qa_so" +echo "产物：$PLANTAI_OUTPUT" +echo "产物：$RAG_OUTPUT" +echo "产物：$QA_OUTPUT" +echo "审计：$AUDIT_DIR" +echo "未替换现有运行目录；阶段 C 将只接受带有效构建标记的产物。" diff --git a/tooling/legacy_plantai_server_package/phase_b_init_keys.sh b/tooling/legacy_plantai_server_package/phase_b_init_keys.sh new file mode 100755 index 0000000..66d5915 --- /dev/null +++ b/tooling/legacy_plantai_server_package/phase_b_init_keys.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# 阶段 B：初始化供应商签名密钥与当前版本 K_master；已存在时绝不覆盖。 +set -euo pipefail +umask 077 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +[ -f "$SCRIPT_DIR/server.env" ] && . "$SCRIPT_DIR/server.env" +VERSION="${VERSION:-1.0.0}" +PYTHON="${PLANTAI_PYTHON:-python3}" +SECRETS_DIR="${SECRETS_DIR:-$SCRIPT_DIR/secrets}" +PASS_FILE="${SUPPLIER_KEY_PASSPHRASE_FILE:-$SECRETS_DIR/private_key.pass}" +ADMIN="$SCRIPT_DIR/bin/plantai_admin.py" + +[[ "$VERSION" =~ ^[A-Za-z0-9._-]+$ ]] && [[ "$VERSION" != *..* ]] || { echo "非法 VERSION: $VERSION" >&2; exit 1; } +[ -x "$PYTHON" ] || { echo "Python 不可执行: $PYTHON" >&2; exit 1; } +mkdir -p "$SECRETS_DIR/master_keys" "$SCRIPT_DIR/out" +chmod 700 "$SECRETS_DIR" "$SECRETS_DIR/master_keys" "$SCRIPT_DIR/out" + +if [ ! -f "$PASS_FILE" ]; then + "$PYTHON" -c 'import secrets; print(secrets.token_urlsafe(48))' > "$PASS_FILE" + chmod 600 "$PASS_FILE" + echo "已生成随机私钥口令文件：$PASS_FILE" + echo "请立即离线备份 secrets/；丢失口令或私钥将无法继续为已有客户签发。" +fi +[ "$(stat -c '%a' "$PASS_FILE")" = 600 ] || { echo "口令文件权限必须为 600: $PASS_FILE" >&2; exit 1; } + +if [ ! -f "$SECRETS_DIR/supplier_private_key.pem" ]; then + "$PYTHON" "$ADMIN" gen-keys \ + --private-key "$SECRETS_DIR/supplier_private_key.pem" \ + --public-key "$SECRETS_DIR/supplier_public_key.pem" \ + --key-passphrase-file "$PASS_FILE" +else + echo "复用现有供应商私钥（不会覆盖）" +fi + +MASTER_KEY="$SECRETS_DIR/master_keys/$VERSION.key" +if [ ! -f "$MASTER_KEY" ]; then + "$PYTHON" "$ADMIN" gen-master-key --output "$MASTER_KEY" +else + echo "复用版本 $VERSION 的 K_master（不会覆盖）" +fi + +chmod 600 "$SECRETS_DIR/supplier_private_key.pem" "$MASTER_KEY" "$PASS_FILE" +echo "阶段 B 完成：VERSION=$VERSION" +sha256sum "$SECRETS_DIR/supplier_public_key.pem" diff --git a/tooling/legacy_plantai_server_package/phase_c_build_dual_release.sh b/tooling/legacy_plantai_server_package/phase_c_build_dual_release.sh new file mode 100755 index 0000000..6e58f28 --- /dev/null +++ b/tooling/legacy_plantai_server_package/phase_c_build_dual_release.sh @@ -0,0 +1,272 @@ +#!/usr/bin/env bash +# 阶段 C：只使用阶段 A 已验证的三项目产物，构建并加密联合发行包。 +set -euo pipefail +umask 077 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${SERVER_ENV_FILE:-$SCRIPT_DIR/server.env}" +[[ -f "$ENV_FILE" ]] || { echo "缺少配置文件: $ENV_FILE" >&2; exit 1; } +# shellcheck source=/dev/null +. "$ENV_FILE" + +TEST_ROOT="${TEST_ROOT:-$(cd "$SCRIPT_DIR/../.." && pwd)}" +SERVER_ROOT="${SERVER_ROOT:-$TEST_ROOT/server}" +VERSION="${VERSION:-1.0.0}" +PYTHON="${PLANTAI_PYTHON:-python3}" +QA_PYTHON="${QA_PYTHON:-python3}" +SECRETS_DIR="${SECRETS_DIR:-$SCRIPT_DIR/secrets}" +PASS_FILE="${SUPPLIER_KEY_PASSPHRASE_FILE:-$SECRETS_DIR/private_key.pass}" +PLANTAI_COMPILED="${PLANTAI_COMPILED:-${PLANTAI_OUTPUT:-$SERVER_ROOT/PlantAI_compiled.new}}" +RAG_COMPILED="${RAG_COMPILED:-${RAG_OUTPUT:-$SERVER_ROOT/RAG.new}}" +QA_COMPILED="${QA_COMPILED:-${QA_OUTPUT:-$SERVER_ROOT/RAG_QA.new}}" +QA_QDRANT_STORAGE_SOURCE="${QA_QDRANT_STORAGE_SOURCE:?请设置 QA_QDRANT_STORAGE_SOURCE}" +QA_NEO4J_DATA_SOURCE="${QA_NEO4J_DATA_SOURCE:?请设置 QA_NEO4J_DATA_SOURCE}" +QA_RERANKER_MODEL_SOURCE="${QA_RERANKER_MODEL_SOURCE:?请设置 QA_RERANKER_MODEL_SOURCE}" +OLLAMA_MODELS_SOURCE="${OLLAMA_MODELS_SOURCE:?请设置 OLLAMA_MODELS_SOURCE}" +QA_OLLAMA_GENERATION_MODEL="${QA_OLLAMA_GENERATION_MODEL:-qwen2.5:14b-instruct}" +QA_OLLAMA_EMBED_MODEL="${QA_OLLAMA_EMBED_MODEL:-bge-m3:567m}" +OUT_DIR="${OUT_DIR:-$SCRIPT_DIR/out/$VERSION}" +OUT_ROOT="$SCRIPT_DIR/out" +ADMIN="$SCRIPT_DIR/bin/plantai_admin.py" +AUDITOR="$SCRIPT_DIR/tools/audit_compiled_tree.py" +SMOKE="$SCRIPT_DIR/tools/smoke_dual_build.py" +OLLAMA_EXPORTER="$SCRIPT_DIR/tools/export_ollama_models.py" +MANIFEST_BUILDER="$SCRIPT_DIR/tools/build_delivery_manifest.py" + +[[ "$VERSION" =~ ^[A-Za-z0-9._-]+$ ]] && [[ "$VERSION" != *..* ]] || { + echo "非法 VERSION: $VERSION" >&2 + exit 1 +} +for command_name in realpath rsync tar sha256sum sort xargs; do + command -v "$command_name" >/dev/null || { + echo "缺少阶段 C 工具: $command_name" >&2 + exit 1 + } +done +for path in \ + "$PLANTAI_COMPILED" \ + "$RAG_COMPILED" \ + "$QA_COMPILED" \ + "$PLANTAI_COMPILED/models" \ + "$PLANTAI_COMPILED/prompts" \ + "$RAG_COMPILED/prompts" \ + "$QA_QDRANT_STORAGE_SOURCE" \ + "$QA_NEO4J_DATA_SOURCE" \ + "$QA_RERANKER_MODEL_SOURCE" \ + "$OLLAMA_MODELS_SOURCE" \ + "$SECRETS_DIR/supplier_private_key.pem" \ + "$SECRETS_DIR/supplier_public_key.pem" \ + "$SECRETS_DIR/master_keys/$VERSION.key" \ + "$PASS_FILE" \ + "$ADMIN" \ + "$AUDITOR" \ + "$SMOKE" \ + "$OLLAMA_EXPORTER" \ + "$MANIFEST_BUILDER" +do + [[ -e "$path" ]] || { echo "缺少阶段 C 输入: $path" >&2; exit 1; } +done +[[ -x "$PYTHON" ]] || { echo "Python 不可执行: $PYTHON" >&2; exit 1; } +[[ -x "$QA_PYTHON" ]] || { echo "QA Python 不可执行: $QA_PYTHON" >&2; exit 1; } + +OUT_ROOT="$(realpath -m "$OUT_ROOT")" +OUT_DIR="$(realpath -m "$OUT_DIR")" +case "$OUT_DIR" in + "$OUT_ROOT"/*) ;; + *) echo "OUT_DIR 必须位于 $OUT_ROOT 下" >&2; exit 1 ;; +esac + +PAYLOAD="$OUT_DIR/release" +ARCHIVE="$OUT_ROOT/plantai-rag-release-$VERSION.tar.gz" +DATA_ARCHIVE="$OUT_ROOT/plantai-rag-runtime-data-$VERSION.tar" +DATA_ARCHIVE_TMP="$OUT_ROOT/.plantai-rag-runtime-data-$VERSION.tar.$$" +echo "[C/6] 验证阶段 A 构建标记与 ELF 状态" +"$PYTHON" -B "$AUDITOR" \ + --tree "$PLANTAI_COMPILED" --project PlantAI --require-marker +"$PYTHON" -B "$AUDITOR" \ + --tree "$RAG_COMPILED" --project RAG --require-marker +"$QA_PYTHON" -B "$AUDITOR" \ + --tree "$QA_COMPILED" --project QA --require-marker + +echo "[C/6] 再次运行代表性导入与行为冒烟测试" +mkdir -p "$OUT_DIR/validation" +"$PYTHON" -B "$SMOKE" \ + --plantai "$PLANTAI_COMPILED" \ + --rag "$RAG_COMPILED" \ + --qa "$QA_COMPILED" \ + --python "$PYTHON" \ + --qa-python "$QA_PYTHON" \ + --output "$OUT_DIR/validation/smoke.json" +if [[ "${PHASE_C_PREFLIGHT_ONLY:-0}" == "1" ]]; then + "$PYTHON" -B "$OLLAMA_EXPORTER" \ + --source "$OLLAMA_MODELS_SOURCE" \ + --output "$OUT_DIR/validation/ollama-models-unused" \ + --model "$QA_OLLAMA_GENERATION_MODEL" \ + --model "$QA_OLLAMA_EMBED_MODEL" \ + --check-only + echo "阶段 C 预检完成；未复制或加密发行数据。" + exit 0 +fi +[[ ! -e "$PAYLOAD" ]] || { + echo "发行暂存目录已存在；拒绝覆盖: $PAYLOAD" >&2 + exit 1 +} +[[ ! -e "$ARCHIVE" ]] || { + echo "发行包已存在；拒绝覆盖: $ARCHIVE" >&2 + exit 1 +} +[[ ! -e "$DATA_ARCHIVE" ]] || { + echo "运行数据包已存在；拒绝覆盖: $DATA_ARCHIVE" >&2 + exit 1 +} +[[ ! -e "$DATA_ARCHIVE.sha256" ]] || { + echo "运行数据校验文件已存在；拒绝覆盖: $DATA_ARCHIVE.sha256" >&2 + exit 1 +} + +echo "[C/6] 复制三项目编译产物" +mkdir -p \ + "$PAYLOAD/app/PlantAI_compiled" \ + "$PAYLOAD/app/RAG" \ + "$PAYLOAD/app/QA" \ + "$PAYLOAD/assets" \ + "$PAYLOAD/config" +rsync -a \ + --exclude='.hardened-build.json' \ + --exclude='__pycache__/' --exclude='*.pyc' \ + --exclude='models/' --exclude='prompts/' --exclude='data/hmm/' --exclude='rag_kb_store/' \ + --exclude='cost_logs/' --exclude='memory_storage/' \ + --exclude='workspace/' --exclude='logs/' \ + "$PLANTAI_COMPILED/" "$PAYLOAD/app/PlantAI_compiled/" +rsync -a \ + --exclude='.hardened-build.json' \ + --exclude='__pycache__/' --exclude='*.pyc' \ + --exclude='models/' --exclude='prompts/' --exclude='rag_kb_store/' \ + --exclude='qdrant_docker/' --exclude='logs/' \ + "$RAG_COMPILED/" "$PAYLOAD/app/RAG/" +rsync -a \ + --exclude='.hardened-build.json' \ + --exclude='__pycache__/' --exclude='*.pyc' \ + --exclude='models/' --exclude='logs/' --exclude='data/' \ + "$QA_COMPILED/" "$PAYLOAD/app/QA/" +install -m 0755 "$SCRIPT_DIR/templates/start_all_client.sh" "$PAYLOAD/app/start_all.sh" + +echo "[C/6] 从阶段 A 已验证产物组装运行资产" +ASSET_STAGE="$(mktemp -d "$OUT_ROOT/runtime-assets.XXXXXX")" +cleanup() { + case "$ASSET_STAGE" in + "$OUT_ROOT"/runtime-assets.*) rm -rf -- "$ASSET_STAGE" ;; + esac + if [[ -n "${DATA_STAGE:-}" ]]; then + case "$DATA_STAGE" in + "$OUT_ROOT"/runtime-data.*) rm -rf -- "$DATA_STAGE" ;; + esac + fi + [[ ! -f "$DATA_ARCHIVE_TMP" ]] || rm -f -- "$DATA_ARCHIVE_TMP" +} +trap cleanup EXIT +mkdir -p \ + "$ASSET_STAGE/runtime-assets/PlantAI_compiled" \ + "$ASSET_STAGE/runtime-assets/RAG" +cp -a \ + "$PLANTAI_COMPILED/models" \ + "$ASSET_STAGE/runtime-assets/PlantAI_compiled/models" +cp -a \ + "$PLANTAI_COMPILED/prompts" \ + "$ASSET_STAGE/runtime-assets/PlantAI_compiled/prompts" + +[[ -f "$PLANTAI_COMPILED/data/hmm/Pfam-A.hmm" ]] || { + echo "阶段 A 产物缺少 PlantAI Pfam-A.hmm: $PLANTAI_COMPILED/data/hmm/Pfam-A.hmm" >&2 + exit 1 +} +mkdir -p "$ASSET_STAGE/runtime-assets/PlantAI_compiled/data" +cp -a \ + "$PLANTAI_COMPILED/data/hmm" \ + "$ASSET_STAGE/runtime-assets/PlantAI_compiled/data/hmm" +[[ ! -d "$RAG_COMPILED/models" ]] || \ + cp -a "$RAG_COMPILED/models" "$ASSET_STAGE/runtime-assets/RAG/models" +cp -a \ + "$RAG_COMPILED/prompts" \ + "$ASSET_STAGE/runtime-assets/RAG/prompts" + +assert_volume_not_live() { + local source="$1" + command -v docker >/dev/null || return 0 + local container_id mounted + while IFS= read -r container_id; do + [[ -n "$container_id" ]] || continue + while IFS= read -r mounted; do + [[ -n "$mounted" ]] || continue + if [[ "$(realpath -m "$mounted")" == "$(realpath -m "$source")" ]]; then + echo "拒绝打包运行中的数据库卷: $source（容器 $container_id）" >&2 + echo "请先停止使用该卷的容器，完成阶段 C 后再启动，以保证数据一致性。" >&2 + exit 1 + fi + done < <(docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$container_id") + done < <(docker ps -q) +} +assert_volume_not_live "$QA_QDRANT_STORAGE_SOURCE" +assert_volume_not_live "$QA_NEO4J_DATA_SOURCE" + +DATA_STAGE="$(mktemp -d "$OUT_ROOT/runtime-data.XXXXXX")" +mkdir -p \ + "$DATA_STAGE/runtime-assets/QA/models" \ + "$DATA_STAGE/runtime-assets/qdrant" \ + "$DATA_STAGE/runtime-assets/neo4j" +ln -s "$QA_RERANKER_MODEL_SOURCE" \ + "$DATA_STAGE/runtime-assets/QA/models/bge-reranker-base" +ln -s "$QA_QDRANT_STORAGE_SOURCE" \ + "$DATA_STAGE/runtime-assets/qdrant/storage" +ln -s "$QA_NEO4J_DATA_SOURCE" \ + "$DATA_STAGE/runtime-assets/neo4j/data" +"$PYTHON" -B "$OLLAMA_EXPORTER" \ + --source "$OLLAMA_MODELS_SOURCE" \ + --output "$DATA_STAGE/runtime-assets/ollama-models" \ + --model "$QA_OLLAMA_GENERATION_MODEL" \ + --model "$QA_OLLAMA_EMBED_MODEL" +"$PYTHON" -B "$MANIFEST_BUILDER" \ + --runtime-root "$DATA_STAGE/runtime-assets" \ + --output "$PAYLOAD/DELIVERY_MANIFEST.json" \ + --version "$VERSION" \ + --generation-model "$QA_OLLAMA_GENERATION_MODEL" \ + --embed-model "$QA_OLLAMA_EMBED_MODEL" + +echo "[C/6] AES-256-GCM 加密并签名 runtime-assets" +"$PYTHON" "$ADMIN" build-encrypt-asset \ + --source "$ASSET_STAGE/runtime-assets" \ + --master-key "$SECRETS_DIR/master_keys/$VERSION.key" \ + --private-key "$SECRETS_DIR/supplier_private_key.pem" \ + --key-passphrase-file "$PASS_FILE" \ + --output-dir "$PAYLOAD/assets" \ + --release-version "$VERSION" \ + --name runtime-assets + +echo "[C/6] 打包 QA 数据卷与本地模型（持久化数据，不进入 tmpfs）" +tar -C "$DATA_STAGE" -chf "$DATA_ARCHIVE_TMP" runtime-assets + +echo "[C/6] 写入公钥、版本与完整性清单" +install -m 0644 \ + "$SECRETS_DIR/supplier_public_key.pem" \ + "$PAYLOAD/config/supplier_public_key.pem" +printf '%s\n' "$VERSION" > "$PAYLOAD/VERSION" +( + cd "$PAYLOAD" + find . -type f ! -name SHA256SUMS -print0 \ + | sort -z \ + | xargs -0 sha256sum > SHA256SUMS +) + +echo "[C/6] 打包并执行秘密泄漏检查" +tar -C "$PAYLOAD" -czf "$ARCHIVE" VERSION DELIVERY_MANIFEST.json app assets config SHA256SUMS +if tar -tzf "$ARCHIVE" \ + | grep -Eq 'supplier_private|master_keys|private_key\.pass|plantai_admin\.py|server\.env|\.hardened-build\.json' +then + echo "安全检查失败：发行包包含服务端文件或构建标记" >&2 + exit 1 +fi + +echo "阶段 C 完成：$ARCHIVE" +sha256sum "$ARCHIVE" +mv -- "$DATA_ARCHIVE_TMP" "$DATA_ARCHIVE" +echo "QA 运行数据包：$DATA_ARCHIVE" +sha256sum "$DATA_ARCHIVE" | tee "$DATA_ARCHIVE.sha256" diff --git a/tooling/legacy_plantai_server_package/phase_e_issue_client.sh b/tooling/legacy_plantai_server_package/phase_e_issue_client.sh new file mode 100755 index 0000000..b8e2cd1 --- /dev/null +++ b/tooling/legacy_plantai_server_package/phase_e_issue_client.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# 阶段 E：为一个已完成 TPM 设置的指纹签发授权。 +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +[ -f "$SCRIPT_DIR/server.env" ] && . "$SCRIPT_DIR/server.env" + +FINGERPRINT="${FINGERPRINT:?请设置 FINGERPRINT=/安全路径/device_fingerprint.json}" +CUSTOMER_ID="${CUSTOMER_ID:?请设置 CUSTOMER_ID}" +CUSTOMER_NAME="${CUSTOMER_NAME:?请设置 CUSTOMER_NAME}" +SERIAL_NO="${SERIAL_NO:?请设置 SERIAL_NO}" +VERSION="${VERSION:-1.0.0}" + +[ -f "$FINGERPRINT" ] || { echo "指纹文件不存在: $FINGERPRINT" >&2; exit 1; } +"${PLANTAI_PYTHON:-python3}" - "$FINGERPRINT" <<'PY' +import json, sys +fp = json.load(open(sys.argv[1], encoding="utf-8")) +assert fp.get("device_id"), "missing device_id" +if fp.get("test_only_swtpm"): + import os + assert os.environ.get("ALLOW_SOFTWARE_TPM_TEST") == "1", ( + "这是软件 TPM 测试指纹；仅测试签发时显式设置 ALLOW_SOFTWARE_TPM_TEST=1" + ) +tpm = fp.get("tpm_public_key") or {} +assert tpm.get("public_key_pem"), "没有 TPM 公钥；请先在目标机执行 tpm2-setup 后重新采集" +assert tpm.get("attestation"), "没有 TPM attestation；拒绝生产签发" +print("TPM fingerprint precheck passed:", fp["device_id"]) +PY + +exec "$SCRIPT_DIR/issue_client.sh" \ + --fingerprint "$FINGERPRINT" --release-version "$VERSION" \ + --customer-id "$CUSTOMER_ID" --customer-name "$CUSTOMER_NAME" \ + --serial-no "$SERIAL_NO" \ + --allowed-assets runtime-assets \ + --allow-unverified-tpm-key diff --git a/tooling/legacy_plantai_server_package/phase_f_migrate_data.sh b/tooling/legacy_plantai_server_package/phase_f_migrate_data.sh new file mode 100755 index 0000000..8d12bef --- /dev/null +++ b/tooling/legacy_plantai_server_package/phase_f_migrate_data.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +# 阶段 F（供应商侧）：备份三个核心知识库数据，打包发给客户。 +# 你的一千多篇文章存在三个地方： +# 1. rag_kb_store/ ← 原文文件 +# 2. Qdrant ← 向量索引（语义搜索用） +# 3. Neo4j ← 知识图谱（实体关系用） +# +# 客户侧恢复请使用 client/restore_data.sh。 +set -euo pipefail +umask 077 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +[ -f "$SCRIPT_DIR/server.env" ] && . "$SCRIPT_DIR/server.env" +TEST_ROOT="${TEST_ROOT:-$(cd "$SCRIPT_DIR/../.." && pwd)}" +SERVER_ROOT="${SERVER_ROOT:-$TEST_ROOT/server}" +VERSION="${VERSION:-1.0.0}" +OUT_DIR="${OUT_DIR:-$SCRIPT_DIR/out/$VERSION}" + +usage() { + cat >&2 </dev/null 2>&1; } + +echo "==============================================" +echo " 供应商侧 - 知识库迁移打包" +echo " 版本: $VERSION" +echo " 输出目录: $OUT_DIR" +echo "" +echo " 备份内容（你的一千篇文章的三种形态）：" +echo " 1. rag_kb_store/ ← 原文文件" +echo " 2. Qdrant 快照 ← 向量索引（语义搜索）" +echo " 3. Neo4j dump ← 知识图谱（实体关系）" +echo "==============================================" + +# ============================================================ +# 1. rag_kb_store — 原始文章文件 +# ============================================================ +if [ "$SKIP_RAG_KB" = false ]; then + if [ -d "$RAG_KB_DIR/rag_kb_store" ]; then + echo "[1/3] 打包文件知识库 rag_kb_store（原文文件）" + tar --xattrs --acls -czf "$OUT_DIR/knowledge-$VERSION.tar.gz" \ + -C "$RAG_KB_DIR" rag_kb_store + echo " -> 大小: $(du -h "$OUT_DIR/knowledge-$VERSION.tar.gz" | cut -f1)" + else + echo "[1/3] 错误: $RAG_KB_DIR/rag_kb_store 不存在，请指定 --rag-kb-dir" + exit 1 + fi +fi + +# ============================================================ +# 2. Qdrant — 向量索引（文章段落的数学指纹） +# ============================================================ +if [ "$SKIP_QDRANT" = false ]; then + echo "[2/3] Qdrant 向量库快照" + if have_cmd curl; then + collections="$(curl -sf "$QDRANT_URL/collections" | python3 -c " +import json,sys +data=json.load(sys.stdin) +cols=data.get('result',data.get('collections',[])) +for c in cols: + print(c.get('name',c)) +" 2>/dev/null)" || collections="" + if [ -n "$collections" ]; then + mkdir -p "$OUT_DIR/qdrant-snapshots/$VERSION" + while IFS= read -r col; do + [ -z "$col" ] && continue + echo " -> 创建集合 '$col' 快照" + curl -sf -X POST "$QDRANT_URL/collections/$col/snapshots" >/dev/null && sleep 2 + snaps="$(curl -sf "$QDRANT_URL/collections/$col/snapshots" | python3 -c " +import json,sys +data=json.load(sys.stdin) +for s in data.get('result',[]): + print(s.get('name','')) +" 2>/dev/null)" || snaps="" + for snap in $snaps; do + snap_path="$("${PLANTAI_PYTHON:-python3}" -c " +import json,sys +data=json.load(sys.stdin) +for s in data.get('result',[]): + if s.get('name') == '$snap': + print(s.get('path','')) +" 2>/dev/null <<< "$(curl -sf "$QDRANT_URL/collections/$col/snapshots")")" || true + if [ -n "$snap_path" ] && [ -f "$snap_path" ]; then + cp "$snap_path" "$OUT_DIR/qdrant-snapshots/$VERSION/${col}__${snap}" + echo " 已保存: ${col}__${snap}" + fi + done + done <<< "$collections" + echo " -> 快照总大小: $(du -sh "$OUT_DIR/qdrant-snapshots/$VERSION" | cut -f1)" + else + echo " -> Qdrant 无响应或无集合，跳过" + fi + fi +fi + +# ============================================================ +# 3. Neo4j — 知识图谱（文章之间的实体关系网络） +# ============================================================ +if [ "$SKIP_NEO4J" = false ]; then + echo "[3/3] Neo4j 图数据库 dump" + if docker ps --format '{{.Names}}' 2>/dev/null | grep -q "$NEO4J_CONTAINER"; then + echo " -> 停止容器 $NEO4J_CONTAINER" + docker stop "$NEO4J_CONTAINER" + NEO4J_DATA_DIR="$(docker inspect "$NEO4J_CONTAINER" --format '{{range .Mounts}}{{if eq .Destination "/data"}}{{.Source}}{{end}}{{end}}' 2>/dev/null)" || true + if [ -n "$NEO4J_DATA_DIR" ] && [ -d "$NEO4J_DATA_DIR" ]; then + mkdir -p "$OUT_DIR/neo4j-backup" + echo " -> 执行 neo4j-admin database dump" + docker run --rm \ + -v "${NEO4J_DATA_DIR}:/data" \ + -v "$OUT_DIR/neo4j-backup:/backups" \ + "neo4j:${NEO4J_VERSION}" \ + neo4j-admin database dump neo4j --to-path=/backups --overwrite-destination=true + echo " -> dump 文件: $(ls -lh "$OUT_DIR/neo4j-backup/")" + fi + echo " -> 重启容器 $NEO4J_CONTAINER" + docker start "$NEO4J_CONTAINER" + else + echo " -> 未运行 Neo4j 容器，跳过" + fi +fi + +echo "" +echo "==============================================" +echo " 打包完成！发给客户的产物：" +echo " ┌────────────────────────────────────────────┐" +echo " │ 1. knowledge-$VERSION.tar.gz ← 原文文件 │" +ls -lh "$OUT_DIR/knowledge-$VERSION.tar.gz" 2>/dev/null || echo " │ （未打包）" +echo " │ 2. qdrant-snapshots/ ← 向量索引 │" +ls -lh "$OUT_DIR/qdrant-snapshots/$VERSION/" 2>/dev/null | head -3 || echo " │ （未打包）" +echo " │ 3. neo4j-backup/ ← 知识图谱 │" +ls -lh "$OUT_DIR/neo4j-backup/" 2>/dev/null | head -3 || echo " │ （未打包）" +echo " └────────────────────────────────────────────┘" +echo "" +echo " 客户机上执行 client/restore_data.sh 恢复。" +echo " 另外还需要发给客户的（跟知识库无关）：" +echo " - plantai-rag-code-.tar.gz （代码包）" +echo " - plantai-provision-.tar.gz （授权文件）" +echo "==============================================" diff --git a/tooling/legacy_plantai_server_package/server.env b/tooling/legacy_plantai_server_package/server.env new file mode 100644 index 0000000..826e47e --- /dev/null +++ b/tooling/legacy_plantai_server_package/server.env @@ -0,0 +1,57 @@ +# ============================================================================= +# server.env — 服务端阶段脚本统一配置 +# ============================================================================= + +# 工作区根目录。 +TEST_ROOT=/home/xzq/5090/PlanAI-cooperate/hsj-workspace/encryption-test +SERVER_ROOT=/home/xzq/5090/PlanAI-cooperate/hsj-workspace/encryption-test/server + +# 阶段 A/C 的源码输入。只需要在这里修改，脚本不再内置源码路径。 +PLANTAI_SOURCE=/home/xzq/5090/PlanAI-cooperate/hsj-workspace/rag-window +RAG_SOURCE=/home/xzq/5090/PlanAI-cooperate/RAG/rag-develop +QA_SOURCE=/home/xzq/5090/PlanAI-cooperate/RAG/rag-qa-v1_1_73 + +# 阶段 A 的新产物；阶段 C 默认只消费这两个目录。 +PLANTAI_OUTPUT=/home/xzq/5090/PlanAI-cooperate/hsj-workspace/encryption-test/server/PlantAI_compiled.new +RAG_OUTPUT=/home/xzq/5090/PlanAI-cooperate/hsj-workspace/encryption-test/server/RAG.new +QA_OUTPUT=/home/xzq/5090/PlanAI-cooperate/hsj-workspace/encryption-test/server/RAG_QA.new +PLANTAI_COMPILED=/home/xzq/5090/PlanAI-cooperate/hsj-workspace/encryption-test/server/PlantAI_compiled.new +RAG_COMPILED=/home/xzq/5090/PlanAI-cooperate/hsj-workspace/encryption-test/server/RAG.new +QA_COMPILED=/home/xzq/5090/PlanAI-cooperate/hsj-workspace/encryption-test/server/RAG_QA.new + +# QA 交付运行资产。阶段 C 将复制并加密这些目录，不会修改源数据卷。 +QA_QDRANT_STORAGE_SOURCE=/home/xzq/home/wutx/gene-family-mas-v2/data/qdrant_qa_v1_1_73/storage +QA_NEO4J_DATA_SOURCE=/home/xzq/home/wutx/gene-family-mas-v2/data/neo4j_qa_v1_1_73/data +QA_RERANKER_MODEL_SOURCE=/home/xzq/5090/PlantAI-rag/bge-reranker-base +OLLAMA_MODELS_SOURCE=/home/xzq/5090/PlanAI-cooperate/RAG/runtime/ollama_models_qa_v1_1_73 +QA_OLLAMA_GENERATION_MODEL=qwen2.5:14b-instruct +QA_OLLAMA_EMBED_MODEL=bge-m3:567m +QA_ENV_FILE=/home/xzq/5090/PlanAI-cooperate/RAG/runtime/qa_v1_1_73.env + +# 客户授权信息（阶段 E）。 +FINGERPRINT=/home/xzq/5090/PlanAI-cooperate/hsj-workspace/encryption-test/server/plantai_server_package/server_fingers/test/device_fingerprint.json +CUSTOMER_ID=CUS-0001 +CUSTOMER_NAME=测试客户 +SERIAL_NO=PlantAI-RAG-0001 + +# 软件 TPM 只能用于测试；生产硬件授权不要启用。 +export ALLOW_SOFTWARE_TPM_TEST=1 + +# 每次正式发新版时修改。 +VERSION=1.0.0 + +# 服务端秘密目录，绝不随发行包外发。 +SECRETS_DIR=/home/xzq/5090/PlanAI-cooperate/hsj-workspace/encryption-test/server/plantai_server_package/secrets + +# 构建发行包的输出目录（可选）。 +# OUT_DIR=/home/xzq/5090/PlanAI-cooperate/hsj-workspace/encryption-test/server/plantai_server_package/out/1.0.0 + +# 与目标部署环境 ABI 一致的 Python。 +PLANTAI_PYTHON=/home/xzq/miniconda3/envs/test4py311/bin/python +QA_PYTHON=/home/xzq/miniconda3/envs/gfpy312/bin/python + +# 供应商私钥口令文件，权限应为 600。 +SUPPLIER_KEY_PASSPHRASE_FILE=/home/xzq/5090/PlanAI-cooperate/hsj-workspace/encryption-test/server/plantai_server_package/secrets/private_key.pass +# 或改用保存口令的环境变量名称（与上一项二选一）： +# SUPPLIER_KEY_PASSPHRASE_ENV=PLANTAI_SUPPLIER_KEY_PASSPHRASE +BBA_PFAM_DIR=/home/xzq/5090/PlantAI-dev4090-7/PlantAI-dev/data/hmm diff --git a/tooling/legacy_plantai_server_package/start_all.sh b/tooling/legacy_plantai_server_package/start_all.sh new file mode 100755 index 0000000..386f870 --- /dev/null +++ b/tooling/legacy_plantai_server_package/start_all.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +# Test-server launcher for hardened PlantAI + RAG + QA. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${SERVER_ENV_FILE:-$SCRIPT_DIR/server.env}" +[[ -f "$ENV_FILE" ]] && . "$ENV_FILE" +QA_ENV_FILE="${QA_ENV_FILE:-}" +[[ -z "$QA_ENV_FILE" || ! -f "$QA_ENV_FILE" ]] || . "$QA_ENV_FILE" + +PLANTAI_DIR="${1:-${PLANTAI_COMPILED:-${PLANTAI_OUTPUT:-}}}" +RAG_DIR="${2:-${RAG_COMPILED:-${RAG_OUTPUT:-}}}" +QA_DIR="${3:-${QA_COMPILED:-${QA_OUTPUT:-}}}" +PLANTAI_PYTHON="${PLANTAI_PYTHON:-python3}" +RAG_PYTHON="${RAG_PYTHON:-$PLANTAI_PYTHON}" +QA_PYTHON="${QA_PYTHON:-python3}" +PLANTAI_PORT="${PLANTAI_WEB_PORT:-7880}" +RAG_PORT="${RAG_PORT:-7900}" +QA_PORT="${QA_PORT:-7925}" +QA_QDRANT_PORT="${QA_QDRANT_PORT:-16333}" +QA_QDRANT_GRPC_PORT="${QA_QDRANT_GRPC_PORT:-16334}" +QA_NEO4J_HTTP_PORT="${QA_NEO4J_HTTP_PORT:-17474}" +QA_NEO4J_BOLT_PORT="${QA_NEO4J_BOLT_PORT:-17687}" +QA_NEO4J_PASSWORD="${QA_NEO4J_PASSWORD:-${NEO4J_PASSWORD:-plant_srt_direct_password}}" +OLLAMA_PORT="${OLLAMA_PORT:-11434}" +QA_OLLAMA_GENERATION_MODEL="${QA_OLLAMA_GENERATION_MODEL:-qwen2.5:14b-instruct}" +QA_OLLAMA_EMBED_MODEL="${QA_OLLAMA_EMBED_MODEL:-bge-m3:567m}" +QA_QDRANT_STORAGE_SOURCE="${QA_QDRANT_STORAGE_SOURCE:-}" +QA_NEO4J_DATA_SOURCE="${QA_NEO4J_DATA_SOURCE:-}" +QA_RERANKER_MODEL_SOURCE="${QA_RERANKER_MODEL_SOURCE:-}" + +PID_FILE="${PID_FILE:-/tmp/plantai_all_services.pid}" +PLANTAI_LOG="${PLANTAI_LOG:-/tmp/plantai_server.log}" +RAG_LOG="${RAG_LOG:-/tmp/rag_server.log}" +QA_LOG="${QA_LOG:-/tmp/qa_server.log}" +BBAV5_QUEUE_PREFIX="${BBAV5_QUEUE_PREFIX:-plantai_dev_${PLANTAI_PORT}}" +BBAV5_REDIS_URL="${BBAV5_REDIS_URL:-redis://127.0.0.1:6379/10}" +BBAV5_CELERY_LOG_DIR="${BBAV5_CELERY_LOG_DIR:-$PLANTAI_DIR/logs/celery_${BBAV5_QUEUE_PREFIX}}" +BBAV5_CELERY_LOGLEVEL="${BBAV5_CELERY_LOGLEVEL:-info}" +BBAV5_DEFAULT_CONCURRENCY="${BBAV5_DEFAULT_CONCURRENCY:-1}" +BBAV5_HEAVY_CONCURRENCY="${BBAV5_HEAVY_CONCURRENCY:-2}" +BBAV5_MEDIUM_CONCURRENCY="${BBAV5_MEDIUM_CONCURRENCY:-2}" +BBAV5_LIGHT_CONCURRENCY="${BBAV5_LIGHT_CONCURRENCY:-4}" +BBAV5_RNASEQ_CONCURRENCY="${BBAV5_RNASEQ_CONCURRENCY:-2}" +BBAV5_WORKER_NAME_PREFIX="${BBAV5_WORKER_NAME_PREFIX:-${BBAV5_QUEUE_PREFIX//[^A-Za-z0-9_]/_}}" +export PATH="$(dirname "$PLANTAI_PYTHON"):$PATH" +export BBA_PFAM_DIR="${BBA_PFAM_DIR:-$PLANTAI_DIR/data/hmm}" +export BBAV5_PROJECT_ROOT="$PLANTAI_DIR" +export BBAV5_QUEUE_PREFIX +export BBAV5_REDIS_URL + +usage() { + echo "用法: bash $0 [PlantAI目录] [RAG目录] [QA目录]" >&2 + echo "未传参数时自动读取 $ENV_FILE" >&2 +} +for value in "$PLANTAI_DIR" "$RAG_DIR" "$QA_DIR"; do + [[ -n "$value" ]] || { usage; exit 2; } + [[ -d "$value" ]] || { echo "目录不存在: $value" >&2; exit 2; } +done +PLANTAI_DIR="$(realpath "$PLANTAI_DIR")" +RAG_DIR="$(realpath "$RAG_DIR")" +QA_DIR="$(realpath "$QA_DIR")" +export BBAV5_PROJECT_ROOT="$PLANTAI_DIR" +[[ -f "$PLANTAI_DIR/apps/web/vue3/main.py" ]] || { echo "缺少 PlantAI 入口" >&2; exit 2; } +[[ -f "$RAG_DIR/api/app.py" ]] || { echo "缺少 RAG 入口" >&2; exit 2; } +[[ -f "$QA_DIR/scripts/qa_v1_1_73_server.py" ]] || { echo "缺少 QA 入口" >&2; exit 2; } +for python_bin in "$PLANTAI_PYTHON" "$RAG_PYTHON" "$QA_PYTHON"; do + [[ -x "$python_bin" ]] || { echo "Python 不可执行: $python_bin" >&2; exit 2; } +done + +wait_http() { + local url="$1" label="$2" + for _ in {1..90}; do + curl -fsS --max-time 2 "$url" >/dev/null 2>&1 && return 0 + sleep 1 + done + echo "$label 未能通过健康检查: $url" >&2 + return 1 +} + +ensure_container() { + local name="$1" image="$2" source="$3" target="$4" + shift 4 + if docker inspect "$name" >/dev/null 2>&1; then + local mounted_source + mounted_source="$(docker inspect -f \ + "{{range .Mounts}}{{if eq .Destination \"$target\"}}{{.Source}}{{end}}{{end}}" \ + "$name")" + [[ "$(realpath -m "$mounted_source")" == "$(realpath -m "$source")" ]] || { + echo "$name 已存在但 $target 挂载不匹配: $mounted_source != $source" >&2 + exit 3 + } + [[ "$(docker inspect -f '{{.State.Running}}' "$name")" == "true" ]] || + docker start "$name" >/dev/null + return + fi + [[ -d "$source" ]] || { echo "$name 数据目录不存在: $source" >&2; exit 3; } + docker run -d --name "$name" --restart unless-stopped \ + -v "$source:$target" "$@" "$image" >/dev/null +} + +command -v curl >/dev/null +command -v docker >/dev/null +command -v ollama >/dev/null +ensure_container qdrant-qa-v1-1-73 qdrant/qdrant:latest \ + "$QA_QDRANT_STORAGE_SOURCE" /qdrant/storage \ + -p "$QA_QDRANT_PORT:6333" -p "$QA_QDRANT_GRPC_PORT:6334" +ensure_container neo4j-qa-v1-1-73 neo4j:5.15.0-community \ + "$QA_NEO4J_DATA_SOURCE" /data \ + -p "$QA_NEO4J_HTTP_PORT:7474" -p "$QA_NEO4J_BOLT_PORT:7687" \ + -e "NEO4J_AUTH=neo4j/$QA_NEO4J_PASSWORD" +wait_http "http://127.0.0.1:$QA_QDRANT_PORT/collections" Qdrant +wait_http "http://127.0.0.1:$QA_NEO4J_HTTP_PORT" Neo4j +wait_http "http://127.0.0.1:$OLLAMA_PORT/api/tags" Ollama +ollama list | awk 'NR > 1 {print $1}' | grep -Fxq "$QA_OLLAMA_GENERATION_MODEL" || + { echo "Ollama 缺少模型: $QA_OLLAMA_GENERATION_MODEL" >&2; exit 3; } +ollama list | awk 'NR > 1 {print $1}' | grep -Fxq "$QA_OLLAMA_EMBED_MODEL" || + { echo "Ollama 缺少模型: $QA_OLLAMA_EMBED_MODEL" >&2; exit 3; } + +export METHODOLOGY_RAG_API="http://127.0.0.1:$RAG_PORT/api/v1" +export BBA_RAG_FAMILY_PFAM_URL="http://127.0.0.1:$RAG_PORT/api/v1/rag/family/pfam" +export BBA_RAG_PFAM_URL="http://127.0.0.1:$RAG_PORT/api/v1/rag/chat" +export DUAL_KB_QA_BASE_URL="http://127.0.0.1:$QA_PORT" +export DUAL_KB_QA_ROUTE="qa-v1-1-73" +export PLANTAI_RAG_ROOT="$RAG_DIR" +export GENE_FAMILY_ROOT="$QA_DIR" +export V15_QDRANT_URL="http://127.0.0.1:$QA_QDRANT_PORT" +export QDRANT_URL="$V15_QDRANT_URL" +export V15_QDRANT_COLLECTION="${V15_QDRANT_COLLECTION:-plant_srt_direct_v1_bge_m3_20260702}" +export NEO4J_URI="bolt://127.0.0.1:$QA_NEO4J_BOLT_PORT" +export NEO4J_USER="${NEO4J_USER:-neo4j}" +export NEO4J_PASSWORD="$QA_NEO4J_PASSWORD" +export QA_V1_BGE_RERANKER_MODEL="${QA_V1_BGE_RERANKER_MODEL:-$QA_RERANKER_MODEL_SOURCE}" +export QA_V1_ALIAS_CONFIG="${QA_V1_ALIAS_CONFIG:-$QA_DIR/configs/rag/entity_alias.yaml}" +export QA_V1_TAXONOMY_CONFIG="${QA_V1_TAXONOMY_CONFIG:-$QA_DIR/configs/rag/relation_taxonomy.yaml}" +export QA_V1_LLM_API_KEY="ollama-local" +export QA_V1_LLM_BASE_URL="http://127.0.0.1:$OLLAMA_PORT/v1" +export QA_V1_LLM_MODEL="$QA_OLLAMA_GENERATION_MODEL" +export V15_OLLAMA_EMBED_URL="${V15_OLLAMA_EMBED_URL:-http://127.0.0.1:$OLLAMA_PORT/api/embed}" +export V15_OLLAMA_EMBED_MODEL="$QA_OLLAMA_EMBED_MODEL" + +pids=() +celery_pids=() +cleanup() { + for pid in "${pids[@]:-}"; do kill "$pid" 2>/dev/null || true; done + rm -f -- "$PID_FILE" +} +trap cleanup EXIT INT TERM +: > "$PID_FILE" + +start_celery_worker() { + local role="$1" concurrency="$2" + [[ "$concurrency" =~ ^[0-9]+$ ]] || + { echo "Celery $role concurrency 必须是非负整数: $concurrency" >&2; exit 2; } + (( concurrency > 0 )) || return 0 + local worker_name="${BBAV5_WORKER_NAME_PREFIX}_${role}" + local queue_name="${BBAV5_QUEUE_PREFIX}.${role}" + mkdir -p "$BBAV5_CELERY_LOG_DIR" + (cd "$PLANTAI_DIR" && PYTHONPATH="$RAG_DIR:$PLANTAI_DIR" \ + "$PLANTAI_PYTHON" -m celery -A celery_app worker \ + --loglevel="$BBAV5_CELERY_LOGLEVEL" \ + --hostname="${worker_name}@%h" \ + --queues="$queue_name" \ + --concurrency="$concurrency" \ + --prefetch-multiplier=1) \ + >>"$BBAV5_CELERY_LOG_DIR/${worker_name}.log" 2>&1 & + pids+=("$!") + celery_pids+=("$!") + echo "$!" >>"$PID_FILE" +} + +start_celery_worker default "$BBAV5_DEFAULT_CONCURRENCY" +start_celery_worker heavy "$BBAV5_HEAVY_CONCURRENCY" +start_celery_worker medium "$BBAV5_MEDIUM_CONCURRENCY" +start_celery_worker light "$BBAV5_LIGHT_CONCURRENCY" +start_celery_worker rnaseq "$BBAV5_RNASEQ_CONCURRENCY" +sleep "${BBAV5_CELERY_START_WAIT_SEC:-4}" +for pid in "${celery_pids[@]}"; do + kill -0 "$pid" 2>/dev/null || + { echo "Celery worker 启动后提前退出，请检查 $BBAV5_CELERY_LOG_DIR" >&2; exit 3; } +done + +(cd "$QA_DIR" && PYTHONPATH="$QA_DIR" "$QA_PYTHON" -m uvicorn \ + scripts.qa_v1_1_73_server:app --host 127.0.0.1 --port "$QA_PORT") \ + >>"$QA_LOG" 2>&1 & +pids+=("$!"); echo "$!" >>"$PID_FILE" +wait_http "http://127.0.0.1:$QA_PORT/api/v1/qa-v1-1-73/health" QA + +(cd "$RAG_DIR" && PYTHONPATH="$RAG_DIR:$RAG_DIR/src" "$RAG_PYTHON" -m uvicorn \ + api.app:app --host 127.0.0.1 --port "$RAG_PORT" --workers 2) \ + >>"$RAG_LOG" 2>&1 & +pids+=("$!"); echo "$!" >>"$PID_FILE" +wait_http "http://127.0.0.1:$RAG_PORT/health" RAG + +(cd "$PLANTAI_DIR" && PYTHONPATH="$RAG_DIR:$PLANTAI_DIR" \ + PLANTAI_WEB_PORT="$PLANTAI_PORT" "$PLANTAI_PYTHON" apps/web/vue3/main.py) \ + >>"$PLANTAI_LOG" 2>&1 & +pids+=("$!"); echo "$!" >>"$PID_FILE" +wait_http "http://127.0.0.1:$PLANTAI_PORT" PlantAI + +echo "三项目与 Celery workers 已启动：PlantAI=$PLANTAI_PORT RAG=$RAG_PORT QA=$QA_PORT" +echo "Celery: prefix=$BBAV5_QUEUE_PREFIX broker=$BBAV5_REDIS_URL logs=$BBAV5_CELERY_LOG_DIR" +echo "PID 文件: $PID_FILE；日志: $PLANTAI_LOG $RAG_LOG $QA_LOG" +wait -n "${pids[@]}" +echo "一个应用服务意外退出，正在停止其余应用服务" >&2 +exit 1 diff --git a/tooling/legacy_plantai_server_package/stop_all.sh b/tooling/legacy_plantai_server_package/stop_all.sh new file mode 100755 index 0000000..e18592f --- /dev/null +++ b/tooling/legacy_plantai_server_package/stop_all.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# 停止 PlantAI + RAG + QA 及 Celery workers，与 start_all.sh 对应。 +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${SERVER_ENV_FILE:-$SCRIPT_DIR/plantai_server_package/server.env}" +[[ -f "$ENV_FILE" ]] && . "$ENV_FILE" + +PID_FILE="${PID_FILE:-/tmp/plantai_all_services.pid}" +PLANTAI_PORT="${PLANTAI_WEB_PORT:-7880}" +RAG_PORT="${RAG_PORT:-7900}" +QA_PORT="${QA_PORT:-7925}" +STOP_CONTAINERS=false + +for arg in "$@"; do + case "$arg" in + --stop-containers) STOP_CONTAINERS=true ;; + *) echo "未知参数: $arg" >&2; exit 2 ;; + esac +done + +stop_by_pid_file() { + [[ -f "$PID_FILE" ]] || return 1 + local stopped=0 + while IFS= read -r pid; do + [[ "$pid" =~ ^[0-9]+$ ]] || continue + if kill -0 "$pid" 2>/dev/null; then + kill -TERM "$pid" 2>/dev/null && echo "SIGTERM -> PID $pid" || true + stopped=$((stopped + 1)) + else + echo "PID $pid 已不存在，跳过" + fi + done < "$PID_FILE" + + # 等待最多 10 秒让进程优雅退出 + local waited=0 + while (( waited < 10 )); do + local alive=0 + while IFS= read -r pid; do + [[ "$pid" =~ ^[0-9]+$ ]] || continue + kill -0 "$pid" 2>/dev/null && alive=$((alive + 1)) || true + done < "$PID_FILE" + (( alive == 0 )) && break + sleep 1 + waited=$((waited + 1)) + done + + # 仍存活则强制杀死 + while IFS= read -r pid; do + [[ "$pid" =~ ^[0-9]+$ ]] || continue + if kill -0 "$pid" 2>/dev/null; then + kill -KILL "$pid" 2>/dev/null && echo "SIGKILL -> PID $pid（未能优雅退出）" || true + fi + done < "$PID_FILE" + + rm -f -- "$PID_FILE" + echo "PID 文件已清除: $PID_FILE" + return 0 +} + +stop_by_port() { + local port="$1" label="$2" + local pids_on_port + pids_on_port=$(lsof -t -i:"$port" 2>/dev/null || true) + if [[ -n "$pids_on_port" ]]; then + echo "$pids_on_port" | xargs -r kill -TERM 2>/dev/null && echo "SIGTERM -> $label 端口 $port 上的进程" || true + else + echo "$label（端口 $port）未发现运行中的进程" + fi +} + +echo "=== 停止 PlantAI 服务 ===" +if ! stop_by_pid_file; then + echo "未找到 PID 文件，改用端口方式停止..." + stop_by_port "$PLANTAI_PORT" PlantAI + stop_by_port "$RAG_PORT" RAG + stop_by_port "$QA_PORT" QA +fi + +if [[ "$STOP_CONTAINERS" == "true" ]]; then + echo "=== 停止 Docker 容器 ===" + for container in qdrant-qa-v1-1-73 neo4j-qa-v1-1-73; do + if docker inspect "$container" >/dev/null 2>&1; then + if [[ "$(docker inspect -f '{{.State.Running}}' "$container")" == "true" ]]; then + docker stop "$container" && echo "已停止容器: $container" || true + else + echo "容器 $container 已处于停止状态" + fi + else + echo "容器 $container 不存在，跳过" + fi + done +fi + +echo "=== 所有服务已停止 ===" diff --git a/tooling/legacy_plantai_server_package/templates/start_all_client.sh b/tooling/legacy_plantai_server_package/templates/start_all_client.sh new file mode 100755 index 0000000..38c1e79 --- /dev/null +++ b/tooling/legacy_plantai_server_package/templates/start_all_client.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# Customer launcher, called after license validation and runtime-assets decryption. +set -euo pipefail + +BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +[[ ! -f "$BASE_DIR/runtime.env" ]] || . "$BASE_DIR/runtime.env" +PLANTAI_DIR="${1:-$BASE_DIR/PlantAI_compiled}" +RAG_DIR="${2:-$BASE_DIR/RAG}" +QA_DIR="${3:-$BASE_DIR/QA}" +PLANTAI_ROOT="${PLANTAI_ROOT:-$(cd "$BASE_DIR/../../.." && pwd)}" +RUNTIME_ROOT="${RUNTIME_ROOT:-${PLANTAI_RUNTIME_DATA:-$PLANTAI_ROOT/data/runtime-assets}}" +[[ -d "$RUNTIME_ROOT" ]] || { echo "解密后的运行资产目录不存在: $RUNTIME_ROOT" >&2; exit 2; } + +PLANTAI_PYTHON="${PLANTAI_PYTHON:-python3}" +RAG_PYTHON="${RAG_PYTHON:-$PLANTAI_PYTHON}" +QA_PYTHON="${QA_PYTHON:-python3}" +PLANTAI_PORT="${PLANTAI_WEB_PORT:-7880}" +RAG_PORT="${RAG_PORT:-7900}" +QA_PORT="${QA_PORT:-7925}" +QA_QDRANT_PORT="${QA_QDRANT_PORT:-16333}" +QA_QDRANT_GRPC_PORT="${QA_QDRANT_GRPC_PORT:-16334}" +QA_NEO4J_HTTP_PORT="${QA_NEO4J_HTTP_PORT:-17474}" +QA_NEO4J_BOLT_PORT="${QA_NEO4J_BOLT_PORT:-17687}" +QA_NEO4J_PASSWORD="${QA_NEO4J_PASSWORD:-${NEO4J_PASSWORD:-}}" +[[ -n "$QA_NEO4J_PASSWORD" ]] || + { echo "请在 install.env/service.env 设置 QA_NEO4J_PASSWORD 或 NEO4J_PASSWORD" >&2; exit 2; } +OLLAMA_PORT="${OLLAMA_PORT:-11435}" +QA_OLLAMA_GENERATION_MODEL="${QA_OLLAMA_GENERATION_MODEL:-qwen2.5:14b-instruct}" +QA_OLLAMA_EMBED_MODEL="${QA_OLLAMA_EMBED_MODEL:-bge-m3:567m}" +BBAV5_QUEUE_PREFIX="${BBAV5_QUEUE_PREFIX:-plantai_dev_${PLANTAI_PORT}}" +BBAV5_REDIS_URL="${BBAV5_REDIS_URL:-redis://127.0.0.1:6379/10}" +BBAV5_CELERY_LOG_DIR="${BBAV5_CELERY_LOG_DIR:-$PLANTAI_DIR/logs/celery_${BBAV5_QUEUE_PREFIX}}" +BBAV5_CELERY_LOGLEVEL="${BBAV5_CELERY_LOGLEVEL:-info}" +BBAV5_DEFAULT_CONCURRENCY="${BBAV5_DEFAULT_CONCURRENCY:-1}" +BBAV5_HEAVY_CONCURRENCY="${BBAV5_HEAVY_CONCURRENCY:-2}" +BBAV5_MEDIUM_CONCURRENCY="${BBAV5_MEDIUM_CONCURRENCY:-2}" +BBAV5_LIGHT_CONCURRENCY="${BBAV5_LIGHT_CONCURRENCY:-4}" +BBAV5_RNASEQ_CONCURRENCY="${BBAV5_RNASEQ_CONCURRENCY:-2}" +BBAV5_WORKER_NAME_PREFIX="${BBAV5_WORKER_NAME_PREFIX:-${BBAV5_QUEUE_PREFIX//[^A-Za-z0-9_]/_}}" +QDRANT_DATA="$RUNTIME_ROOT/qdrant/storage" +NEO4J_DATA="$RUNTIME_ROOT/neo4j/data" +RERANKER_MODEL="$RUNTIME_ROOT/QA/models/bge-reranker-base" +OLLAMA_MODELS="$RUNTIME_ROOT/ollama-models" + +for path in "$PLANTAI_DIR" "$RAG_DIR" "$QA_DIR" "$QDRANT_DATA" "$NEO4J_DATA" "$RERANKER_MODEL" "$OLLAMA_MODELS"; do + [[ -d "$path" ]] || { echo "交付目录不存在: $path" >&2; exit 2; } +done +for python_bin in "$PLANTAI_PYTHON" "$RAG_PYTHON" "$QA_PYTHON"; do + command -v "$python_bin" >/dev/null 2>&1 || [[ -x "$python_bin" ]] || + { echo "Python 不可用: $python_bin" >&2; exit 2; } +done + +wait_http() { + local url="$1" label="$2" + for _ in {1..120}; do + curl -fsS --max-time 2 "$url" >/dev/null 2>&1 && return 0 + sleep 1 + done + echo "$label 健康检查失败: $url" >&2 + return 1 +} +ensure_container() { + local name="$1" image="$2" source="$3" target="$4" + shift 4 + if docker inspect "$name" >/dev/null 2>&1; then + local mounted_source + mounted_source="$(docker inspect -f \ + "{{range .Mounts}}{{if eq .Destination \"$target\"}}{{.Source}}{{end}}{{end}}" \ + "$name")" + [[ "$(realpath -m "$mounted_source")" == "$(realpath -m "$source")" ]] || { + echo "$name 已存在但 $target 挂载不匹配: $mounted_source != $source" >&2 + exit 3 + } + [[ "$(docker inspect -f '{{.State.Running}}' "$name")" == "true" ]] || + docker start "$name" >/dev/null + else + docker run -d --name "$name" --restart unless-stopped \ + -v "$source:$target" "$@" "$image" >/dev/null + fi +} + +command -v docker >/dev/null +command -v curl >/dev/null +command -v ollama >/dev/null +ensure_container plantai-qdrant-qa qdrant/qdrant:latest "$QDRANT_DATA" /qdrant/storage \ + -p "$QA_QDRANT_PORT:6333" -p "$QA_QDRANT_GRPC_PORT:6334" +ensure_container plantai-neo4j-qa neo4j:5.15.0-community "$NEO4J_DATA" /data \ + -p "$QA_NEO4J_HTTP_PORT:7474" -p "$QA_NEO4J_BOLT_PORT:7687" \ + -e "NEO4J_AUTH=neo4j/$QA_NEO4J_PASSWORD" +wait_http "http://127.0.0.1:$QA_QDRANT_PORT/collections" Qdrant +wait_http "http://127.0.0.1:$QA_NEO4J_HTTP_PORT" Neo4j + +pids=() +cleanup() { for pid in "${pids[@]:-}"; do kill "$pid" 2>/dev/null || true; done; } +trap cleanup EXIT INT TERM +if ! curl -fsS --max-time 2 "http://127.0.0.1:$OLLAMA_PORT/api/tags" >/dev/null 2>&1; then + (OLLAMA_HOST="127.0.0.1:$OLLAMA_PORT" OLLAMA_MODELS="$OLLAMA_MODELS" ollama serve) & + pids+=("$!") +fi +wait_http "http://127.0.0.1:$OLLAMA_PORT/api/tags" Ollama +OLLAMA_HOST="127.0.0.1:$OLLAMA_PORT" OLLAMA_MODELS="$OLLAMA_MODELS" ollama list | + awk 'NR > 1 {print $1}' | grep -Fxq "$QA_OLLAMA_GENERATION_MODEL" || + { echo "交付 Ollama 模型缺失: $QA_OLLAMA_GENERATION_MODEL" >&2; exit 3; } +OLLAMA_HOST="127.0.0.1:$OLLAMA_PORT" OLLAMA_MODELS="$OLLAMA_MODELS" ollama list | + awk 'NR > 1 {print $1}' | grep -Fxq "$QA_OLLAMA_EMBED_MODEL" || + { echo "交付 Ollama 模型缺失: $QA_OLLAMA_EMBED_MODEL" >&2; exit 3; } + +export METHODOLOGY_RAG_API="http://127.0.0.1:$RAG_PORT/api/v1" +export BBA_RAG_FAMILY_PFAM_URL="http://127.0.0.1:$RAG_PORT/api/v1/rag/family/pfam" +export BBA_RAG_PFAM_URL="http://127.0.0.1:$RAG_PORT/api/v1/rag/chat" +export DUAL_KB_QA_BASE_URL="http://127.0.0.1:$QA_PORT" +export DUAL_KB_QA_ROUTE="qa-v1-1-73" +export PLANTAI_RAG_ROOT="$RAG_DIR" +export GENE_FAMILY_ROOT="$QA_DIR" +export V15_QDRANT_URL="http://127.0.0.1:$QA_QDRANT_PORT" +export QDRANT_URL="$V15_QDRANT_URL" +export V15_QDRANT_COLLECTION="${V15_QDRANT_COLLECTION:-plant_srt_direct_v1_bge_m3_20260702}" +export NEO4J_URI="bolt://127.0.0.1:$QA_NEO4J_BOLT_PORT" +export NEO4J_USER="${NEO4J_USER:-neo4j}" +export NEO4J_PASSWORD="$QA_NEO4J_PASSWORD" +export QA_V1_BGE_RERANKER_MODEL="$RERANKER_MODEL" +export QA_V1_ALIAS_CONFIG="$QA_DIR/configs/rag/entity_alias.yaml" +export QA_V1_TAXONOMY_CONFIG="$QA_DIR/configs/rag/relation_taxonomy.yaml" +export QA_V1_LLM_API_KEY="ollama-local" +export QA_V1_LLM_BASE_URL="http://127.0.0.1:$OLLAMA_PORT/v1" +export QA_V1_LLM_MODEL="$QA_OLLAMA_GENERATION_MODEL" +export V15_OLLAMA_EMBED_URL="http://127.0.0.1:$OLLAMA_PORT/api/embed" +export V15_OLLAMA_EMBED_MODEL="$QA_OLLAMA_EMBED_MODEL" +export PATH="$(dirname "$PLANTAI_PYTHON"):$PATH" +export BBA_PFAM_DIR="${BBA_PFAM_DIR:-$PLANTAI_DIR/data/hmm}" +export BBAV5_PROJECT_ROOT="$PLANTAI_DIR" +export BBAV5_QUEUE_PREFIX +export BBAV5_REDIS_URL + +(cd "$QA_DIR" && PYTHONPATH="$QA_DIR" "$QA_PYTHON" -m uvicorn \ + scripts.qa_v1_1_73_server:app --host 127.0.0.1 --port "$QA_PORT") & +pids+=("$!"); wait_http "http://127.0.0.1:$QA_PORT/api/v1/qa-v1-1-73/health" QA +(cd "$RAG_DIR" && PYTHONPATH="$RAG_DIR:$RAG_DIR/src" "$RAG_PYTHON" -m uvicorn \ + api.app:app --host 127.0.0.1 --port "$RAG_PORT" --workers 2) & +pids+=("$!"); wait_http "http://127.0.0.1:$RAG_PORT/health" RAG + +celery_pids=() +start_celery_worker() { + local role="$1" concurrency="$2" + [[ "$concurrency" =~ ^[0-9]+$ ]] || + { echo "Celery $role concurrency 必须是非负整数: $concurrency" >&2; exit 2; } + (( concurrency > 0 )) || return 0 + local worker_name="${BBAV5_WORKER_NAME_PREFIX}_${role}" + local queue_name="${BBAV5_QUEUE_PREFIX}.${role}" + mkdir -p "$BBAV5_CELERY_LOG_DIR" + (cd "$PLANTAI_DIR" && PYTHONPATH="$RAG_DIR:$PLANTAI_DIR" \ + "$PLANTAI_PYTHON" -m celery -A celery_app worker \ + --loglevel="$BBAV5_CELERY_LOGLEVEL" \ + --hostname="${worker_name}@%h" \ + --queues="$queue_name" \ + --concurrency="$concurrency" \ + --prefetch-multiplier=1) \ + >>"$BBAV5_CELERY_LOG_DIR/${worker_name}.log" 2>&1 & + pids+=("$!") + celery_pids+=("$!") +} +start_celery_worker default "$BBAV5_DEFAULT_CONCURRENCY" +start_celery_worker heavy "$BBAV5_HEAVY_CONCURRENCY" +start_celery_worker medium "$BBAV5_MEDIUM_CONCURRENCY" +start_celery_worker light "$BBAV5_LIGHT_CONCURRENCY" +start_celery_worker rnaseq "$BBAV5_RNASEQ_CONCURRENCY" +sleep "${BBAV5_CELERY_START_WAIT_SEC:-4}" +for pid in "${celery_pids[@]}"; do + kill -0 "$pid" 2>/dev/null || + { echo "Celery worker 启动后提前退出，请检查 $BBAV5_CELERY_LOG_DIR" >&2; exit 3; } +done + +(cd "$PLANTAI_DIR" && PYTHONPATH="$RAG_DIR:$PLANTAI_DIR" \ + PLANTAI_WEB_PORT="$PLANTAI_PORT" "$PLANTAI_PYTHON" apps/web/vue3/main.py) & +pids+=("$!"); wait_http "http://127.0.0.1:$PLANTAI_PORT" PlantAI + +echo "三项目与 Celery workers 已启动：PlantAI=$PLANTAI_PORT RAG=$RAG_PORT QA=$QA_PORT Ollama=$OLLAMA_PORT" +echo "Celery: prefix=$BBAV5_QUEUE_PREFIX broker=$BBAV5_REDIS_URL logs=$BBAV5_CELERY_LOG_DIR" +wait -n "${pids[@]}" +echo "一个应用服务意外退出，正在停止其余应用服务" >&2 +exit 1 diff --git a/tooling/legacy_plantai_server_package/tests/conftest.py b/tooling/legacy_plantai_server_package/tests/conftest.py new file mode 100644 index 0000000..8f74c95 --- /dev/null +++ b/tooling/legacy_plantai_server_package/tests/conftest.py @@ -0,0 +1,8 @@ +"""Make `plantai_admin` importable from the server bin/ directory.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "bin")) diff --git a/tooling/legacy_plantai_server_package/tests/test_admin_phase_e.py b/tooling/legacy_plantai_server_package/tests/test_admin_phase_e.py new file mode 100644 index 0000000..935cbd7 --- /dev/null +++ b/tooling/legacy_plantai_server_package/tests/test_admin_phase_e.py @@ -0,0 +1,254 @@ +"""Phase E (server) tests: TPM attestation verification + K_master wrapping. + +Uses pure-Python fixture fingerprints (helpers.py) so no TPM is required. +Includes regression tests for the two real-world bugs encountered: + - `public_key_pem` holding a tpm2-tools text dump instead of a PEM + (MalformedFraming in load_pem_public_key) +""" + +from __future__ import annotations + +import argparse +import base64 +import json +from pathlib import Path + +import pytest +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import padding as _asym_padding +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from cryptography.hazmat.primitives.asymmetric.rsa import generate_private_key + +from tpm_fixtures import ( + make_attestation, + make_fingerprint, + marshal_tpmt_public, + pem_public_key, + tpm_name, +) +from plantai_admin import verify_tpm_attestation, wrap_key_for_device, wrap_key_with_tpm + +DEVICE_ID = "test-device-001" + + +def _keypair(): + key = generate_private_key(public_exponent=65537, key_size=2048) + return key, key.public_key() + + +@pytest.fixture(scope="module") +def valid_fingerprint(): + wrap_priv, wrap_pub = _keypair() + cert_priv, cert_pub = _keypair() + tpmt = marshal_tpmt_public(wrap_pub) + attestation = make_attestation(wrap_pub, cert_priv, cert_pub, tpmt) + return make_fingerprint(DEVICE_ID, pem_public_key(wrap_pub), attestation), wrap_priv, tpmt + + +# --------------------------------------------------------------------------- +# verify_tpm_attestation +# --------------------------------------------------------------------------- +def test_verify_attestation_accepts_valid_fingerprint(valid_fingerprint): + fp, _, _ = valid_fingerprint + ok, reason = verify_tpm_attestation(fp["tpm_public_key"]) + assert ok is True, reason + assert "verified" in reason.lower() + + +def test_verify_attestation_rejects_wrong_wrap_key(valid_fingerprint): + fp, _, _ = valid_fingerprint + tpm_info = dict(fp["tpm_public_key"]) + # Submit a DIFFERENT RSA public key than the one the attestation covers. + other_priv, other_pub = _keypair() + tpm_info["public_key_pem"] = pem_public_key(other_pub) + ok, reason = verify_tpm_attestation(tpm_info) + assert ok is False + assert "mismatch" in reason.lower() + + +def test_verify_attestation_rejects_bad_signature(valid_fingerprint): + fp, _, _ = valid_fingerprint + tpm_info = dict(fp["tpm_public_key"]) + attestation = dict(tpm_info["attestation"]) + # Sign the blob with a random key -> invalid signature. + other_priv, other_pub = _keypair() + blob = base64.b64decode(attestation["attestation_blob_b64"]) + attestation["attestation_signature_b64"] = base64.b64encode( + other_priv.sign(blob, _asym_padding.PKCS1v15(), hashes.SHA256()) + ).decode("ascii") + tpm_info["attestation"] = attestation + ok, reason = verify_tpm_attestation(tpm_info) + assert ok is False + assert "signature verification FAILED" in reason + + +def test_verify_attestation_rejects_name_mismatch(valid_fingerprint): + fp, _, _ = valid_fingerprint + tpm_info = dict(fp["tpm_public_key"]) + attestation = dict(tpm_info["attestation"]) + # Attestation certifies key A's TPM Name, but we submit key B's TPMT_PUBLIC. + _, other_pub = _keypair() + other_tpmt = marshal_tpmt_public(other_pub) + attestation["wrapkey_tpmt_public_b64"] = base64.b64encode(other_tpmt).decode("ascii") + tpm_info["attestation"] = attestation + ok, reason = verify_tpm_attestation(tpm_info) + assert ok is False + assert "name mismatch" in reason.lower() + + +def test_verify_attestation_rejects_text_dump_pem(valid_fingerprint): + """Regression: public_key_pem containing a tpm2-tools TPMT_PUBLIC text + dump (the tpm2-tools 5.6 bug) must be rejected, not crash.""" + fp, _, _ = valid_fingerprint + tpm_info = dict(fp["tpm_public_key"]) + tpm_info["public_key_pem"] = ( + "name: 000b8a0600ee6b\n" + "attributes:\n value: fixedtpm|fixedparent|sensitivedataorigin|userwithauth|decrypt\n" + "rsa: deadbeef\n" + ) + ok, reason = verify_tpm_attestation(tpm_info) + assert ok is False + assert "could not parse public_key_pem" in reason.lower() + + +def test_verify_attestation_accepts_libtpms_style_blob(valid_fingerprint): + """Regression: swtpm/libtpms marshals TPMS_ATTEST with the certified Name + at a non-spec offset (verified against a real swtpm 2.0 blob, where the + name lands at offset 75 instead of 78). The offset-based parser alone + fails on this, so the name must also be located by direct search. + """ + fp, _, tpmt = valid_fingerprint + tpm_info = dict(fp["tpm_public_key"]) + # Build a fresh attestation with our own cert key so we can re-sign. + wrap_pub = serialization.load_pem_public_key( + tpm_info["public_key_pem"].encode()) + cert_priv, cert_pub = _keypair() + attestation = make_attestation(wrap_pub, cert_priv, cert_pub, tpmt) + # Insert one byte into the clockInfo region, shifting the certified Name + # (and every later field) to a non-spec offset while keeping it intact. + blob = bytearray(base64.b64decode(attestation["attestation_blob_b64"])) + blob[10:10] = b"\x00" + attestation["attestation_blob_b64"] = base64.b64encode(bytes(blob)).decode("ascii") + attestation["attestation_signature_b64"] = base64.b64encode( + cert_priv.sign(bytes(blob), _asym_padding.PKCS1v15(), hashes.SHA256()) + ).decode("ascii") + tpm_info["attestation"] = attestation + ok, reason = verify_tpm_attestation(tpm_info) + assert ok is True, reason + + +# --------------------------------------------------------------------------- +# wrap_key_with_tpm +# --------------------------------------------------------------------------- +def test_wrap_key_roundtrip_with_tpm(valid_fingerprint): + fp, wrap_priv, _ = valid_fingerprint + master_key = b"K_master-" * 4 + + envelope = wrap_key_with_tpm(fp["tpm_public_key"]["public_key_pem"], + master_key, DEVICE_ID) + assert envelope["version"] == 3 + assert envelope["wrap_mode"] == "tpm-rsa-oaep-v1" + + plain = wrap_priv.decrypt( + base64.b64decode(envelope["wrapped_key_b64"]), + _asym_padding.OAEP( + mgf=_asym_padding.MGF1(algorithm=hashes.SHA256()), + algorithm=hashes.SHA256(), + label=None, + ), + ) + assert plain == master_key + + +def test_wrap_key_rejects_malformed_pem(valid_fingerprint): + """Regression: the exact ValueError (MalformedFraming) from the original + bug — the server must fail cleanly on a text-dump public_key_pem.""" + master_key = b"K_master-" * 4 + with pytest.raises(ValueError): + wrap_key_with_tpm( + "name: 000b8a06\nattributes:\n value: fixedtpm|decrypt\nrsa: deadbeef\n", + master_key, DEVICE_ID, + ) + + +# --------------------------------------------------------------------------- +# wrap_key_for_device (CLI handler) +# --------------------------------------------------------------------------- +def test_wrap_key_for_device_end_to_end(tmp_path, valid_fingerprint): + fp, wrap_priv, _ = valid_fingerprint + master_key = b"K_master-" * 4 + + fp_path = tmp_path / "device_fingerprint.json" + fp_path.write_text(json.dumps(fp, ensure_ascii=False), encoding="utf-8") + master_path = tmp_path / "K_master.key" + master_path.write_bytes(master_key) + out_path = tmp_path / "asset.key.enc" + + wrap_key_for_device(argparse.Namespace( + fingerprint=str(fp_path), + master_key=str(master_path), + output=str(out_path), + allow_unverified_tpm_key=False, + allow_legacy_wrap=False, + )) + + envelope = json.loads(out_path.read_text(encoding="utf-8")) + assert envelope["version"] == 3 + plain = wrap_priv.decrypt( + base64.b64decode(envelope["wrapped_key_b64"]), + _asym_padding.OAEP( + mgf=_asym_padding.MGF1(algorithm=hashes.SHA256()), + algorithm=hashes.SHA256(), + label=None, + ), + ) + assert plain == master_key + + +def test_issue_license_binds_tpm_key_and_signs(tmp_path, valid_fingerprint): + fp, _, _ = valid_fingerprint + from plantai_admin import issue_license, sign_payload + + priv = Ed25519PrivateKey.generate() + priv_path = tmp_path / "supplier_private_key.pem" + priv_path.write_bytes(priv.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + )) + pub_path = tmp_path / "supplier_public_key.pem" + pub_path.write_bytes(priv.public_key().public_bytes( + serialization.Encoding.PEM, + serialization.PublicFormat.SubjectPublicKeyInfo, + )) + + fp_path = tmp_path / "device_fingerprint.json" + fp_path.write_text(json.dumps(fp, ensure_ascii=False), encoding="utf-8") + out_path = tmp_path / "license.json" + + issue_license(argparse.Namespace( + fingerprint=str(fp_path), + private_key=str(priv_path), + output=str(out_path), + customer_id="CUS-0001", + customer_name="Test Co", + device_id=None, + serial_no="PlantAI-0001", + edition="standard", + release_version="1.0.0", + asset_key_id="ASSET-KEY-0001", + min_match_score="60", + allow_warning_score="85", + allow_weak_anchor=False, + asset_key_envelope=None, + allowed_assets=["runtime-assets"], + key_passphrase_env="", + key_passphrase_file="", + )) + + license_data = json.loads(out_path.read_text(encoding="utf-8")) + # Signature verifies against the supplier public key. + sign_payload(priv_path, license_data) + # TPM public key hash is bound into the license. + assert license_data["asset_policy"]["tpm_public_key_sha256"] == \ + fp["tpm_public_key"]["public_key_sha256"] diff --git a/tooling/legacy_plantai_server_package/tests/tpm_fixtures.py b/tooling/legacy_plantai_server_package/tests/tpm_fixtures.py new file mode 100644 index 0000000..d41cfb7 --- /dev/null +++ b/tooling/legacy_plantai_server_package/tests/tpm_fixtures.py @@ -0,0 +1,110 @@ +"""Builders that produce a cryptographically-valid TPM fingerprint fixture. + +A real client runs tpm2_readpublic / tpm2_certify against an actual TPM. +These helpers do the equivalent in pure Python using an in-memory RSA key +pair and a correctly-marshaled TPMS_ATTEST blob, so the server-side +verification code can be tested without any TPM. + +Layouts follow the wire formats documented in plantai_admin.py: + TPMT_PUBLIC -> _parse_tpmt_public_rsa_numbers() + TPMS_ATTEST -> _parse_tpms_attest_name() +""" + +from __future__ import annotations + +import base64 +import hashlib +import struct + +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import padding as _asym_padding +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey, RSAPublicKey + +_TPM_ALG_SHA256 = b"\x00\x0b" +# fixedtpm|fixedparent|sensitivedataorigin|userwithauth|decrypt +_DECRYPT_ATTRIBUTES = 0x00020072 + + +def marshal_tpmt_public(pub: RSAPublicKey) -> bytes: + """Marshal the TPMT_PUBLIC area of an RSA decrypt key (no size prefix).""" + n = pub.public_numbers().n + n_bytes = n.to_bytes((n.bit_length() + 7) // 8, "big").rjust(256, b"\x00") + auth_policy = b"\x00\x00" # TPM2B_DIGEST, len 0 + symmetric = b"\x00\x10" # TPM_ALG_NULL (no keyBits/mode) + scheme = b"\x00\x10" # TPM_ALG_NULL, no details + unique = struct.pack(">H", len(n_bytes)) + n_bytes + return ( + struct.pack(">H", 0x0001) # TPM_ALG_RSA + + _TPM_ALG_SHA256 # nameAlg + + struct.pack(">I", _DECRYPT_ATTRIBUTES) + + auth_policy + symmetric + scheme + + struct.pack(">H", 2048) # keyBits + + struct.pack(">I", 0) # exponent 0 => 65537 + + unique + ) + + +def tpm_name(tpmt_public: bytes) -> bytes: + return _TPM_ALG_SHA256 + hashlib.sha256(tpmt_public).digest() + + +def build_attestation_blob(name: bytes) -> bytes: + """Construct a TPMS_ATTEST (TPM_ST_ATTEST_CERTIFY) blob embedding `name`.""" + return ( + struct.pack(">I", 0xFF544347) # magic + + struct.pack(">H", 0x8017) # TPM_ST_ATTEST_CERTIFY + + b"\x00\x00" # qualifiedSigner (len 0) + + b"\x00\x00" # extraData (len 0) + + b"\x00" * 20 # clockInfo + + b"\x00" * 8 # firmwareVersion + + struct.pack(">H", len(name)) + name # certified key Name + + b"\x00\x00" # qualifiedName (len 0) + ) + + +def pem_public_key(pub: RSAPublicKey) -> str: + return pub.public_bytes( + serialization.Encoding.PEM, + serialization.PublicFormat.SubjectPublicKeyInfo, + ).decode("ascii") + + +def make_attestation(wrap_pub: RSAPublicKey, + cert_priv: RSAPrivateKey, cert_pub: RSAPublicKey, + tpmt_public: bytes) -> dict: + """Produce the attestation block exactly as the client would.""" + blob = build_attestation_blob(tpm_name(tpmt_public)) + signature = cert_priv.sign(blob, _asym_padding.PKCS1v15(), hashes.SHA256()) + cert_pem = pem_public_key(cert_pub) + return { + "attestation_blob_b64": base64.b64encode(blob).decode("ascii"), + "attestation_signature_b64": base64.b64encode(signature).decode("ascii"), + "certifying_key_pem": cert_pem, + "certifying_key_sha256": hashlib.sha256(cert_pem.encode()).hexdigest(), + "wrapkey_tpmt_public_b64": base64.b64encode(tpmt_public).decode("ascii"), + "attestation_json": {"type": "TPM_ST_ATTEST_CERTIFY"}, + } + + +def make_fingerprint(device_id: str, wrap_pem: str, attestation: dict) -> dict: + return { + "device_id": device_id, + "core_id_anchor": "dmi_product_uuid", + "core_id_hash": hashlib.sha256(b"core").hexdigest(), + "fingerprint_material": { + "hostname": "dev", + "dmi": {}, + "cpu_model": "test", + "gpu_uuid": [], + "disk_serial": [], + "network_mac": [], + }, + "tpm_public_key": { + "tpm_version": "2.0", + "persistent_handle": 0x81008000, + "algorithm": "RSA-2048-OAEP-SHA256", + "public_key_pem": wrap_pem, + "public_key_sha256": hashlib.sha256(wrap_pem.encode()).hexdigest(), + "attestation": attestation, + }, + } diff --git a/tooling/legacy_plantai_server_package/tools/audit_compiled_tree.py b/tooling/legacy_plantai_server_package/tools/audit_compiled_tree.py new file mode 100755 index 0000000..ae9bbd5 --- /dev/null +++ b/tooling/legacy_plantai_server_package/tools/audit_compiled_tree.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +"""Audit a compiled tree and bind it to a tamper-evident build marker.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import sys +import sysconfig +from datetime import datetime +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from elf_visibility import summarize # noqa: E402 + + +MARKER_NAME = ".hardened-build.json" +TREE_DIGEST_POLICY = "immutable-runtime-v2" +MUTABLE_TOP_LEVEL_DIRS = { + "cost_logs", + "data", + "input", + "logs", + "memory_storage", + "neo4j", + "output", + "outputs", + "rag_kb_store", + "service_data", + "temp", + "v15_output", + "workspace", +} + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def is_digest_excluded(relative: str) -> bool: + parts = Path(relative).parts + return ( + relative == MARKER_NAME + or not parts + or parts[0] in MUTABLE_TOP_LEVEL_DIRS + or "__pycache__" in parts + or relative.endswith((".pyc", ".pyo")) + ) + + +def tree_digest(root: Path) -> tuple[str, int]: + digest = hashlib.sha256() + count = 0 + for path in sorted( + (item for item in root.rglob("*") if item.is_file()), + key=lambda item: item.relative_to(root).as_posix(), + ): + relative = path.relative_to(root).as_posix() + if is_digest_excluded(relative): + continue + count += 1 + digest.update(relative.encode("utf-8")) + digest.update(b"\0") + digest.update(bytes.fromhex(file_sha256(path))) + digest.update(b"\0") + return digest.hexdigest(), count + + +def audit_tree(root: Path, project: str) -> dict[str, Any]: + errors: list[dict[str, Any]] = [] + extensions = sorted(root.rglob("*.so")) + expected_suffix = sysconfig.get_config_var("EXT_SUFFIX") or ".so" + hidden_non_pyinit = 0 + + if not extensions: + errors.append({"kind": "no_extensions"}) + + for extension in extensions: + relative = extension.relative_to(root).as_posix() + if not extension.name.endswith(expected_suffix): + errors.append( + { + "kind": "python_abi_suffix_mismatch", + "file": relative, + "expected_suffix": expected_suffix, + } + ) + try: + info = summarize(extension, extension.read_bytes()) + except BaseException as exc: + errors.append( + { + "kind": "elf_parse_failed", + "file": relative, + "error": f"{type(exc).__name__}: {exc}", + } + ) + continue + if info["has_symtab"]: + errors.append({"kind": "symtab_present", "file": relative}) + if len(info["pyinit"]) != 1 or not info["all_pyinit_default"]: + errors.append( + { + "kind": "invalid_pyinit", + "file": relative, + "pyinit": info["pyinit"], + } + ) + if not info["all_non_pyinit_hidden"]: + errors.append( + { + "kind": "visible_non_pyinit", + "file": relative, + "symbols": info["non_pyinit"], + } + ) + hidden_non_pyinit += len(info["non_pyinit"]) + + c_files = [ + path.relative_to(root).as_posix() + for path in root.rglob("*.c") + if path.is_file() + ] + if c_files: + errors.append({"kind": "generated_c_retained", "files": c_files}) + + sensitive = [] + for path in root.rglob("*"): + if not path.is_file(): + continue + lowered = path.name.lower() + if ( + lowered.endswith(".key") + or "private_key" in lowered + or "master_key" in lowered + or lowered == "private_key.pass" + ): + sensitive.append(path.relative_to(root).as_posix()) + if sensitive: + errors.append({"kind": "sensitive_filename", "files": sensitive}) + + digest, file_count = tree_digest(root) + return { + "project": project, + "tree_digest_policy": TREE_DIGEST_POLICY, + "python_version": sys.version.split()[0], + "extension_suffix": expected_suffix, + "extension_count": len(extensions), + "hidden_non_pyinit_count": hidden_non_pyinit, + "file_count_without_marker": file_count, + "tree_sha256": digest, + "error_count": len(errors), + "errors": errors, + } + + +def atomic_write_json(path: Path, payload: dict[str, Any]) -> None: + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + temporary.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + os.chmod(temporary, 0o640) + os.replace(temporary, path) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--tree", type=Path, required=True) + parser.add_argument("--project", required=True) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--write-marker", action="store_true") + mode.add_argument("--require-marker", action="store_true") + args = parser.parse_args() + + root = args.tree.resolve() + if not root.is_dir(): + raise SystemExit(f"Compiled tree missing: {root}") + + report = audit_tree(root, args.project) + marker_path = root / MARKER_NAME + + if args.require_marker: + if not marker_path.is_file(): + report["errors"].append({"kind": "marker_missing"}) + else: + marker = json.loads(marker_path.read_text(encoding="utf-8")) + report["marker"] = marker + for field in ( + "project", + "tree_digest_policy", + "python_version", + "extension_suffix", + "extension_count", + "tree_sha256", + ): + if marker.get(field) != report.get(field): + report["errors"].append( + { + "kind": "marker_mismatch", + "field": field, + "expected": marker.get(field), + "actual": report.get(field), + } + ) + + report["error_count"] = len(report["errors"]) + report["ok"] = report["error_count"] == 0 + + if args.write_marker and report["ok"]: + marker = { + key: report[key] + for key in ( + "project", + "tree_digest_policy", + "python_version", + "extension_suffix", + "extension_count", + "hidden_non_pyinit_count", + "file_count_without_marker", + "tree_sha256", + ) + } + marker["created_at"] = datetime.now().astimezone().isoformat( + timespec="seconds" + ) + atomic_write_json(marker_path, marker) + report["marker_path"] = str(marker_path) + + print(json.dumps(report, ensure_ascii=False, sort_keys=True)) + return 0 if report["ok"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tooling/legacy_plantai_server_package/tools/build_delivery_manifest.py b/tooling/legacy_plantai_server_package/tools/build_delivery_manifest.py new file mode 100755 index 0000000..150c9db --- /dev/null +++ b/tooling/legacy_plantai_server_package/tools/build_delivery_manifest.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Write the customer-facing manifest for the three-project offline delivery.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + + +def tree_bytes(path: Path) -> int: + total = 0 + for directory, _, files in os.walk(path, followlinks=True): + for filename in files: + total += (Path(directory) / filename).stat().st_size + return total + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--runtime-root", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--version", required=True) + parser.add_argument("--generation-model", required=True) + parser.add_argument("--embed-model", required=True) + args = parser.parse_args() + + root = args.runtime_root.resolve() + assets = { + "qdrant_storage": root / "qdrant" / "storage", + "neo4j_data": root / "neo4j" / "data", + "bge_reranker": root / "QA" / "models" / "bge-reranker-base", + "ollama_models": root / "ollama-models", + } + missing = [str(path) for path in assets.values() if not path.is_dir()] + if missing: + raise SystemExit("Missing runtime asset directories: " + ", ".join(missing)) + + payload = { + "release_version": args.version, + "runtime_data_archive": f"plantai-rag-runtime-data-{args.version}.tar", + "applications": { + "PlantAI": {"path": "app/PlantAI_compiled", "port": 7880}, + "RAG": {"path": "app/RAG", "port": 7900}, + "QA": {"path": "app/QA", "port": 7925}, + }, + "containers": { + "qdrant": { + "image": "qdrant/qdrant:latest", + "http_port": 16333, + "grpc_port": 16334, + "data_path": "runtime-assets/qdrant/storage", + }, + "neo4j": { + "image": "neo4j:5.15.0-community", + "http_port": 17474, + "bolt_port": 17687, + "data_path": "runtime-assets/neo4j/data", + }, + }, + "models": { + "ollama_generation": args.generation_model, + "ollama_embedding": args.embed_model, + "ollama_path": "runtime-assets/ollama-models", + "reranker_path": "runtime-assets/QA/models/bge-reranker-base", + }, + "asset_bytes": {name: tree_bytes(path) for name, path in assets.items()}, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + print(json.dumps(payload["asset_bytes"], ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tooling/legacy_plantai_server_package/tools/elf_visibility.py b/tooling/legacy_plantai_server_package/tools/elf_visibility.py new file mode 100755 index 0000000..1a009fe --- /dev/null +++ b/tooling/legacy_plantai_server_package/tools/elf_visibility.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Inspect or hide non-PyInit defined symbols in an ELF64 little-endian .dynsym.""" + +from __future__ import annotations + +import argparse +import json +import os +import struct +import tempfile +from pathlib import Path +from typing import Any + + +STN_UNDEF = 0 +STV_DEFAULT = 0 +STV_HIDDEN = 2 + + +def c_string(data: bytes | bytearray, offset: int) -> str: + end = data.find(b"\0", offset) + if end < 0: + end = len(data) + return bytes(data[offset:end]).decode("utf-8", errors="replace") + + +def parse_sections(data: bytes | bytearray) -> tuple[list[dict[str, Any]], bytes]: + if data[:4] != b"\x7fELF": + raise ValueError("not an ELF file") + if data[4] != 2: + raise ValueError("only ELF64 is supported") + if data[5] != 1: + raise ValueError("only little-endian ELF is supported") + + section_offset = struct.unpack_from("= len(sections): + raise ValueError("invalid section-name string-table index") + names_section = sections[section_name_index] + names = bytes( + data[ + names_section["offset"] : names_section["offset"] + names_section["size"] + ] + ) + for section in sections: + section["name"] = c_string(names, section["name_offset"]) + return sections, names + + +def inspect_dynsym(data: bytes | bytearray) -> tuple[list[dict[str, Any]], dict[str, Any]]: + sections, _ = parse_sections(data) + dynsym = next((section for section in sections if section["name"] == ".dynsym"), None) + if dynsym is None: + raise ValueError("missing .dynsym") + if dynsym["link"] >= len(sections): + raise ValueError("invalid .dynsym string-table link") + dynstr_section = sections[dynsym["link"]] + dynstr = data[ + dynstr_section["offset"] : dynstr_section["offset"] + dynstr_section["size"] + ] + entry_size = dynsym["entry_size"] or 24 + if entry_size < 24: + raise ValueError("invalid ELF64 symbol entry size") + + symbols = [] + count = dynsym["size"] // entry_size + for index in range(count): + offset = dynsym["offset"] + index * entry_size + name_offset = struct.unpack_from("> 4, + "type": info & 0xF, + "visibility": other & 0x3, + "section_index": section_index, + "value": value, + "size": size, + "defined": section_index != STN_UNDEF, + } + ) + return symbols, dynsym + + +def summarize(path: Path, data: bytes | bytearray) -> dict[str, Any]: + sections, _ = parse_sections(data) + symbols, _ = inspect_dynsym(data) + defined = [symbol for symbol in symbols if symbol["defined"] and symbol["name"]] + pyinit = [symbol for symbol in defined if symbol["name"].startswith("PyInit_")] + non_pyinit = [ + symbol for symbol in defined if not symbol["name"].startswith("PyInit_") + ] + return { + "path": str(path), + "has_symtab": any(section["name"] == ".symtab" for section in sections), + "defined_dynamic_count": len(defined), + "pyinit": [ + {"name": symbol["name"], "visibility": symbol["visibility"]} + for symbol in pyinit + ], + "non_pyinit": [ + {"name": symbol["name"], "visibility": symbol["visibility"]} + for symbol in non_pyinit + ], + "all_non_pyinit_hidden": all( + symbol["visibility"] == STV_HIDDEN for symbol in non_pyinit + ), + "all_pyinit_default": bool(pyinit) + and all(symbol["visibility"] == STV_DEFAULT for symbol in pyinit), + } + + +def hide_non_pyinit(path: Path) -> dict[str, Any]: + original = path.read_bytes() + data = bytearray(original) + symbols, _ = inspect_dynsym(data) + changed = [] + for symbol in symbols: + if ( + symbol["defined"] + and symbol["name"] + and not symbol["name"].startswith("PyInit_") + and symbol["visibility"] != STV_HIDDEN + ): + offset = symbol["entry_offset"] + 5 + data[offset] = (data[offset] & ~0x3) | STV_HIDDEN + changed.append(symbol["name"]) + + if changed: + stat = path.stat() + with tempfile.NamedTemporaryFile( + mode="wb", dir=path.parent, prefix=f".{path.name}.", delete=False + ) as handle: + temporary = Path(handle.name) + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + os.chmod(temporary, stat.st_mode) + os.replace(temporary, path) + + result = summarize(path, data) + result["changed"] = changed + return result + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("elf", type=Path) + parser.add_argument("--hide-non-pyinit", action="store_true") + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + + path = args.elf.resolve() + result = ( + hide_non_pyinit(path) + if args.hide_non_pyinit + else summarize(path, path.read_bytes()) + ) + print(json.dumps(result, ensure_ascii=False, sort_keys=True)) + if args.check: + valid = ( + not result["has_symtab"] + and result["all_pyinit_default"] + and result["all_non_pyinit_hidden"] + and len(result["pyinit"]) == 1 + ) + return 0 if valid else 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tooling/legacy_plantai_server_package/tools/export_ollama_models.py b/tooling/legacy_plantai_server_package/tools/export_ollama_models.py new file mode 100755 index 0000000..eee9ab8 --- /dev/null +++ b/tooling/legacy_plantai_server_package/tools/export_ollama_models.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Copy selected Ollama manifests and their referenced blobs.""" + +from __future__ import annotations + +import argparse +import json +import shutil +from pathlib import Path + + +def model_manifest(source: Path, model: str) -> Path: + repository, sep, tag = model.partition(":") + if not sep: + tag = "latest" + return source / "manifests" / "registry.ollama.ai" / "library" / repository / tag + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--source", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--model", action="append", required=True) + parser.add_argument("--check-only", action="store_true") + args = parser.parse_args() + + source = args.source.resolve() + output = args.output.resolve() + copied_blobs: set[str] = set() + summary: list[dict[str, object]] = [] + + for model in args.model: + manifest = model_manifest(source, model) + if not manifest.is_file(): + raise SystemExit(f"Ollama model manifest missing: {model}: {manifest}") + payload = json.loads(manifest.read_text(encoding="utf-8")) + digests = [payload["config"]["digest"]] + digests.extend(layer["digest"] for layer in payload.get("layers", [])) + + if not args.check_only: + destination = output / manifest.relative_to(source) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(manifest, destination) + + model_bytes = 0 + for digest in digests: + if not digest.startswith("sha256:"): + raise SystemExit(f"Unsupported Ollama digest for {model}: {digest}") + blob_name = digest.replace(":", "-", 1) + blob_source = source / "blobs" / blob_name + if not blob_source.is_file(): + raise SystemExit(f"Ollama blob missing for {model}: {blob_source}") + model_bytes += blob_source.stat().st_size + if args.check_only or blob_name in copied_blobs: + continue + blob_output = output / "blobs" / blob_name + blob_output.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(blob_source, blob_output) + copied_blobs.add(blob_name) + summary.append({"model": model, "bytes": model_bytes, "blobs": len(digests)}) + + if not args.check_only: + (output / "DELIVERED_MODELS.json").write_text( + json.dumps(summary, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + print(json.dumps(summary, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tooling/legacy_plantai_server_package/tools/generate_stubs.py b/tooling/legacy_plantai_server_package/tools/generate_stubs.py new file mode 100755 index 0000000..7bc5a4c --- /dev/null +++ b/tooling/legacy_plantai_server_package/tools/generate_stubs.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +""" +为 Cython 编译产物生成 .py stub 文件。 + +每个 stub 同时支持： + 1. 子进程调用 (python apps/cli/run_xxx.py --args) + -> __name__ == "__main__" 分支加载 .so 并调用 main() + 2. 模块导入 (from apps.cli.run_xxx import SomeClass) + -> 模块级 __getattr__ 懒加载 .so ，透明转发属性访问 + +用法ￚ + python generate_stubs.py [ ...] +""" +import sys +from pathlib import Path + +STUB_TEMPLATE = ( + "# Auto-generated stub — delegates to the compiled Cython extension.\n" + "# Supports both subprocess execution and module import.\n" + "import importlib.util as _ilu\n" + "import sys as _sys\n" + "from pathlib import Path as _P\n" + "\n" + "_stem = _P(__file__).stem\n" + "_so = next(_P(__file__).parent.glob(_stem + '.*.so'), None)\n" + "_cached_mod = None\n" + "\n" + "\n" + "def _load_compiled():\n" + " global _cached_mod\n" + " if _cached_mod is None:\n" + " if _so is None:\n" + " raise FileNotFoundError(\n" + " f'Compiled module not found for {__file__}'\n" + " )\n" + " _spec = _ilu.spec_from_file_location(_stem, _so)\n" + " _cached_mod = _ilu.module_from_spec(_spec)\n" + " _spec.loader.exec_module(_cached_mod)\n" + " return _cached_mod\n" + "\n" + "\n" + "def __getattr__(name: str):\n" + " \"\"\"Transparent proxy: forward attribute access to the compiled module.\"\"\"\n" + " return getattr(_load_compiled(), name)\n" + "\n" + "\n" + "if __name__ == '__main__':\n" + " _mod = _load_compiled()\n" + " _sys.exit(_mod.main() if hasattr(_mod, 'main') else 0)\n" +) + + +def generate_stubs(output_dir: Path) -> int: + created = 0 + for so_path in sorted(output_dir.rglob("*.so")): + stem = so_path.name.split(".")[0] + if stem == "__init__": + continue + py_path = so_path.parent / f"{stem}.py" + if py_path.exists(): + continue + py_path.write_text(STUB_TEMPLATE, encoding="utf-8") + print(f" stub: {py_path.relative_to(output_dir)}") + created += 1 + return created + + +def main(): + if len(sys.argv) < 2: + print(f"用法: {sys.argv[0]} [ ...]", file=sys.stderr) + sys.exit(1) + + total = 0 + for arg in sys.argv[1:]: + d = Path(arg) + if not d.is_dir(): + print(f"目录不存在，跳过: {d}", file=sys.stderr) + continue + print(f"[generate_stubs] 扫描: {d}") + n = generate_stubs(d) + print(f"[generate_stubs] 已生成 {n} 个 stub: {d}") + total += n + + print(f"[generate_stubs] 合计: {total} 个 stub") + + +if __name__ == "__main__": + main() diff --git a/tooling/legacy_plantai_server_package/tools/smoke_dual_build.py b/tooling/legacy_plantai_server_package/tools/smoke_dual_build.py new file mode 100755 index 0000000..81dbf3e --- /dev/null +++ b/tooling/legacy_plantai_server_package/tools/smoke_dual_build.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Representative import and behavior smoke tests for hardened builds.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from pathlib import Path + + +PLANTAI_PROBE = r""" +import importlib +import json +import sys + +sys.path.insert(0, sys.argv[1]) +module = importlib.import_module("gfa_core.mas.core.orchestrator") + +class Dummy: + def __init__(self, name="dummy"): + self.name = name + def __getattr__(self, name): + return Dummy(f"{self.name}.{name}") + def __call__(self, *args, **kwargs): + return Dummy(f"{self.name}()") + +instance = module.Orchestrator(Dummy("llm"), Dummy("memory"), None) +parsed = instance._parse_intent_from_query( + "identify the WRKY gene family in Arabidopsis" +) +route = instance._route_query({"query": "What is WRKY?"}) +expected = { + "gene_family": "WRKY", + "species": "At", + "stage": "identify", + "task_type": "analysis", +} +assert parsed == expected, (parsed, expected) +assert route == "knowledge", route +print("@@RESULT@@" + json.dumps({ + "module": module.__name__, + "parse_intent": parsed, + "route": route, +})) +""" + + +RAG_PROBE = r""" +import importlib +import json +import sys + +root = sys.argv[1] +sys.path.insert(0, root) +sys.path.insert(0, root + "/src") +module = importlib.import_module("src.rag.retriever") +instance = module.RAGRetriever() +assert instance.count_chunks() == 0 +assert instance.get_context([]) == "" +assert instance.index_documents([]) == 0 +api_module = importlib.import_module("api.app") +route_paths = {route.path for route in api_module.app.routes} +assert "/health" in route_paths +assert "/api/v1/methodology/extract-jobs" in route_paths +print("@@RESULT@@" + json.dumps({ + "module": module.__name__, + "api_module": api_module.__name__, + "methodology_route": True, + "count_chunks": 0, + "empty_context": "", + "empty_index_count": 0, +})) +""" + +QA_PROBE = r""" +import importlib +import json +import sys + +root = sys.argv[1] +sys.path.insert(0, root) +module = importlib.import_module("scripts.qa_v1_1_73_server") +route_paths = {route.path for route in module.app.routes} +assert "/health" in route_paths +assert "/api/v1/qa-v1-1-73/health" in route_paths +assert "/api/v1/qa-v1-1-73/info" in route_paths +assert "/api/v1/qa-v1-1-73/chat" in route_paths +print("@@RESULT@@" + json.dumps({ + "module": module.__name__, + "health_route": True, + "info_route": True, + "chat_route": True, +})) +""" + + +def run_probe(python: str, code: str, root: Path, timeout: int) -> dict: + environment = os.environ.copy() + completed = subprocess.run( + [python, "-B", "-s", "-c", code, str(root)], + cwd=root, + env=environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + check=False, + ) + result = None + for line in completed.stdout.splitlines(): + if line.startswith("@@RESULT@@"): + result = json.loads(line[len("@@RESULT@@") :]) + if completed.returncode != 0 or result is None: + raise RuntimeError( + f"probe failed rc={completed.returncode}\n" + f"stdout={completed.stdout[-3000:]}\n" + f"stderr={completed.stderr[-3000:]}" + ) + result["stdout_tail"] = completed.stdout[-1000:] + return result + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--plantai", type=Path) + parser.add_argument("--rag", type=Path) + parser.add_argument("--qa", type=Path) + parser.add_argument("--python") + parser.add_argument("--qa-python") + parser.add_argument("--timeout", type=int, default=45) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + + if args.plantai is None and args.rag is None and args.qa is None: + parser.error("at least one of --plantai, --rag or --qa is required") + if (args.plantai is not None or args.rag is not None) and not args.python: + parser.error("--python is required for PlantAI/RAG probes") + if args.qa is not None and not (args.qa_python or args.python): + parser.error("--qa-python or --python is required for the QA probe") + + payload = {} + if args.plantai is not None: + payload["plantai"] = run_probe( + args.python, PLANTAI_PROBE, args.plantai.resolve(), args.timeout + ) + if args.rag is not None: + payload["rag"] = run_probe( + args.python, RAG_PROBE, args.rag.resolve(), args.timeout + ) + if args.qa is not None: + payload["qa"] = run_probe( + args.qa_python or args.python, + QA_PROBE, + args.qa.resolve(), + args.timeout, + ) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + print(json.dumps(payload, ensure_ascii=False, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tooling/package_client.sh b/tooling/package_client.sh new file mode 100755 index 0000000..29017af --- /dev/null +++ b/tooling/package_client.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# 生成可交付给客户的 client 包（不含开发者私密配置） +# 用法: ./package_client.sh <客户名称> [输出目录] +# +# 示例: +# ./package_client.sh customer-A /tmp/client-pkg +# +# 输出: +# /tmp/client-pkg/customer-A/ +# install.env.example # 已填入 DEVICE_ID 等客户信息 +# README.md # 含客户专属说明 +# bin/ # 与原 client/bin 一致 +# install_plantai.sh # 与原 client/install_plantai.sh 一致 +# phase_d_tpm.sh # 与原一致 +# phase_d_swtpm_test.sh # 与原一致 + +set -euo pipefail + +CLIENT_NAME="${1:?用法: $0 <客户名称> [输出目录]}" +OUT_DIR="${2:-/tmp/client-pkg/$CLIENT_NAME}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +mkdir -p "$OUT_DIR" + +# 复制不可变逻辑 +cp -a "$SCRIPT_DIR/bin" "$OUT_DIR/bin" +cp -a "$SCRIPT_DIR/install_plantai.sh" "$OUT_DIR/" +cp -a "$SCRIPT_DIR/phase_d_tpm.sh" "$OUT_DIR/" +cp -a "$SCRIPT_DIR/phase_d_swtpm_test.sh" "$OUT_DIR/" + +# 生成客户专属 install.env.example +DEVICE_ID_SUGGEST="${CLIENT_NAME}-$(hostname)" +sed "s/DEVICE_ID=.*/DEVICE_ID=$DEVICE_ID_SUGGEST/" \ + "$SCRIPT_DIR/install.env.example" > "$OUT_DIR/install.env.example" + +# 生成客户专属 README +cp "$SCRIPT_DIR/README.md" "$OUT_DIR/README.md" +cat >> "$OUT_DIR/README.md" </dev/null || echo "待确认") +- **生成日期**: $(date '+%Y-%m-%d %H:%M') +EOF + +echo "✅ 客户交付包已生成: $OUT_DIR" +echo "" +echo "下一步:" +echo " 1. 编辑 $OUT_DIR/install.env.example → 填入客户专属值后另存为 install.env" +echo " 2. 将 plantai-rag-release-.tar.gz 放到客户机的 release/ 目录" +echo " 3. 发送 $OUT_DIR 给客户（不含 install.env！）"