forked from admin/PlantAI-rag-new
adminandClaude Fable 5
f10b86c088
- 176 默认(含开机自启 plantai-apps.service)即为无生信启动版: BIOFLOW_ENABLED=false 时 start_all 不拉起 8098,生信 TAB/入口不可见 - 开关来自 sdk/.env,切回生信版=bash toggle_bioflow_176.sh on Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
225 lines
12 KiB
Bash
Executable File
225 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# build_all.sh —— 一键将 4 部分最新版本编译打包为安装目录 + 部署脚本。
|
||
# 流水线:编译 SDK+3 agent → 双审计 → 冒烟 → (定版后:KB 加密打包)→ 组装 → SHA256SUMS → 泄露门禁
|
||
# 用法:bash tooling/build_all.sh [版本] (版本默认读 tooling.env VERSION)
|
||
set -euo pipefail
|
||
umask 027
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||
. "$SCRIPT_DIR/tooling.env"
|
||
|
||
# ---- 0. 库身份(kb-libraries/<lib>)——不枚举库名,任意含 manifest.yaml 的目录均可 ----
|
||
RAG_LIBRARY="${RAG_LIBRARY:-plant}"
|
||
rest_args=()
|
||
for arg in "$@"; do
|
||
case "$arg" in
|
||
--library=*) RAG_LIBRARY="${arg#--library=}" ;;
|
||
--library) RAG_LIBRARY="__next__" ;;
|
||
*) if [ "$RAG_LIBRARY" = "__next__" ]; then RAG_LIBRARY="$arg"; else rest_args+=("$arg"); fi ;;
|
||
esac
|
||
done
|
||
[ "$RAG_LIBRARY" = "__next__" ] && { echo "build_all: --library 缺值" >&2; exit 2; }
|
||
LIBRARY_DIR="$PROJECT_ROOT/kb-libraries/$RAG_LIBRARY"
|
||
[ -d "$LIBRARY_DIR" ] || LIBRARY_DIR="$RAG_LIBRARY" # 允许传任意目录路径
|
||
[ -f "$LIBRARY_DIR/manifest.yaml" ] || { echo "build_all: 库目录 $LIBRARY_DIR 缺 manifest.yaml" >&2; exit 2; }
|
||
grep -qE '^release_id:' "$LIBRARY_DIR/manifest.yaml" || { echo "build_all: manifest.yaml 缺 release_id" >&2; exit 2; }
|
||
echo "库身份: $RAG_LIBRARY ($LIBRARY_DIR)"
|
||
set -- "${rest_args[@]}"
|
||
|
||
VERSION="${1:-$VERSION}"
|
||
BUILD_ROOT="$SCRIPT_DIR/build/$VERSION"
|
||
RELEASE_ROOT="$BUILD_ROOT/install"
|
||
|
||
echo "== build_all.sh $VERSION =="
|
||
echo "工程根: $PROJECT_ROOT"
|
||
mkdir -p "$BUILD_ROOT"
|
||
|
||
# ---- 1. 编译四部分 ----
|
||
echo "[1/6] 编译 SDK + 3 agent"
|
||
"$SDK_PYTHON" -c 'import Cython' || { echo "SDK_PYTHON 缺 Cython" >&2; exit 1; }
|
||
"$QA_PYTHON" -c 'import Cython' || { echo "QA_PYTHON 缺 Cython" >&2; exit 1; }
|
||
"$LEGACY_PYTHON" -c 'import Cython' || { echo "LEGACY_PYTHON 缺 Cython" >&2; exit 1; }
|
||
|
||
rm -rf "$BUILD_ROOT/compiled"
|
||
mkdir -p "$BUILD_ROOT/compiled"
|
||
bash "$SCRIPT_DIR/compile/compile_sdk.sh" \
|
||
"$PROJECT_ROOT/sdk" "$BUILD_ROOT/compiled/sdk" "$SDK_PYTHON"
|
||
bash "$SCRIPT_DIR/compile/compile_qa.sh" \
|
||
"$PROJECT_ROOT/agent/qa" "$BUILD_ROOT/compiled/qa" "$QA_PYTHON"
|
||
bash "$SCRIPT_DIR/compile/compile_methodology.sh" \
|
||
"$PROJECT_ROOT/agent/methodology" "$BUILD_ROOT/compiled/methodology" "$METHODOLOGY_PYTHON"
|
||
bash "$SCRIPT_DIR/compile/compile_legacy.sh" \
|
||
"$PROJECT_ROOT/agent/legacy-rag" "$BUILD_ROOT/compiled/legacy-rag" "$LEGACY_PYTHON"
|
||
|
||
# 向量库服务（本地嵌入 + 私有资料问答检索；api.py 白名单明文，其余编译 .so）
|
||
bash "$SCRIPT_DIR/compile/compile_vstore.sh" \
|
||
"$PROJECT_ROOT/vstore" "$BUILD_ROOT/compiled/vstore" "$SDK_PYTHON"
|
||
|
||
# ---- 2. 双审计(各树用编译时的 Python——ABI 后缀必须匹配) ----
|
||
echo "[2/6] 编译树审计"
|
||
AUDIT="$SCRIPT_DIR/compile/tools/audit_compiled_tree.py"
|
||
declare -A TREE_PY=( [sdk]="$SDK_PYTHON" [qa]="$QA_PYTHON" [methodology]="$METHODOLOGY_PYTHON" [legacy-rag]="$LEGACY_PYTHON" )
|
||
for tree in sdk qa methodology legacy-rag; do
|
||
py="${TREE_PY[$tree]}"
|
||
"$py" "$AUDIT" --tree "$BUILD_ROOT/compiled/$tree" --project "$tree" --write-marker \
|
||
| python3 -c "import json,sys; r=json.load(sys.stdin); print(f' $tree: ok={r[\"ok\"]} exts={r[\"extension_count\"]}'); sys.exit(0 if r['ok'] else 1)" \
|
||
|| { echo "$tree 审计失败" >&2; exit 2; }
|
||
"$py" "$AUDIT" --tree "$BUILD_ROOT/compiled/$tree" --project "$tree" --require-marker \
|
||
>/dev/null || { echo "$tree marker 校验失败" >&2; exit 2; }
|
||
done
|
||
|
||
# ---- 3. 冒烟(起服务探 /health,全 .so import 检查) ----
|
||
echo "[3/6] 冒烟(import 检查,服务起停由部署脚本承担)"
|
||
for tree in sdk qa methodology legacy-rag; do
|
||
py="$SDK_PYTHON"; [ "$tree" = "qa" ] && py="$QA_PYTHON"; [ "$tree" = "methodology" ] && py="$METHODOLOGY_PYTHON"; [ "$tree" = "legacy-rag" ] && py="$LEGACY_RUN_PYTHON"
|
||
if [ "$tree" = "sdk" ]; then
|
||
( cd "$BUILD_ROOT/compiled/sdk" && PYTHONPATH=. "$py" -c "import app.main" )
|
||
elif [ "$tree" = "qa" ]; then
|
||
( cd "$BUILD_ROOT/compiled/qa" && PYTHONPATH=.:./src "$py" -c "import scripts.qa_v1_1_73_server" )
|
||
elif [ "$tree" = "methodology" ]; then
|
||
( cd "$BUILD_ROOT/compiled/methodology" && PYTHONPATH=. "$py" -c "import scripts.methodology_server" )
|
||
else
|
||
( cd "$BUILD_ROOT/compiled/legacy-rag" && PYTHONPATH=. "$py" -c "import api.app" )
|
||
# vstore 冒烟（与四树同模式：全 .so import 检查）
|
||
( cd "$BUILD_ROOT/compiled/vstore" && PYTHONPATH=. "$SDK_PYTHON" -c "import vstore.api" )
|
||
fi
|
||
echo " $tree import OK"
|
||
done
|
||
|
||
# ---- 4. KB(定版后 SKIP_KB=0 启用) ----
|
||
if [ "${SKIP_KB:-1}" = "0" ]; then
|
||
echo "[4/6] KB 加密打包(定版后)"
|
||
KB_KEY_FILE="${KB_KEY_FILE:-$BUILD_ROOT/kb.key}"
|
||
[ -f "$KB_KEY_FILE" ] || { echo "缺 KB 密钥: $KB_KEY_FILE" >&2; exit 1; }
|
||
KB_KEY_FILE="$KB_KEY_FILE" bash "$SCRIPT_DIR/kb/encrypt_kb.sh" "$VERSION"
|
||
else
|
||
echo "[4/6] KB 跳过(SKIP_KB=1,数据库未定版;操作见 tooling/KB_ENCRYPT_GUIDE.md)"
|
||
fi
|
||
|
||
# ---- 5. 组装安装目录 ----
|
||
echo "[5/6] 组装安装目录"
|
||
rm -rf "$RELEASE_ROOT"
|
||
mkdir -p "$RELEASE_ROOT"
|
||
cp -r "$BUILD_ROOT/compiled/sdk" "$RELEASE_ROOT/sdk"
|
||
cp -r "$BUILD_ROOT/compiled/qa" "$RELEASE_ROOT/qa"
|
||
cp -r "$BUILD_ROOT/compiled/methodology" "$RELEASE_ROOT/methodology"
|
||
cp -r "$BUILD_ROOT/compiled/legacy-rag" "$RELEASE_ROOT/legacy-rag"
|
||
# UI 产物
|
||
( cd "$PROJECT_ROOT/ui" && npm run build --silent ) || { echo "ui build 失败" >&2; exit 5; }
|
||
mkdir -p "$RELEASE_ROOT/ui"
|
||
cp -r "$PROJECT_ROOT/ui/dist" "$RELEASE_ROOT/ui/dist"
|
||
# 启动脚本模板
|
||
cp "$PROJECT_ROOT/agent/qa/scripts/"*.sh "$RELEASE_ROOT/qa/scripts/" 2>/dev/null || true
|
||
cp "$PROJECT_ROOT/agent/methodology/scripts/"*.sh "$RELEASE_ROOT/methodology/scripts/" 2>/dev/null || true
|
||
cp "$PROJECT_ROOT/agent/legacy-rag/scripts/"*.sh "$RELEASE_ROOT/legacy-rag/scripts/" 2>/dev/null || true
|
||
|
||
# 向量化环节（本地模型）：vstore 明文源码 + 启动脚本 + Ollama 拉起脚本
|
||
cp -r "$BUILD_ROOT/compiled/vstore/vstore" "$RELEASE_ROOT/vstore"
|
||
rm -rf "$RELEASE_ROOT/vstore/tests" "$RELEASE_ROOT/vstore/__pycache__"
|
||
cp "$PROJECT_ROOT/vstore/vstore_restart.sh" "$RELEASE_ROOT/vstore_restart.sh"
|
||
cp "$PROJECT_ROOT/deploy/ollama_restart.sh" "$RELEASE_ROOT/ollama_restart.sh"
|
||
# nginx 80 端口反代模板（目标机从零安装章节引用，见 docs/DEPLOY_FLOW.md §5b）
|
||
mkdir -p "$RELEASE_ROOT/nginx"
|
||
cp "$PROJECT_ROOT/deploy/nginx/plantai.conf" "$RELEASE_ROOT/nginx/plantai.conf"
|
||
# QA 数据库裸进程部署脚本（systemd 标准，见 docs/DEPLOY_FLOW.md §5「QA 数据库」）
|
||
mkdir -p "$RELEASE_ROOT/qa-db"
|
||
cp -r "$PROJECT_ROOT/deploy/qa-db/"*.sh "$RELEASE_ROOT/qa-db/" 2>/dev/null || true
|
||
cp -r "$PROJECT_ROOT/deploy/qa-db/"*.service "$RELEASE_ROOT/qa-db/" 2>/dev/null || true
|
||
# 离线二进制载体（deploy/qa-db/bin/ 为 gitignore；存在则随包分发，目标机免下载）
|
||
[ -d "$PROJECT_ROOT/deploy/qa-db/bin" ] && cp -r "$PROJECT_ROOT/deploy/qa-db/bin" "$RELEASE_ROOT/qa-db/bin" || true
|
||
# QA reranker 模型入包（构建机仓库同级 PlantAI-rag；缺失只告警不阻断）
|
||
RERANKER_SRC="$PROJECT_ROOT/../PlantAI-rag/bge-reranker-base"
|
||
if [ -d "$RERANKER_SRC" ]; then
|
||
mkdir -p "$RELEASE_ROOT/models"
|
||
cp -r "$RERANKER_SRC" "$RELEASE_ROOT/models/bge-reranker-base"
|
||
echo "reranker 模型已入包: models/bge-reranker-base"
|
||
else
|
||
echo "警告: reranker 模型源不存在($RERANKER_SRC)，安装包不含本地 reranker"
|
||
fi
|
||
# env 模板（VSTORE_ROOT/KEY/EMBED 等；真实值不入包）
|
||
cp "$PROJECT_ROOT/vstore/.env.example" "$RELEASE_ROOT/vstore/.env.example" 2>/dev/null || true
|
||
|
||
# env 模板
|
||
for t in qa methodology legacy-rag; do
|
||
mkdir -p "$RELEASE_ROOT/$t/env"
|
||
cp "$PROJECT_ROOT/agent/$t/env/prod.env.example" "$RELEASE_ROOT/$t/env/prod.env.example" 2>/dev/null || true
|
||
done
|
||
cp "$PROJECT_ROOT/sdk/.env.example" "$RELEASE_ROOT/sdk/.env.example" 2>/dev/null || true
|
||
# ---- 库身份装包:manifest + configs + active_library.env ----
|
||
mkdir -p "$RELEASE_ROOT/install/kb" "$RELEASE_ROOT/qa/env"
|
||
cp "$LIBRARY_DIR/manifest.yaml" "$RELEASE_ROOT/install/kb/manifest.yaml"
|
||
if [ -d "$LIBRARY_DIR/configs" ]; then
|
||
cp "$LIBRARY_DIR"/configs/*.yaml "$RELEASE_ROOT/qa/configs/rag/" 2>/dev/null || true
|
||
fi
|
||
bash "$PROJECT_ROOT/tooling/aqua_kb/render_active_library_env.sh" \
|
||
"$LIBRARY_DIR/manifest.yaml" "$RELEASE_ROOT/qa/env/active_library.env"
|
||
# KB 包(定版后)
|
||
[ -f "$BUILD_ROOT/knowledge-$VERSION.tar" ] && cp "$BUILD_ROOT/knowledge-$VERSION.tar"* "$RELEASE_ROOT/"
|
||
# 部署脚本
|
||
cat > "$RELEASE_ROOT/start_all.sh" <<'DEPLOY'
|
||
#!/usr/bin/env bash
|
||
# start_all.sh —— 安装目录一键启动(端口 8090 段)
|
||
# 用法:bash start_all.sh [bind] (bind 默认 127.0.0.1——nginx 80 为唯一客户入口)
|
||
set -euo pipefail
|
||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
BIND="${1:-127.0.0.1}"
|
||
# Python 解析：安装包内 venv 优先（install/venv，可链到部署根），缺失退回 PATH python3
|
||
PYBIN="${PLANTAI_PYTHON:-$ROOT/venv/bin/python}"
|
||
[[ -x "$PYBIN" ]] || PYBIN="$(command -v python3)"
|
||
echo "== 启动 PlantAI-rag-new 四服务 =="
|
||
# 向量化本地模型（bge-m3 嵌入）：先拉起 Ollama（已运行则跳过），模型缺失只告警不阻断
|
||
bash "$ROOT/ollama_restart.sh" 2>/dev/null || echo "[ollama] 拉起失败，向量化降级（不阻断其余服务）"
|
||
bash "$ROOT/qa/scripts/start.sh" 8091 || true
|
||
bash "$ROOT/methodology/scripts/start.sh" 8092 || true
|
||
bash "$ROOT/legacy-rag/scripts/start.sh" 8093 app || true
|
||
bash "$ROOT/vstore_restart.sh" || echo "[vstore] 拉起失败，向量化不可用（不阻断其余服务）"
|
||
# 生信服务（可选编排，2026-09-12 陷阱 6）：主动探测独立生信部署树是否存在，
|
||
# 存在则启动/重启（restart.sh 自带「有旧进程先 kill、无则直接起」语义），
|
||
# 不存在则跳过——未部署生信的机器无此树，不应报错。失败不阻断主服务。
|
||
# 布局约定：生信树在 install 同级 bioflow/（176: /srv/plantai/bioflow/）。
|
||
BIOFLOW_RESTART="$ROOT/../bioflow/bioflow_service/restart.sh"
|
||
# 2026-09-17:生信开关来自 sdk/.env(与环境变量同名时后者优先;未设默认 false)
|
||
[ -f "$ROOT/sdk/.env" ] && . "$ROOT/sdk/.env"
|
||
BIOFLOW_ENABLED_VAL="${BIOFLOW_ENABLED:-false}"
|
||
if [ "$BIOFLOW_ENABLED_VAL" = "true" ] && [ -x "$BIOFLOW_RESTART" ]; then
|
||
echo "[bioflow] 探测到生信服务，启动/重启中"
|
||
if ( bash "$BIOFLOW_RESTART" >/dev/null 2>&1 && sleep 3 \
|
||
&& curl -s --max-time 8 http://127.0.0.1:8098/api/bioflow/health >/dev/null ); then
|
||
echo "[bioflow] 8098 health ok"
|
||
else
|
||
echo "[bioflow] 拉起/探活失败，生信不可用（不阻断主服务；日志 bioflow_service/bioflow.log）"
|
||
fi
|
||
else
|
||
echo "[bioflow] 未部署生信服务，跳过"
|
||
fi
|
||
sleep 3
|
||
# BFF env：sdk/.env 含 DOCGEN_LLM_*、VSTORE_BASE_URL、BFF_NEO4J_*（无此 env BFF 上游不可用）
|
||
set -a
|
||
[ -f "$ROOT/sdk/.env" ] && . "$ROOT/sdk/.env"
|
||
set +a
|
||
( cd "$ROOT/sdk" && setsid "$PYBIN" -m uvicorn app.main:app --host "$BIND" --port 8090 \
|
||
> /tmp/plantai-sdk.log 2>&1 < /dev/null & )
|
||
disown -a 2>/dev/null || true
|
||
# 2026-09-12 部署实测:BFF 段缺 disown 时,脚本尾部 bash 隐式 wait 长驻 BFF 进程,
|
||
# start_all 永不返回(vstore_restart.sh 有 disown 故不卡)。此处补上。
|
||
echo "BFF 启动中(8090),5 秒后自检:"
|
||
sleep 5
|
||
curl -s --max-time 15 http://127.0.0.1:8090/api/health | head -c 300
|
||
echo
|
||
DEPLOY
|
||
chmod +x "$RELEASE_ROOT/start_all.sh"
|
||
|
||
# ---- 6. SHA256SUMS + 泄露门禁 ----
|
||
echo "[6/6] SHA256SUMS + 泄露检查"
|
||
( cd "$RELEASE_ROOT" && find . -type f -exec sha256sum {} \; | sort -k2 > SHA256SUMS )
|
||
LEAKS=$(grep -rnE "BEGIN (RSA |EC |OPENSSH )?PRIVATE KEY|sk-[a-zA-Z0-9]{20}|kb\.key" "$RELEASE_ROOT" 2>/dev/null | grep -v SHA256SUMS | head -5 || true)
|
||
if [ -n "$LEAKS" ]; then
|
||
echo "泄露门禁失败:" >&2; echo "$LEAKS" >&2; exit 6
|
||
fi
|
||
echo "泄露检查: 净"
|
||
echo
|
||
echo "== 构建完成: $RELEASE_ROOT =="
|
||
du -sh "$RELEASE_ROOT"
|
||
echo "下一步:空目录全新安装验收(备用端口 8094/8095/8096/8097)"
|