From 7b63765d693ffa3c51cc4a366b7ed30e883f2cb4 Mon Sep 17 00:00:00 2001 From: admin Date: Tue, 8 Sep 2026 01:25:44 +0800 Subject: [PATCH] =?UTF-8?q?chore(deploy):=20vstore=20=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E7=A1=AC=E5=8C=96=E7=BA=B3=E5=85=A5=E6=89=93=E5=8C=85=E6=B5=81?= =?UTF-8?q?=E6=B0=B4=E7=BA=BF=E2=80=94=E2=80=94=E7=9B=AE=E6=A0=87=E6=9C=BA?= =?UTF-8?q?=E4=B8=8D=E5=86=8D=E5=87=BA=E7=8E=B0=E5=90=91=E9=87=8F=E5=BA=93?= =?UTF-8?q?=E6=98=8E=E6=96=87=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - tooling/compile/compile_vstore.sh(新): 复制 compile_sdk.sh 改白名单 (api.py 入口明文+__init__.py,其余 12 模块 Cython .so+strip+elf 校验) - build_all.sh: 编译段追加 vstore 编译(SDK_PYTHON);冒烟段追加 import 检查; 组装段改复制编译产物(此前复制明文源码) - 实测: 编译 12 .so 成功;冒烟 health+search-chunks 正常;bash -n 过 Co-Authored-By: Claude Fable 5 --- tooling/build_all.sh | 8 ++- tooling/compile/compile_vstore.sh | 102 ++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) create mode 100755 tooling/compile/compile_vstore.sh diff --git a/tooling/build_all.sh b/tooling/build_all.sh index 1fad21e..dbb1c15 100755 --- a/tooling/build_all.sh +++ b/tooling/build_all.sh @@ -33,6 +33,10 @@ bash "$SCRIPT_DIR/compile/compile_methodology.sh" \ bash "$SCRIPT_DIR/compile/compile_legacy.sh" \ "$PROJECT_ROOT/agent/legacy-rag" "$BUILD_ROOT/compiled/legacy-rag" "$LEGACY_PYTHON" +# 向量库服务（本地嵌入 + 私有资料问答检索；api.py 白名单明文，其余编译 .so） +bash "$SCRIPT_DIR/compile/compile_vstore.sh" \ + "$PROJECT_ROOT/vstore" "$BUILD_ROOT/compiled/vstore" "$SDK_PYTHON" + # ---- 2. 双审计(各树用编译时的 Python——ABI 后缀必须匹配) ---- echo "[2/6] 编译树审计" AUDIT="$SCRIPT_DIR/compile/tools/audit_compiled_tree.py" @@ -58,6 +62,8 @@ for tree in sdk qa methodology legacy-rag; do ( cd "$BUILD_ROOT/compiled/methodology" && PYTHONPATH=. "$py" -c "import scripts.methodology_server" ) else ( cd "$BUILD_ROOT/compiled/legacy-rag" && PYTHONPATH=. "$py" -c "import api.app" ) +# vstore 冒烟（与四树同模式：全 .so import 检查） +( cd "$BUILD_ROOT/compiled/vstore" && PYTHONPATH=. "$SDK_PYTHON" -c "import vstore.api" ) fi echo " $tree import OK" done @@ -90,7 +96,7 @@ cp "$PROJECT_ROOT/agent/methodology/scripts/"*.sh "$RELEASE_ROOT/methodology/scr cp "$PROJECT_ROOT/agent/legacy-rag/scripts/"*.sh "$RELEASE_ROOT/legacy-rag/scripts/" 2>/dev/null || true # 向量化环节（本地模型）：vstore 明文源码 + 启动脚本 + Ollama 拉起脚本 -cp -r "$PROJECT_ROOT/vstore" "$RELEASE_ROOT/vstore" +cp -r "$BUILD_ROOT/compiled/vstore/vstore" "$RELEASE_ROOT/vstore" rm -rf "$RELEASE_ROOT/vstore/tests" "$RELEASE_ROOT/vstore/__pycache__" cp "$PROJECT_ROOT/vstore/vstore_restart.sh" "$RELEASE_ROOT/vstore_restart.sh" cp "$PROJECT_ROOT/deploy/ollama_restart.sh" "$RELEASE_ROOT/ollama_restart.sh" diff --git a/tooling/compile/compile_vstore.sh b/tooling/compile/compile_vstore.sh new file mode 100755 index 0000000..16b8d3a --- /dev/null +++ b/tooling/compile/compile_vstore.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# compile_vstore.sh —— 向量库服务(vstore)编译硬化。 +# 白名单(明文):api.py(FastAPI 入口 + 运维可读面,与 sdk main.py 同理由) + __init__.py。 +# 用法:compile_vstore.sh [python] +set -euo pipefail +umask 027 + +SOURCE_ROOT="${1:?Usage: $0 [python]}" +OUTPUT_DIR="${2:?Usage: $0 [python]}" +PYTHON="${3:-python3}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VISIBILITY_TOOL="${ELF_VISIBILITY_TOOL:-$SCRIPT_DIR/tools/elf_visibility.py}" + +SOURCE_ROOT="$(realpath "$SOURCE_ROOT")" +OUTPUT_DIR="$(realpath -m "$OUTPUT_DIR")" + +[[ -d "$SOURCE_ROOT" ]] || { echo "vstore source missing: $SOURCE_ROOT" >&2; exit 2; } +[[ -x "$PYTHON" ]] || { echo "Python is not executable: $PYTHON" >&2; exit 2; } +[[ -f "$VISIBILITY_TOOL" ]] || { echo "ELF visibility tool missing: $VISIBILITY_TOOL" >&2; exit 2; } +for command_name in gcc strip rsync; do command -v "$command_name" >/dev/null; done + +case "$OUTPUT_DIR" in + "/"|"$(realpath -m "$HOME")"|"$SOURCE_ROOT") + echo "Unsafe output directory: $OUTPUT_DIR" >&2; exit 2 ;; +esac +[[ ! -e "$OUTPUT_DIR" ]] || { echo "Output exists; refusing to overwrite: $OUTPUT_DIR" >&2; exit 2; } + +echo "[VSTORE 1/5] Copying source (excluding tests)" +mkdir -p "$OUTPUT_DIR/vstore" +rsync -a --exclude '__pycache__' --exclude 'test_*.py' --exclude 'tests' --exclude '.env' \ + "$SOURCE_ROOT/" "$OUTPUT_DIR/vstore/" +# 修正工作根：编译对象是包内容，模块名 vstore.* 由 OUTPUT_DIR/vstore 承载 + +# 白名单明文:api.py(FastAPI 入口,uvicorn 以模块字符串加载的入口面) +# + __init__.py(包标记)。其余(vaul/chunks/embed/retrieval/parsing/backend/ +# crypto/settings/flow 等)全部编译 .so。 +WHITELIST_PY=("api.py") +_is_whitelisted() { + local rel="$1" + for w in "${WHITELIST_PY[@]}"; do [[ "$rel" == "$w" ]] && return 0; done + [[ "$(basename "$rel")" == "__init__.py" ]] && return 0 + return 1 +} + +echo "[VSTORE 2/5] Discovering Python modules" +PY_FILES=() +while IFS= read -r -d '' path; do PY_FILES+=("$path"); done \ + < <(find "$OUTPUT_DIR/vstore" -type f -name '*.py' -print0) +FILTERED=() +for py_file in "${PY_FILES[@]}"; do + rel="${py_file#"$OUTPUT_DIR/vstore"/}" + if _is_whitelisted "$rel"; then + echo " [whitelist] $rel (明文保留)" + continue + fi + FILTERED+=("$py_file") +done +PY_FILES=("${FILTERED[@]}") +TOTAL="${#PY_FILES[@]}" +(( TOTAL > 0 )) || { echo "No vstore modules selected" >&2; exit 3; } + +INCLUDE_PY="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("INCLUDEPY") or "")')" +EXT_SUFFIX="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX") or ".so")')" + +echo "[VSTORE 3/5] Cython preflight ($TOTAL modules)" +COUNT=0; FAILURES=0 +for py_file in "${PY_FILES[@]}"; do + COUNT=$((COUNT+1)) + directory="$(dirname "$py_file")" + module="$(basename "$py_file" .py)" + c_file="$directory/$module.c" + echo " [$COUNT/$TOTAL] ${py_file#"$OUTPUT_DIR/vstore"/}" + if ! "$PYTHON" -m cython -3 \ + --directive always_allow_keywords=true,annotation_typing=False \ + --no-docstrings "$py_file" -o "$c_file" 2>/dev/null; then + FAILURES=$((FAILURES+1)) + fi +done +(( FAILURES == 0 )) || { echo "vstore Cython failed: $FAILURES" >&2; exit 3; } + +echo "[VSTORE 4/5] GCC hidden visibility + strip" +COUNT=0 +for py_file in "${PY_FILES[@]}"; do + COUNT=$((COUNT+1)) + directory="$(dirname "$py_file")" + module="$(basename "$py_file" .py)" + c_file="$directory/$module.c" + so_file="$directory/$module$EXT_SUFFIX" + gcc -shared -pthread -fPIC -O2 -fvisibility=hidden \ + -I"$INCLUDE_PY" -o "$so_file" "$c_file" + strip --strip-unneeded "$so_file" + "$PYTHON" -B "$VISIBILITY_TOOL" --hide-non-pyinit --check "$so_file" >/dev/null + rm -f -- "$c_file" "$py_file" +done + +echo "[VSTORE 5/5] Final checks" +SO_COUNT="$(find "$OUTPUT_DIR/vstore" -type f -name '*.so' | wc -l)" +[[ "$SO_COUNT" -eq "$TOTAL" ]] || { echo "count mismatch: $TOTAL vs $SO_COUNT" >&2; exit 4; } +if find "$OUTPUT_DIR/vstore" -type f -name '*.c' -print -quit | grep -q .; then + echo "Generated C source remains" >&2; exit 4 +fi +echo "VSTORE compile+harden complete: modules=$SO_COUNT output=$OUTPUT_DIR"