chore(deploy): vstore 编译硬化纳入打包流水线——目标机不再出现向量库明文代码
- tooling/compile/compile_vstore.sh(新): 复制 compile_sdk.sh 改白名单 (api.py 入口明文+__init__.py,其余 12 模块 Cython .so+strip+elf 校验) - build_all.sh: 编译段追加 vstore 编译(SDK_PYTHON);冒烟段追加 import 检查; 组装段改复制编译产物(此前复制明文源码) - 实测: 编译 12 .so 成功;冒烟 health+search-chunks 正常;bash -n 过 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -33,6 +33,10 @@ bash "$SCRIPT_DIR/compile/compile_methodology.sh" \
|
||||
bash "$SCRIPT_DIR/compile/compile_legacy.sh" \
|
||||
"$PROJECT_ROOT/agent/legacy-rag" "$BUILD_ROOT/compiled/legacy-rag" "$LEGACY_PYTHON"
|
||||
|
||||
# 向量库服务(本地嵌入 + 私有资料问答检索;api.py 白名单明文,其余编译 .so)
|
||||
bash "$SCRIPT_DIR/compile/compile_vstore.sh" \
|
||||
"$PROJECT_ROOT/vstore" "$BUILD_ROOT/compiled/vstore" "$SDK_PYTHON"
|
||||
|
||||
# ---- 2. 双审计(各树用编译时的 Python——ABI 后缀必须匹配) ----
|
||||
echo "[2/6] 编译树审计"
|
||||
AUDIT="$SCRIPT_DIR/compile/tools/audit_compiled_tree.py"
|
||||
@@ -58,6 +62,8 @@ for tree in sdk qa methodology legacy-rag; do
|
||||
( cd "$BUILD_ROOT/compiled/methodology" && PYTHONPATH=. "$py" -c "import scripts.methodology_server" )
|
||||
else
|
||||
( cd "$BUILD_ROOT/compiled/legacy-rag" && PYTHONPATH=. "$py" -c "import api.app" )
|
||||
# vstore 冒烟(与四树同模式:全 .so import 检查)
|
||||
( cd "$BUILD_ROOT/compiled/vstore" && PYTHONPATH=. "$SDK_PYTHON" -c "import vstore.api" )
|
||||
fi
|
||||
echo " $tree import OK"
|
||||
done
|
||||
@@ -90,7 +96,7 @@ cp "$PROJECT_ROOT/agent/methodology/scripts/"*.sh "$RELEASE_ROOT/methodology/scr
|
||||
cp "$PROJECT_ROOT/agent/legacy-rag/scripts/"*.sh "$RELEASE_ROOT/legacy-rag/scripts/" 2>/dev/null || true
|
||||
|
||||
# 向量化环节(本地模型):vstore 明文源码 + 启动脚本 + Ollama 拉起脚本
|
||||
cp -r "$PROJECT_ROOT/vstore" "$RELEASE_ROOT/vstore"
|
||||
cp -r "$BUILD_ROOT/compiled/vstore/vstore" "$RELEASE_ROOT/vstore"
|
||||
rm -rf "$RELEASE_ROOT/vstore/tests" "$RELEASE_ROOT/vstore/__pycache__"
|
||||
cp "$PROJECT_ROOT/vstore/vstore_restart.sh" "$RELEASE_ROOT/vstore_restart.sh"
|
||||
cp "$PROJECT_ROOT/deploy/ollama_restart.sh" "$RELEASE_ROOT/ollama_restart.sh"
|
||||
|
||||
Executable
+102
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env bash
|
||||
# compile_vstore.sh —— 向量库服务(vstore)编译硬化。
|
||||
# 白名单(明文):api.py(FastAPI 入口 + 运维可读面,与 sdk main.py 同理由) + __init__.py。
|
||||
# 用法:compile_vstore.sh <source-root> <output-dir> [python]
|
||||
set -euo pipefail
|
||||
umask 027
|
||||
|
||||
SOURCE_ROOT="${1:?Usage: $0 <source-root> <output-dir> [python]}"
|
||||
OUTPUT_DIR="${2:?Usage: $0 <source-root> <output-dir> [python]}"
|
||||
PYTHON="${3:-python3}"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
VISIBILITY_TOOL="${ELF_VISIBILITY_TOOL:-$SCRIPT_DIR/tools/elf_visibility.py}"
|
||||
|
||||
SOURCE_ROOT="$(realpath "$SOURCE_ROOT")"
|
||||
OUTPUT_DIR="$(realpath -m "$OUTPUT_DIR")"
|
||||
|
||||
[[ -d "$SOURCE_ROOT" ]] || { echo "vstore source missing: $SOURCE_ROOT" >&2; exit 2; }
|
||||
[[ -x "$PYTHON" ]] || { echo "Python is not executable: $PYTHON" >&2; exit 2; }
|
||||
[[ -f "$VISIBILITY_TOOL" ]] || { echo "ELF visibility tool missing: $VISIBILITY_TOOL" >&2; exit 2; }
|
||||
for command_name in gcc strip rsync; do command -v "$command_name" >/dev/null; done
|
||||
|
||||
case "$OUTPUT_DIR" in
|
||||
"/"|"$(realpath -m "$HOME")"|"$SOURCE_ROOT")
|
||||
echo "Unsafe output directory: $OUTPUT_DIR" >&2; exit 2 ;;
|
||||
esac
|
||||
[[ ! -e "$OUTPUT_DIR" ]] || { echo "Output exists; refusing to overwrite: $OUTPUT_DIR" >&2; exit 2; }
|
||||
|
||||
echo "[VSTORE 1/5] Copying source (excluding tests)"
|
||||
mkdir -p "$OUTPUT_DIR/vstore"
|
||||
rsync -a --exclude '__pycache__' --exclude 'test_*.py' --exclude 'tests' --exclude '.env' \
|
||||
"$SOURCE_ROOT/" "$OUTPUT_DIR/vstore/"
|
||||
# 修正工作根:编译对象是包内容,模块名 vstore.* 由 OUTPUT_DIR/vstore 承载
|
||||
|
||||
# 白名单明文:api.py(FastAPI 入口,uvicorn 以模块字符串加载的入口面)
|
||||
# + __init__.py(包标记)。其余(vaul/chunks/embed/retrieval/parsing/backend/
|
||||
# crypto/settings/flow 等)全部编译 .so。
|
||||
WHITELIST_PY=("api.py")
|
||||
_is_whitelisted() {
|
||||
local rel="$1"
|
||||
for w in "${WHITELIST_PY[@]}"; do [[ "$rel" == "$w" ]] && return 0; done
|
||||
[[ "$(basename "$rel")" == "__init__.py" ]] && return 0
|
||||
return 1
|
||||
}
|
||||
|
||||
echo "[VSTORE 2/5] Discovering Python modules"
|
||||
PY_FILES=()
|
||||
while IFS= read -r -d '' path; do PY_FILES+=("$path"); done \
|
||||
< <(find "$OUTPUT_DIR/vstore" -type f -name '*.py' -print0)
|
||||
FILTERED=()
|
||||
for py_file in "${PY_FILES[@]}"; do
|
||||
rel="${py_file#"$OUTPUT_DIR/vstore"/}"
|
||||
if _is_whitelisted "$rel"; then
|
||||
echo " [whitelist] $rel (明文保留)"
|
||||
continue
|
||||
fi
|
||||
FILTERED+=("$py_file")
|
||||
done
|
||||
PY_FILES=("${FILTERED[@]}")
|
||||
TOTAL="${#PY_FILES[@]}"
|
||||
(( TOTAL > 0 )) || { echo "No vstore modules selected" >&2; exit 3; }
|
||||
|
||||
INCLUDE_PY="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("INCLUDEPY") or "")')"
|
||||
EXT_SUFFIX="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX") or ".so")')"
|
||||
|
||||
echo "[VSTORE 3/5] Cython preflight ($TOTAL modules)"
|
||||
COUNT=0; FAILURES=0
|
||||
for py_file in "${PY_FILES[@]}"; do
|
||||
COUNT=$((COUNT+1))
|
||||
directory="$(dirname "$py_file")"
|
||||
module="$(basename "$py_file" .py)"
|
||||
c_file="$directory/$module.c"
|
||||
echo " [$COUNT/$TOTAL] ${py_file#"$OUTPUT_DIR/vstore"/}"
|
||||
if ! "$PYTHON" -m cython -3 \
|
||||
--directive always_allow_keywords=true,annotation_typing=False \
|
||||
--no-docstrings "$py_file" -o "$c_file" 2>/dev/null; then
|
||||
FAILURES=$((FAILURES+1))
|
||||
fi
|
||||
done
|
||||
(( FAILURES == 0 )) || { echo "vstore Cython failed: $FAILURES" >&2; exit 3; }
|
||||
|
||||
echo "[VSTORE 4/5] GCC hidden visibility + strip"
|
||||
COUNT=0
|
||||
for py_file in "${PY_FILES[@]}"; do
|
||||
COUNT=$((COUNT+1))
|
||||
directory="$(dirname "$py_file")"
|
||||
module="$(basename "$py_file" .py)"
|
||||
c_file="$directory/$module.c"
|
||||
so_file="$directory/$module$EXT_SUFFIX"
|
||||
gcc -shared -pthread -fPIC -O2 -fvisibility=hidden \
|
||||
-I"$INCLUDE_PY" -o "$so_file" "$c_file"
|
||||
strip --strip-unneeded "$so_file"
|
||||
"$PYTHON" -B "$VISIBILITY_TOOL" --hide-non-pyinit --check "$so_file" >/dev/null
|
||||
rm -f -- "$c_file" "$py_file"
|
||||
done
|
||||
|
||||
echo "[VSTORE 5/5] Final checks"
|
||||
SO_COUNT="$(find "$OUTPUT_DIR/vstore" -type f -name '*.so' | wc -l)"
|
||||
[[ "$SO_COUNT" -eq "$TOTAL" ]] || { echo "count mismatch: $TOTAL vs $SO_COUNT" >&2; exit 4; }
|
||||
if find "$OUTPUT_DIR/vstore" -type f -name '*.c' -print -quit | grep -q .; then
|
||||
echo "Generated C source remains" >&2; exit 4
|
||||
fi
|
||||
echo "VSTORE compile+harden complete: modules=$SO_COUNT output=$OUTPUT_DIR"
|
||||
Reference in New Issue
Block a user