jerryandClaude Fable 5
5024a12947
- 加密工具双模式往返验证通过(字段级 properties_json / 整文件 parquet) - 取消真实加密执行:数据库未定版,中间副本已清理,kb.key 保留 - tooling/KB_ENCRYPT_GUIDE.md:完整操作步骤/踩坑记录/解密验证/流水线接入 - 踩坑固化:Qdrant 必须 snapshot API(禁 rsync 活动卷);Neo4j 5.15 community 不支持在线 dump,走 frozen 目录冷拷;canonical 库不在 KB 交付内 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
92 lines
4.6 KiB
Bash
Executable File
92 lines
4.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# encrypt_kb.sh —— 交付副本加密打包(生产库不动,单一 tar)。
|
|
# 流程:① 一致性快照(Qdrant snapshot API / neo4j-admin dump,绝不动活动卷)
|
|
# ② 断言导出文件 AES-256-GCM(raw_export.jsonl properties_json 字段 + parquet 整文件)
|
|
# ③ tar 单一文件(不压缩,92G 二进制)+ sha256
|
|
# ④ 计数校验(backend_guard 常量)
|
|
# 用法:bash encrypt_kb.sh <版本> [stage目录]
|
|
set -euo pipefail
|
|
umask 027
|
|
|
|
VERSION="${1:?用法: encrypt_kb.sh <版本>}"
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
TOOLING_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
STAGE_ROOT="${2:-$TOOLING_ROOT/build/$VERSION/knowledge-stage}"
|
|
OUT_DIR="$TOOLING_ROOT/build/$VERSION"
|
|
|
|
QDRANT_URL="${QDRANT_URL:-http://127.0.0.1:27333}"
|
|
QDRANT_COLLECTION="${V15_QDRANT_COLLECTION:-plant_srt_pc2025_v3_20260817_normalized}"
|
|
NEO4J_NORM_NAME="neo4j-plant-srt-pc2025-v3-20260817-normalized"
|
|
NEO4J_DB="${NEO4J_DB:-neo4j}"
|
|
# 2026-08-31 修正:KB 交付不含 canonical(18788 属 SDK KG 浏览依赖);
|
|
# normalized 数据目录 08-24 起无写入(frozen),在线 dump 不支持(5.15 community),改冷拷
|
|
STAGING="/home/xzq/phase2b-5090-staging-20260823T191150Z/plant_srt_pc2025_merge_20260817_v3_normalized"
|
|
ASSERT_DIR="$STAGING/frozen_inputs/ef0d58871e05afbb957d652a5fe932eda24b68b3655bbbc5feb40d5ae9c47dc8/baseline_assertions"
|
|
REGISTRY_SRC="/home/xzq/5090/teamWorkspace/zh/PlantAI-rag/data/knowledge_base/paper_metadata_registry_v1_20260717/paper_metadata_registry_full.jsonl"
|
|
KEY_FILE="${KB_KEY_FILE:-$OUT_DIR/kb.key}"
|
|
TAR_FILE="$OUT_DIR/knowledge-$VERSION.tar"
|
|
|
|
AVAIL_GB=$(df -BG "$OUT_DIR" 2>/dev/null | awk 'NR==2 {gsub("G","",$4); print $4}' || echo 0)
|
|
[[ "${AVAIL_GB:-0}" -ge 200 ]] || { echo "磁盘余量不足(需 ≥200G, 当前 ${AVAIL_GB}G)" >&2; exit 1; }
|
|
[ -f "$KEY_FILE" ] || { echo "缺少 KB 主密钥: $KEY_FILE (先运行 tooling/crypto/init_keys.sh)" >&2; exit 1; }
|
|
mkdir -p "$STAGE_ROOT" "$OUT_DIR"
|
|
|
|
# ---- ① 一致性快照 ----
|
|
echo "[1/5] Qdrant snapshot API 导出(在线一致,$QDRANT_COLLECTION)..."
|
|
curl -sf -X POST "$QDRANT_URL/collections/$QDRANT_COLLECTION/snapshots" > /tmp/qdrant_snap_resp.json
|
|
SNAP_NAME=$(python3 -c "import json; print(json.load(open('/tmp/qdrant_snap_resp.json'))['result']['name'])")
|
|
echo " snapshot: $SNAP_NAME(生成中,轮询...)"
|
|
for i in $(seq 1 120); do
|
|
sleep 10
|
|
curl -sf "$QDRANT_URL/collections/$QDRANT_COLLECTION/snapshots" | grep -q "\"name\":\"$SNAP_NAME\"" && break
|
|
done
|
|
# 从容器拷贝 snapshot(容器内路径 /qdrant/snapshots/<collection>/<name>)
|
|
docker cp "$(docker ps -q -f name=qdrant-plant-srt-pc2025-v3-20260817-normalized):/qdrant/snapshots/$QDRANT_COLLECTION/$SNAP_NAME" "$STAGE_ROOT/" 2>/dev/null \
|
|
|| { echo "snapshot 拷贝失败" >&2; exit 1; }
|
|
echo " Qdrant snapshot 拷贝完成: $(du -sh "$STAGE_ROOT/$SNAP_NAME" | cut -f1)"
|
|
|
|
echo "[1/5] Neo4j normalized 冷拷(8.2G,数据目录 08-24 起无写入=frozen)..."
|
|
NEO4J_NORM_SRC="/home/xzq/phase2b-5090-staging-20260823T191150Z/plant_srt_pc2025_merge_20260817_v3_normalized/databases/neo4j/data"
|
|
rsync -a --info=progress2 "$NEO4J_NORM_SRC/" "$STAGE_ROOT/neo4j-normalized/data/" 2>&1 | tail -1
|
|
echo " normalized 冷拷: $(du -sh "$STAGE_ROOT/neo4j-normalized/data" | cut -f1)"
|
|
|
|
# ---- ② 断言导出文件加密 ----
|
|
mkdir -p "$STAGE_ROOT/assertions"
|
|
cp "$REGISTRY_SRC" "$STAGE_ROOT/registry.jsonl"
|
|
echo "[2/5] AES-GCM 加密 raw_export.jsonl properties_json(9.5G,数分钟)..."
|
|
python3 "$SCRIPT_DIR/kb_encrypt_backfill.py" \
|
|
--input "$ASSERT_DIR/raw_export.jsonl" \
|
|
--output "$STAGE_ROOT/assertions/raw_export.enc.jsonl" \
|
|
--key-file "$KEY_FILE" \
|
|
--fields properties_json
|
|
echo "[2/5] AES-GCM 整文件加密 assertions.parquet(2.7G)..."
|
|
python3 "$SCRIPT_DIR/kb_encrypt_backfill.py" \
|
|
--input "$ASSERT_DIR/assertions.parquet" \
|
|
--output "$STAGE_ROOT/assertions/assertions.parquet.enc" \
|
|
--key-file "$KEY_FILE" \
|
|
--file-encrypt
|
|
|
|
# ---- ③ tar 单一文件(不压缩)+ sha256 ----
|
|
echo "[3/5] 打包单一 tar(92G+,约10分钟)..."
|
|
tar -C "$STAGE_ROOT" -cf "$TAR_FILE" .
|
|
sha256sum "$TAR_FILE" > "$TAR_FILE.sha256"
|
|
echo "tar 完成: $TAR_FILE ($(du -sh "$TAR_FILE" | cut -f1))"
|
|
|
|
# ---- ④ 计数校验 ----
|
|
echo "[4/5] 计数校验(backend_guard 常量,解密安装后由 QA 验证)..."
|
|
cat > "$OUT_DIR/EXPECTED_COUNTS.json" <<JSON
|
|
{
|
|
"papers": 31207,
|
|
"canonical_entities": 3820996,
|
|
"assertions": 8246875,
|
|
"normalized_entities": 6316654,
|
|
"qdrant_points": 12963472,
|
|
"neo4j_asserts": 12963472,
|
|
"neo4j_supported_by": 13012368,
|
|
"neo4j_papers": 96648
|
|
}
|
|
JSON
|
|
echo " 期望计数已写入 EXPECTED_COUNTS.json"
|
|
echo "[5/5] KB 加密打包完成: $TAR_FILE"
|
|
echo " sha256: $(cat "$TAR_FILE.sha256")"
|