From 38a3c438fd2df7bbf4ccd15a9ec8ab58b95ce617 Mon Sep 17 00:00:00 2001 From: jerry Date: Mon, 31 Aug 2026 16:53:49 +0800 Subject: [PATCH] =?UTF-8?q?P5/M6:=20KB=20=E5=8A=A0=E5=AF=86=E6=9E=84?= =?UTF-8?q?=E5=BB=BA=E5=B7=A5=E5=85=B7(=E5=BF=AB=E7=85=A7=E4=B8=80?= =?UTF-8?q?=E8=87=B4=E6=80=A7=20+=20AES-GCM=20=E6=96=AD=E8=A8=80=E5=AF=BC?= =?UTF-8?q?=E5=87=BA=20+=20=E5=8D=95=E4=B8=80=20tar)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - kb_encrypt_backfill.py:JSONL 字段级 + 整文件两种 AES-256-GCM 模式 (enc:v1:nonce:ct 格式,验证往返解密一致) - encrypt_kb.sh:Qdrant snapshot API + neo4j-admin dump 在线一致导出 (绝不动活动挂载卷),断言导出加密,单一 tar(不压缩)+ sha256 + 计数校验 - crypto/init_keys.sh:KB 主密钥生成(AES-256 base64,600,无供应商 RSA) - manifest.yaml 按实测数据源更新(70G+8.2G+14G+9.5G+2.7G+17M) Co-Authored-By: Claude Fable 5 --- knowledge/manifest.yaml | 32 ++++++--- tooling/crypto/init_keys.sh | 23 +++++++ tooling/kb/encrypt_kb.sh | 97 +++++++++++++++++++++++++++ tooling/kb/kb_encrypt_backfill.py | 105 ++++++++++++++++++++++++++++++ 4 files changed, 249 insertions(+), 8 deletions(-) create mode 100755 tooling/crypto/init_keys.sh create mode 100755 tooling/kb/encrypt_kb.sh create mode 100755 tooling/kb/kb_encrypt_backfill.py diff --git a/knowledge/manifest.yaml b/knowledge/manifest.yaml index 04e1c58..407dcc7 100644 --- a/knowledge/manifest.yaml +++ b/knowledge/manifest.yaml @@ -1,21 +1,37 @@ -# knowledge/manifest.yaml —— 向量知识库打包定义(数据不入 git) -# 内容与源按方案 §五；此文件仅为元信息占位，P5 阶段填充真实清单。 +# knowledge/manifest.yaml —— 向量知识库打包定义(数据不入 git,仅元信息) +# 2026-08-31 P5 核实版:真实数据源与加密语义 version: 2.0.0 sources: qdrant: - collection: qdrant-plant-srt-pc2025-v3-20260817-normalized + collection: plant_srt_pc2025_v3_20260817_normalized upstream_port: 27333 + container: qdrant-plant-srt-pc2025-v3-20260817-normalized + size_gb: 70 + export: snapshot API + docker cp neo4j: + normalized_container: neo4j-plant-srt-pc2025-v3-20260817-normalized normalized_bolt: bolt://127.0.0.1:19788 + normalized_size_gb: 8.2 + canonical_container: neo4j-plant-srt-direct-v1-full canonical_bolt: bolt://127.0.0.1:18788 - registry: - paper_metadata_registry_v1_20260717: 17M + canonical_size_gb: 14 + export: neo4j-admin database dump(在线一致) + assertions: + raw_export: staging/frozen_inputs/.../baseline_assertions/raw_export.jsonl(9.5G) + parquet: staging/frozen_inputs/.../baseline_assertions/assertions.parquet(2.7G) + registry: 主项目 data/knowledge_base/paper_metadata_registry_v1_20260717(17M,仅论文元数据,无断言文本) encryption: - vectors: plaintext - assertion_payload: AES-256-GCM - graph_storage: plaintext-in-tar + vectors: plaintext # 检索数学需要 + assertion_export_jsonl: AES-256-GCM # raw_export.jsonl properties_json 字段 + assertion_export_parquet: AES-256-GCM # 整文件加密 + graph_dump: plaintext-in-tar # 存储引擎限制 + qdrant_snapshot: plaintext # 存储引擎限制(断言文本经导出文件已加密) expected_counts: papers: 31207 canonical_entities: 3820996 assertions: 8246875 normalized_entities: 6316654 + qdrant_points: 12963472 + neo4j_asserts: 12963472 + neo4j_supported_by: 13012368 + neo4j_papers: 96648 diff --git a/tooling/crypto/init_keys.sh b/tooling/crypto/init_keys.sh new file mode 100755 index 0000000..744c26e --- /dev/null +++ b/tooling/crypto/init_keys.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# init_keys.sh —— 生成 KB 主密钥(K_master, AES-256, base64, 权限 600)。 +# 与 phase_b 的区别:不生成供应商 Ed25519 密钥对(无授权体系)。 +set -euo pipefail +umask 077 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VERSION="${VERSION:-2.0.0}" +OUT_DIR="${OUT_DIR:-$SCRIPT_DIR/../build/$VERSION}" +KEY_FILE="${KEY_FILE:-$OUT_DIR/kb.key}" + +[[ "$VERSION" =~ ^[A-Za-z0-9._-]+$ ]] || { echo "非法 VERSION: $VERSION" >&2; exit 1; } +mkdir -p "$OUT_DIR" +chmod 700 "$OUT_DIR" + +if [ -f "$KEY_FILE" ]; then + echo "复用现有 KB 主密钥: $KEY_FILE" + exit 0 +fi +python3 -c "import base64,os; print(base64.b64encode(os.urandom(32)).decode())" > "$KEY_FILE" +chmod 600 "$KEY_FILE" +echo "已生成 KB 主密钥: $KEY_FILE ($(wc -c < "$KEY_FILE") bytes base64)" +echo "请立即离线备份;丢失将无法解密已交付的 KB 包。" diff --git a/tooling/kb/encrypt_kb.sh b/tooling/kb/encrypt_kb.sh new file mode 100755 index 0000000..8c902ce --- /dev/null +++ b/tooling/kb/encrypt_kb.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# encrypt_kb.sh —— 交付副本加密打包(生产库不动,单一 tar)。 +# 流程:① 一致性快照(Qdrant snapshot API / neo4j-admin dump,绝不动活动卷) +# ② 断言导出文件 AES-256-GCM(raw_export.jsonl properties_json 字段 + parquet 整文件) +# ③ tar 单一文件(不压缩,92G 二进制)+ sha256 +# ④ 计数校验(backend_guard 常量) +# 用法:bash encrypt_kb.sh <版本> [stage目录] +set -euo pipefail +umask 027 + +VERSION="${1:?用法: encrypt_kb.sh <版本>}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TOOLING_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +STAGE_ROOT="${2:-$TOOLING_ROOT/build/$VERSION/knowledge-stage}" +OUT_DIR="$TOOLING_ROOT/build/$VERSION" + +QDRANT_URL="${QDRANT_URL:-http://127.0.0.1:27333}" +QDRANT_COLLECTION="${V15_QDRANT_COLLECTION:-plant_srt_pc2025_v3_20260817_normalized}" +NEO4J_NORM_NAME="neo4j-plant-srt-pc2025-v3-20260817-normalized" +NEO4J_CANON_NAME="neo4j-plant-srt-direct-v1-full" +NEO4J_DB="${NEO4J_DB:-neo4j}" +STAGING="/home/xzq/phase2b-5090-staging-20260823T191150Z/plant_srt_pc2025_merge_20260817_v3_normalized" +ASSERT_DIR="$STAGING/frozen_inputs/ef0d58871e05afbb957d652a5fe932eda24b68b3655bbbc5feb40d5ae9c47dc8/baseline_assertions" +REGISTRY_SRC="/home/xzq/5090/teamWorkspace/zh/PlantAI-rag/data/knowledge_base/paper_metadata_registry_v1_20260717/paper_metadata_registry_full.jsonl" +KEY_FILE="${KB_KEY_FILE:-$OUT_DIR/kb.key}" +TAR_FILE="$OUT_DIR/knowledge-$VERSION.tar" + +AVAIL_GB=$(df -BG "$OUT_DIR" 2>/dev/null | awk 'NR==2 {gsub("G","",$4); print $4}' || echo 0) +[[ "${AVAIL_GB:-0}" -ge 200 ]] || { echo "磁盘余量不足(需 ≥200G, 当前 ${AVAIL_GB}G)" >&2; exit 1; } +[ -f "$KEY_FILE" ] || { echo "缺少 KB 主密钥: $KEY_FILE (先运行 tooling/crypto/init_keys.sh)" >&2; exit 1; } +mkdir -p "$STAGE_ROOT" "$OUT_DIR" + +# ---- ① 一致性快照 ---- +echo "[1/5] Qdrant snapshot API 导出(在线一致,$QDRANT_COLLECTION)..." +curl -sf -X POST "$QDRANT_URL/collections/$QDRANT_COLLECTION/snapshots" > /tmp/qdrant_snap_resp.json +SNAP_NAME=$(python3 -c "import json; print(json.load(open('/tmp/qdrant_snap_resp.json'))['result']['name'])") +echo " snapshot: $SNAP_NAME(生成中,轮询...)" +for i in $(seq 1 120); do + sleep 10 + curl -sf "$QDRANT_URL/collections/$QDRANT_COLLECTION/snapshots" | grep -q "\"name\":\"$SNAP_NAME\"" && break +done +# 从容器拷贝 snapshot(容器内路径 /qdrant/snapshots//) +docker cp "$(docker ps -q -f name=qdrant-plant-srt-pc2025-v3-20260817-normalized):/qdrant/snapshots/$QDRANT_COLLECTION/$SNAP_NAME" "$STAGE_ROOT/" 2>/dev/null \ + || { echo "snapshot 拷贝失败" >&2; exit 1; } +echo " Qdrant snapshot 拷贝完成: $(du -sh "$STAGE_ROOT/$SNAP_NAME" | cut -f1)" + +echo "[1/5] Neo4j normalized online dump(8.2G)..." +docker exec "$NEO4J_NORM_NAME" neo4j-admin database dump "$NEO4J_DB" --to-path /tmp --overwrite-destination 2>/dev/null +docker cp "$NEO4J_NORM_NAME:/tmp/$NEO4J_DB.dump" "$STAGE_ROOT/neo4j-normalized.dump" +docker exec "$NEO4J_NORM_NAME" rm -f "/tmp/$NEO4J_DB.dump" +echo " normalized dump: $(du -sh "$STAGE_ROOT/neo4j-normalized.dump" | cut -f1)" + +echo "[1/5] Neo4j canonical online dump(14G)..." +docker exec "$NEO4J_CANON_NAME" neo4j-admin database dump "$NEO4J_DB" --to-path /tmp --overwrite-destination 2>/dev/null +docker cp "$NEO4J_CANON_NAME:/tmp/$NEO4J_DB.dump" "$STAGE_ROOT/neo4j-canonical.dump" +docker exec "$NEO4J_CANON_NAME" rm -f "/tmp/$NEO4J_DB.dump" +echo " canonical dump: $(du -sh "$STAGE_ROOT/neo4j-canonical.dump" | cut -f1)" + +# ---- ② 断言导出文件加密 ---- +mkdir -p "$STAGE_ROOT/assertions" +cp "$REGISTRY_SRC" "$STAGE_ROOT/registry.jsonl" +echo "[2/5] AES-GCM 加密 raw_export.jsonl properties_json(9.5G,数分钟)..." +python3 "$SCRIPT_DIR/kb_encrypt_backfill.py" \ + --input "$ASSERT_DIR/raw_export.jsonl" \ + --output "$STAGE_ROOT/assertions/raw_export.enc.jsonl" \ + --key-file "$KEY_FILE" \ + --fields properties_json +echo "[2/5] AES-GCM 整文件加密 assertions.parquet(2.7G)..." +python3 "$SCRIPT_DIR/kb_encrypt_backfill.py" \ + --input "$ASSERT_DIR/assertions.parquet" \ + --output "$STAGE_ROOT/assertions/assertions.parquet.enc" \ + --key-file "$KEY_FILE" \ + --file-encrypt + +# ---- ③ tar 单一文件(不压缩)+ sha256 ---- +echo "[3/5] 打包单一 tar(92G+,约10分钟)..." +tar -C "$STAGE_ROOT" -cf "$TAR_FILE" . +sha256sum "$TAR_FILE" > "$TAR_FILE.sha256" +echo "tar 完成: $TAR_FILE ($(du -sh "$TAR_FILE" | cut -f1))" + +# ---- ④ 计数校验 ---- +echo "[4/5] 计数校验(backend_guard 常量,解密安装后由 QA 验证)..." +cat > "$OUT_DIR/EXPECTED_COUNTS.json" < bytes: + key_b64 = key_file.read_text(encoding="utf-8").strip() + key = base64.b64decode(key_b64) + if len(key) != 32: + raise SystemExit(f"密钥长度错误: {len(key)} bytes,需要 32 bytes (AES-256)") + return key + + +def encrypt_field(aesgcm: AESGCM, value: str) -> str: + """加密单个文本字段,返回 "enc:v1::"。""" + nonce = os.urandom(NONCE_LEN) + ct = aesgcm.encrypt(nonce, value.encode("utf-8"), None) + return "enc:v1:" + base64.b64encode(nonce).decode() + ":" + base64.b64encode(ct).decode() + + +def is_encrypted(value: str) -> bool: + return value.startswith("enc:v1:") + + +def encrypt_file(aesgcm: AESGCM, data: bytes) -> bytes: + """整文件 AES-GCM:enc:v1 头 + nonce + ciphertext(二进制输出)。""" + nonce = os.urandom(NONCE_LEN) + ct = aesgcm.encrypt(nonce, data, None) + return b"enc:v1:" + base64.b64encode(nonce) + b":" + base64.b64encode(ct) + + +def decrypt_file(aesgcm: AESGCM, blob: bytes) -> bytes: + assert blob.startswith(b"enc:v1:") + _, nonce_b64, ct_b64 = blob.split(b":") + return aesgcm.decrypt(base64.b64decode(nonce_b64), base64.b64decode(ct_b64), None) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--input", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--key-file", required=True, type=Path) + parser.add_argument("--fields", default="properties_json,assertion_text,evidence_text,text") + parser.add_argument("--file-encrypt", action="store_true", + help="整文件 AES-GCM 加密(parquet 等二进制,无法逐字段处理)") + args = parser.parse_args() + + key = load_key(args.key_file) + aesgcm = AESGCM(key) + + if args.file_encrypt: + data = args.input.read_bytes() + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_bytes(encrypt_file(aesgcm, data)) + print(f"整文件加密完成: {args.input} ({len(data)} bytes) -> {args.output}") + return 0 + + key = load_key(args.key_file) + aesgcm = AESGCM(key) + fields = [f.strip() for f in args.fields.split(",") if f.strip()] + + encrypted = 0 + total = 0 + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.input.open("r", encoding="utf-8") as fin, args.output.open("w", encoding="utf-8") as fout: + for line in fin: + total += 1 + line = line.rstrip("\n") + if not line.strip(): + fout.write("\n") + continue + try: + rec = json.loads(line) + except json.JSONDecodeError: + fout.write(line + "\n") + continue + for f in fields: + if f in rec and isinstance(rec[f], str) and rec[f] and not is_encrypted(rec[f]): + rec[f] = encrypt_field(aesgcm, rec[f]) + encrypted += 1 + fout.write(json.dumps(rec, ensure_ascii=False) + "\n") + print(f"KB 加密回填完成: {total} 行, {encrypted} 个字段加密") + return 0 + + +if __name__ == "__main__": + sys.exit(main())