PlantAI-rag-new/tooling/compile/compile_legacy.sh

#!/usr/bin/env bash
# Compile the legacy-rag route family service into stripped Cython extensions.
# 按 agent/legacy-rag/compile.manifest 的黄金清单(生产 7900 容器同面)迁移并编译。
# 用法:compile_legacy.sh <source-root> <output-dir> [python]
set -euo pipefail
umask 027

SOURCE_ROOT="${1:?Usage: $0 <source-root> <output-dir> [python]}"
OUTPUT_DIR="${2:?Usage: $0 <source-root> <output-dir> [python]}"
PYTHON="${3:-python3}"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
VISIBILITY_TOOL="${ELF_VISIBILITY_TOOL:-$SCRIPT_DIR/tools/elf_visibility.py}"
MANIFEST="$SOURCE_ROOT/compile.manifest"
ENTRY_SCRIPT="api/app.py"

SOURCE_ROOT="$(realpath "$SOURCE_ROOT")"
OUTPUT_DIR="$(realpath -m "$OUTPUT_DIR")"

[[ -d "$SOURCE_ROOT" ]] || { echo "legacy source directory missing: $SOURCE_ROOT" >&2; exit 2; }
[[ -f "$MANIFEST" ]] || { echo "compile.manifest missing: $MANIFEST" >&2; exit 2; }
[[ -x "$PYTHON" ]] || { echo "Python is not executable: $PYTHON" >&2; exit 2; }
[[ -f "$VISIBILITY_TOOL" ]] || { echo "ELF visibility tool missing: $VISIBILITY_TOOL" >&2; exit 2; }
for command_name in gcc strip rsync; do command -v "$command_name" >/dev/null; done

case "$OUTPUT_DIR" in
  "/"|"$(realpath -m "$HOME")"|"$SOURCE_ROOT")
    echo "Unsafe output directory: $OUTPUT_DIR" >&2
    exit 2
    ;;
esac
[[ ! -e "$OUTPUT_DIR" ]] || {
  echo "Output already exists; refusing to overwrite: $OUTPUT_DIR" >&2
  exit 2
}

echo "[legacy 1/6] Copying per compile.manifest ($(wc -l < "$MANIFEST") modules; $(wc -l < "$SOURCE_ROOT/compile.exclude-v13-v15" 2>/dev/null || echo 0) excluded v13-v15)"
while IFS= read -r rel; do
  src_file="$SOURCE_ROOT/$rel.py"
  [[ -f "$src_file" ]] || { echo "MISSING: $rel.py" >&2; exit 3; }
  mkdir -p "$OUTPUT_DIR/$(dirname "$rel")"
  install -m 0640 "$src_file" "$OUTPUT_DIR/$rel.py"
done < "$MANIFEST"

# 白名单明文:入口 app.py / main.py(裁剪版) / kg_routes.py / normalized_release
install -m 0750 "$SOURCE_ROOT/$ENTRY_SCRIPT" "$OUTPUT_DIR/$ENTRY_SCRIPT"
[[ -f "$SOURCE_ROOT/api/main.py" ]] && install -m 0750 "$SOURCE_ROOT/api/main.py" "$OUTPUT_DIR/api/main.py"
[[ -f "$SOURCE_ROOT/api/kg_routes.py" ]] && install -m 0750 "$SOURCE_ROOT/api/kg_routes.py" "$OUTPUT_DIR/api/kg_routes.py"
[[ -f "$SOURCE_ROOT/src/rag/normalized_release.py" ]] && install -m 0750 "$SOURCE_ROOT/src/rag/normalized_release.py" "$OUTPUT_DIR/src/rag/normalized_release.py"

# 资源文件(configs/prompts/requirements/scripts/utils)
rsync -a "$SOURCE_ROOT/configs/" "$OUTPUT_DIR/configs/" 2>/dev/null || true
rsync -a "$SOURCE_ROOT/prompts/" "$OUTPUT_DIR/prompts/" 2>/dev/null || true
[[ -f "$SOURCE_ROOT/requirements.txt" ]] && install -m 0640 "$SOURCE_ROOT/requirements.txt" "$OUTPUT_DIR/requirements.txt"
[[ -f "$SOURCE_ROOT/requirements-rag.txt" ]] && install -m 0640 "$SOURCE_ROOT/requirements-rag.txt" "$OUTPUT_DIR/requirements-rag.txt"
mkdir -p "$OUTPUT_DIR/scripts/utils"
rsync -a "$SOURCE_ROOT/scripts/utils/" "$OUTPUT_DIR/scripts/utils/" 2>/dev/null || true

# 包标记(与容器白名单一致的 __init__.py 位置)
for d in api scripts src src/db src/llm src/llm/providers src/rag src/rag/advanced \
         src/rag/advanced/raptor src/rag/cache src/rag/db src/rag/dual_kb_taxonomy_rag \
         src/rag/entity_normalization src/rag/extractors src/rag/extractors/methodology \
         src/rag/extractors/v16_1_1 src/rag/extractors/v16_1_2 \
         src/rag/prompts src/rag/qa_v1_0_0 src/rag/temporal src/rag/unified src/rag/utils \
         src/schemas src/utils tools; do
  mkdir -p "$OUTPUT_DIR/$d"
  [[ -f "$SOURCE_ROOT/$d/__init__.py" ]] && install -m 0640 "$SOURCE_ROOT/$d/__init__.py" "$OUTPUT_DIR/$d/__init__.py" || touch "$OUTPUT_DIR/$d/__init__.py"
done

echo "[legacy 2/6] Discovering Python modules (whitelist: $(grep -c '^src/' "$SOURCE_ROOT/compile.whitelist" 2>/dev/null || echo 0) 明文保留)"
WHITELIST_FILE="$SOURCE_ROOT/compile.whitelist"
WHITELISTED=()
if [[ -f "$WHITELIST_FILE" ]]; then
  while IFS= read -r rel; do
    [[ "$rel" =~ ^src/ ]] && WHITELISTED+=("$rel")
  done < "$WHITELIST_FILE"
fi
_is_whitelisted() {
  local rel="$1"
  for w in "${WHITELISTED[@]}"; do
    [[ "$rel" == "$w" ]] && return 0
  done
  return 1
}
PY_FILES=()
while IFS= read -r -d '' path; do PY_FILES+=("$path"); done \
  < <(find "$OUTPUT_DIR" -type f -name '*.py' \
      -not -path "*/api/app.py" \
      -not -path "*/api/main.py" \
      -not -path "*/api/kg_routes.py" \
      -not -path "*/src/rag/normalized_release.py" \
      -not -name '__init__.py' -print0)
FILTERED=()
for py_file in "${PY_FILES[@]}"; do
  rel="${py_file#"$OUTPUT_DIR"/}"
  if _is_whitelisted "$rel"; then
    echo "  [whitelist] $rel (明文保留,理由见 compile.whitelist)"
    continue
  fi
  FILTERED+=("$py_file")
done
PY_FILES=("${FILTERED[@]}")
TOTAL="${#PY_FILES[@]}"
(( TOTAL > 0 )) || { echo "No legacy modules selected for compilation" >&2; exit 3; }

INCLUDE_PY="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("INCLUDEPY") or "")')"
EXT_SUFFIX="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX") or ".so")')"
[[ -n "$INCLUDE_PY" ]] || { echo "Python INCLUDEPY is empty" >&2; exit 3; }

echo "[legacy 3/6] Cython preflight ($TOTAL modules)"
COUNT=0
CYTHON_FAILURES=0
FAILED_MODULES=()
for py_file in "${PY_FILES[@]}"; do
  COUNT=$((COUNT + 1))
  directory="$(dirname "$py_file")"
  module="$(basename "$py_file" .py)"
  c_file="$directory/$module.c"
  echo "  [$COUNT/$TOTAL] ${py_file#"$OUTPUT_DIR"/}"
  if ! "$PYTHON" -m cython -3 \
      --directive always_allow_keywords=true,annotation_typing=False \
      --no-docstrings "$py_file" -o "$c_file" 2>/dev/null; then
    CYTHON_FAILURES=$((CYTHON_FAILURES + 1))
    FAILED_MODULES+=("${py_file#"$OUTPUT_DIR"/}")
  fi
done
if (( CYTHON_FAILURES > 0 )); then
  echo "legacy Cython preflight failed: modules=$CYTHON_FAILURES" >&2
  printf '  %s\n' "${FAILED_MODULES[@]}" >&2
  echo "以上模块需加入 compile.manifest 白名单(记录理由),绝不允许静默明文" >&2
  exit 3
fi

echo "[legacy 4/6] GCC hidden visibility + strip ($TOTAL modules)"
COUNT=0
for py_file in "${PY_FILES[@]}"; do
  COUNT=$((COUNT + 1))
  directory="$(dirname "$py_file")"
  module="$(basename "$py_file" .py)"
  c_file="$directory/$module.c"
  so_file="$directory/$module$EXT_SUFFIX"
  echo "  [$COUNT/$TOTAL] ${py_file#"$OUTPUT_DIR"/}"
  gcc -shared -pthread -fPIC -O2 -fvisibility=hidden \
    -I"$INCLUDE_PY" -o "$so_file" "$c_file"
  strip --strip-unneeded "$so_file"
  "$PYTHON" -B "$VISIBILITY_TOOL" \
    --hide-non-pyinit --check "$so_file" >/dev/null
  rm -f -- "$c_file" "$py_file"
done

echo "[legacy 5/6] Restoring package markers"
while IFS= read -r -d '' directory; do
  if find "$directory" -maxdepth 1 -type f \( -name '*.so' -o -name '*.py' \) -print -quit | grep -q .; then
    [[ -f "$directory/__init__.py" ]] || touch "$directory/__init__.py"
  fi
done < <(find "$OUTPUT_DIR" -type d -print0)

echo "[legacy 6/6] Final checks"
SO_COUNT="$(find "$OUTPUT_DIR" -type f -name '*.so' | wc -l)"
[[ "$SO_COUNT" -eq "$TOTAL" ]] || {
  echo "legacy count mismatch: selected=$TOTAL produced=$SO_COUNT" >&2
  exit 4
}
if find "$OUTPUT_DIR" -type f -name '*.c' -print -quit | grep -q .; then
  echo "Generated C source remains in output" >&2
  exit 4
fi

echo "legacy compile+harden complete: modules=$SO_COUNT output=$OUTPUT_DIR"