124 lines
4.7 KiB
Bash
Executable File
124 lines
4.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Compile the Methodology extraction service into stripped Cython extensions.
|
|
# 用法:compile_methodology.sh <source-root> <output-dir> [python]
|
|
set -euo pipefail
|
|
umask 027
|
|
|
|
SOURCE_ROOT="${1:?Usage: $0 <source-root> <output-dir> [python]}"
|
|
OUTPUT_DIR="${2:?Usage: $0 <source-root> <output-dir> [python]}"
|
|
PYTHON="${3:-python3}"
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
VISIBILITY_TOOL="${ELF_VISIBILITY_TOOL:-$SCRIPT_DIR/tools/elf_visibility.py}"
|
|
ENTRY_SCRIPT="scripts/methodology_server.py"
|
|
|
|
SOURCE_ROOT="$(realpath "$SOURCE_ROOT")"
|
|
OUTPUT_DIR="$(realpath -m "$OUTPUT_DIR")"
|
|
|
|
[[ -d "$SOURCE_ROOT" ]] || { echo "Methodology source directory missing: $SOURCE_ROOT" >&2; exit 2; }
|
|
[[ -x "$PYTHON" ]] || { echo "Python is not executable: $PYTHON" >&2; exit 2; }
|
|
[[ -f "$VISIBILITY_TOOL" ]] || { echo "ELF visibility tool missing: $VISIBILITY_TOOL" >&2; exit 2; }
|
|
for command_name in gcc strip rsync; do command -v "$command_name" >/dev/null; done
|
|
|
|
case "$OUTPUT_DIR" in
|
|
"/"|"$(realpath -m "$HOME")"|"$SOURCE_ROOT")
|
|
echo "Unsafe output directory: $OUTPUT_DIR" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
[[ ! -e "$OUTPUT_DIR" ]] || {
|
|
echo "Output already exists; refusing to overwrite: $OUTPUT_DIR" >&2
|
|
exit 2
|
|
}
|
|
|
|
mkdir -p \
|
|
"$OUTPUT_DIR/scripts" \
|
|
"$OUTPUT_DIR/api" \
|
|
"$OUTPUT_DIR/src/rag/unified" \
|
|
"$OUTPUT_DIR/src/rag/extractors/methodology"
|
|
|
|
echo "[Methodology 1/6] Copying the audited runtime surface"
|
|
install -m 0750 "$SOURCE_ROOT/$ENTRY_SCRIPT" "$OUTPUT_DIR/$ENTRY_SCRIPT"
|
|
install -m 0640 "$SOURCE_ROOT/api/methodology_routes.py" "$OUTPUT_DIR/api/methodology_routes.py"
|
|
install -m 0640 "$SOURCE_ROOT/src/rag/project_config.py" "$OUTPUT_DIR/src/rag/project_config.py"
|
|
rsync -a --include='*.py' --exclude='*' \
|
|
"$SOURCE_ROOT/src/rag/unified/" "$OUTPUT_DIR/src/rag/unified/"
|
|
rsync -a --include='*.py' --exclude='*' \
|
|
"$SOURCE_ROOT/src/rag/extractors/methodology/" "$OUTPUT_DIR/src/rag/extractors/methodology/"
|
|
|
|
touch \
|
|
"$OUTPUT_DIR/scripts/__init__.py" \
|
|
"$OUTPUT_DIR/api/__init__.py" \
|
|
"$OUTPUT_DIR/src/__init__.py" \
|
|
"$OUTPUT_DIR/src/rag/__init__.py" \
|
|
"$OUTPUT_DIR/src/rag/extractors/__init__.py"
|
|
|
|
echo "[Methodology 2/6] Discovering Python modules"
|
|
PY_FILES=()
|
|
while IFS= read -r -d '' path; do PY_FILES+=("$path"); done \
|
|
< <(find "$OUTPUT_DIR" -type f -name '*.py' \
|
|
-not -path "*/$ENTRY_SCRIPT" \
|
|
-not -name '__init__.py' -print0)
|
|
TOTAL="${#PY_FILES[@]}"
|
|
(( TOTAL > 0 )) || { echo "No Methodology modules selected for compilation" >&2; exit 3; }
|
|
|
|
INCLUDE_PY="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("INCLUDEPY") or "")')"
|
|
EXT_SUFFIX="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX") or ".so")')"
|
|
[[ -n "$INCLUDE_PY" ]] || { echo "Python INCLUDEPY is empty" >&2; exit 3; }
|
|
|
|
echo "[Methodology 3/6] Cython preflight ($TOTAL modules)"
|
|
COUNT=0
|
|
CYTHON_FAILURES=0
|
|
for py_file in "${PY_FILES[@]}"; do
|
|
COUNT=$((COUNT + 1))
|
|
directory="$(dirname "$py_file")"
|
|
module="$(basename "$py_file" .py)"
|
|
c_file="$directory/$module.c"
|
|
echo " [$COUNT/$TOTAL] ${py_file#"$OUTPUT_DIR"/}"
|
|
if ! "$PYTHON" -m cython -3 \
|
|
--directive always_allow_keywords=true,annotation_typing=False \
|
|
--no-docstrings "$py_file" -o "$c_file"; then
|
|
CYTHON_FAILURES=$((CYTHON_FAILURES + 1))
|
|
fi
|
|
done
|
|
(( CYTHON_FAILURES == 0 )) || {
|
|
echo "Methodology Cython preflight failed: modules=$CYTHON_FAILURES" >&2
|
|
exit 3
|
|
}
|
|
|
|
echo "[Methodology 4/6] GCC hidden visibility + strip ($TOTAL modules)"
|
|
COUNT=0
|
|
for py_file in "${PY_FILES[@]}"; do
|
|
COUNT=$((COUNT + 1))
|
|
directory="$(dirname "$py_file")"
|
|
module="$(basename "$py_file" .py)"
|
|
c_file="$directory/$module.c"
|
|
so_file="$directory/$module$EXT_SUFFIX"
|
|
echo " [$COUNT/$TOTAL] ${py_file#"$OUTPUT_DIR"/}"
|
|
gcc -shared -pthread -fPIC -O2 -fvisibility=hidden \
|
|
-I"$INCLUDE_PY" -o "$so_file" "$c_file"
|
|
strip --strip-unneeded "$so_file"
|
|
"$PYTHON" -B "$VISIBILITY_TOOL" \
|
|
--hide-non-pyinit --check "$so_file" >/dev/null
|
|
rm -f -- "$c_file" "$py_file"
|
|
done
|
|
|
|
echo "[Methodology 5/6] Restoring package markers"
|
|
while IFS= read -r -d '' directory; do
|
|
if find "$directory" -maxdepth 1 -type f \( -name '*.so' -o -name '*.py' \) -print -quit | grep -q .; then
|
|
[[ -f "$directory/__init__.py" ]] || touch "$directory/__init__.py"
|
|
fi
|
|
done < <(find "$OUTPUT_DIR" -type d -print0)
|
|
|
|
echo "[Methodology 6/6] Final checks"
|
|
SO_COUNT="$(find "$OUTPUT_DIR" -type f -name '*.so' | wc -l)"
|
|
[[ "$SO_COUNT" -eq "$TOTAL" ]] || {
|
|
echo "Methodology count mismatch: selected=$TOTAL produced=$SO_COUNT" >&2
|
|
exit 4
|
|
}
|
|
if find "$OUTPUT_DIR" -type f -name '*.c' -print -quit | grep -q .; then
|
|
echo "Generated C source remains in output" >&2
|
|
exit 4
|
|
fi
|
|
|
|
echo "Methodology compile+harden complete: modules=$SO_COUNT output=$OUTPUT_DIR"
|