P1/M2: tooling/compile/compile_qa.sh(参数化 + normalized_release 补丁)
- 收编工具链遗漏主项目 08-29 新增 src/rag/normalized_release.py(backend_guard 依赖) - 编译树 45 模块全部 .so,audit 0 错误,7922 健康检查金标过 - 修复:compile 目录相对依赖 tools/(elf_visibility.py) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jerry
Executable
+139
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env bash
|
||||
# Compile the QA V1.1.73 runtime surface into stripped Cython extensions.
|
||||
set -euo pipefail
|
||||
umask 027
|
||||
|
||||
SOURCE_ROOT="${1:?Usage: $0 <source-root> <output-dir> [python]}"
|
||||
OUTPUT_DIR="${2:?Usage: $0 <source-root> <output-dir> [python]}"
|
||||
PYTHON="${3:-python3}"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
VISIBILITY_TOOL="${ELF_VISIBILITY_TOOL:-$SCRIPT_DIR/tools/elf_visibility.py}"
|
||||
ENTRY_SCRIPT="scripts/qa_v1_1_73_server.py"
|
||||
|
||||
SOURCE_ROOT="$(realpath "$SOURCE_ROOT")"
|
||||
OUTPUT_DIR="$(realpath -m "$OUTPUT_DIR")"
|
||||
|
||||
[[ -d "$SOURCE_ROOT" ]] || { echo "QA source directory missing: $SOURCE_ROOT" >&2; exit 2; }
|
||||
[[ -x "$PYTHON" ]] || { echo "Python is not executable: $PYTHON" >&2; exit 2; }
|
||||
[[ -f "$VISIBILITY_TOOL" ]] || { echo "ELF visibility tool missing: $VISIBILITY_TOOL" >&2; exit 2; }
|
||||
for command_name in gcc strip rsync; do command -v "$command_name" >/dev/null; done
|
||||
|
||||
case "$OUTPUT_DIR" in
|
||||
"/"|"$(realpath -m "$HOME")"|"$SOURCE_ROOT")
|
||||
echo "Unsafe output directory: $OUTPUT_DIR" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
[[ ! -e "$OUTPUT_DIR" ]] || {
|
||||
echo "Output already exists; refusing to overwrite: $OUTPUT_DIR" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
mkdir -p \
|
||||
"$OUTPUT_DIR/scripts" \
|
||||
"$OUTPUT_DIR/src/rag/qa_v1_1_73" \
|
||||
"$OUTPUT_DIR/src/rag/retrieval" \
|
||||
"$OUTPUT_DIR/src/rag/dual_kb_taxonomy_rag" \
|
||||
"$OUTPUT_DIR/configs/rag"
|
||||
|
||||
echo "[QA 1/6] Copying the audited runtime surface"
|
||||
install -m 0750 "$SOURCE_ROOT/$ENTRY_SCRIPT" "$OUTPUT_DIR/$ENTRY_SCRIPT"
|
||||
rsync -a --include='*.py' --exclude='*' \
|
||||
"$SOURCE_ROOT/src/rag/qa_v1_1_73/" "$OUTPUT_DIR/src/rag/qa_v1_1_73/"
|
||||
rsync -a --include='*.py' --exclude='*' \
|
||||
"$SOURCE_ROOT/src/rag/retrieval/" "$OUTPUT_DIR/src/rag/retrieval/"
|
||||
install -m 0640 \
|
||||
"$SOURCE_ROOT/src/rag/dual_kb_taxonomy_rag/entity_alias.py" \
|
||||
"$OUTPUT_DIR/src/rag/dual_kb_taxonomy_rag/entity_alias.py"
|
||||
# 2026-08-31 主项目新增:normalized_release.py(backend_guard 校验 + 图适配器)
|
||||
[[ -f "$SOURCE_ROOT/src/rag/normalized_release.py" ]] && install -m 0640 \
|
||||
"$SOURCE_ROOT/src/rag/normalized_release.py" \
|
||||
"$OUTPUT_DIR/src/rag/normalized_release.py"
|
||||
for relative in \
|
||||
configs/rag/entity_alias.yaml \
|
||||
configs/rag/relation_taxonomy.yaml \
|
||||
pyproject.toml \
|
||||
requirements.txt \
|
||||
requirements-rag.txt
|
||||
do
|
||||
if [[ -f "$SOURCE_ROOT/$relative" ]]; then
|
||||
install -D -m 0640 "$SOURCE_ROOT/$relative" "$OUTPUT_DIR/$relative"
|
||||
fi
|
||||
done
|
||||
|
||||
# Namespace/package markers are intentionally left as minimal Python files.
|
||||
touch \
|
||||
"$OUTPUT_DIR/scripts/__init__.py" \
|
||||
"$OUTPUT_DIR/src/__init__.py" \
|
||||
"$OUTPUT_DIR/src/rag/__init__.py" \
|
||||
"$OUTPUT_DIR/src/rag/dual_kb_taxonomy_rag/__init__.py"
|
||||
|
||||
echo "[QA 2/6] Discovering Python modules"
|
||||
PY_FILES=()
|
||||
while IFS= read -r -d '' path; do PY_FILES+=("$path"); done \
|
||||
< <(find "$OUTPUT_DIR" -type f -name '*.py' \
|
||||
-not -path "*/$ENTRY_SCRIPT" \
|
||||
-not -name '__init__.py' -print0)
|
||||
TOTAL="${#PY_FILES[@]}"
|
||||
(( TOTAL > 0 )) || { echo "No QA modules selected for compilation" >&2; exit 3; }
|
||||
|
||||
INCLUDE_PY="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("INCLUDEPY") or "")')"
|
||||
EXT_SUFFIX="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX") or ".so")')"
|
||||
[[ -n "$INCLUDE_PY" ]] || { echo "Python INCLUDEPY is empty" >&2; exit 3; }
|
||||
|
||||
echo "[QA 3/6] Cython preflight ($TOTAL modules)"
|
||||
COUNT=0
|
||||
CYTHON_FAILURES=0
|
||||
for py_file in "${PY_FILES[@]}"; do
|
||||
COUNT=$((COUNT + 1))
|
||||
directory="$(dirname "$py_file")"
|
||||
module="$(basename "$py_file" .py)"
|
||||
c_file="$directory/$module.c"
|
||||
echo " [$COUNT/$TOTAL] ${py_file#"$OUTPUT_DIR"/}"
|
||||
if ! "$PYTHON" -m cython -3 \
|
||||
--directive always_allow_keywords=true,annotation_typing=False \
|
||||
--no-docstrings "$py_file" -o "$c_file"; then
|
||||
CYTHON_FAILURES=$((CYTHON_FAILURES + 1))
|
||||
fi
|
||||
done
|
||||
(( CYTHON_FAILURES == 0 )) || {
|
||||
echo "QA Cython preflight failed: modules=$CYTHON_FAILURES" >&2
|
||||
exit 3
|
||||
}
|
||||
|
||||
echo "[QA 4/6] GCC hidden visibility + strip ($TOTAL modules)"
|
||||
COUNT=0
|
||||
for py_file in "${PY_FILES[@]}"; do
|
||||
COUNT=$((COUNT + 1))
|
||||
directory="$(dirname "$py_file")"
|
||||
module="$(basename "$py_file" .py)"
|
||||
c_file="$directory/$module.c"
|
||||
so_file="$directory/$module$EXT_SUFFIX"
|
||||
echo " [$COUNT/$TOTAL] ${py_file#"$OUTPUT_DIR"/}"
|
||||
gcc -shared -pthread -fPIC -O2 -fvisibility=hidden \
|
||||
-I"$INCLUDE_PY" -o "$so_file" "$c_file"
|
||||
strip --strip-unneeded "$so_file"
|
||||
"$PYTHON" -B "$VISIBILITY_TOOL" \
|
||||
--hide-non-pyinit --check "$so_file" >/dev/null
|
||||
rm -f -- "$c_file" "$py_file"
|
||||
done
|
||||
|
||||
echo "[QA 5/6] Restoring package markers"
|
||||
while IFS= read -r -d '' directory; do
|
||||
if find "$directory" -maxdepth 1 -type f \( -name '*.so' -o -name '*.py' \) -print -quit | grep -q .; then
|
||||
[[ -f "$directory/__init__.py" ]] || touch "$directory/__init__.py"
|
||||
fi
|
||||
done < <(find "$OUTPUT_DIR" -type d -print0)
|
||||
|
||||
echo "[QA 6/6] Final checks"
|
||||
SO_COUNT="$(find "$OUTPUT_DIR" -type f -name '*.so' | wc -l)"
|
||||
[[ "$SO_COUNT" -eq "$TOTAL" ]] || {
|
||||
echo "QA count mismatch: selected=$TOTAL produced=$SO_COUNT" >&2
|
||||
exit 4
|
||||
}
|
||||
if find "$OUTPUT_DIR" -type f -name '*.c' -print -quit | grep -q .; then
|
||||
echo "Generated C source remains in output" >&2
|
||||
exit 4
|
||||
fi
|
||||
|
||||
echo "QA compile+harden complete: modules=$SO_COUNT output=$OUTPUT_DIR"
|
||||
Executable
+242
@@ -0,0 +1,242 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Audit a compiled tree and bind it to a tamper-evident build marker."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import sysconfig
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from elf_visibility import summarize # noqa: E402
|
||||
|
||||
|
||||
MARKER_NAME = ".hardened-build.json"
|
||||
TREE_DIGEST_POLICY = "immutable-runtime-v2"
|
||||
MUTABLE_TOP_LEVEL_DIRS = {
|
||||
"cost_logs",
|
||||
"data",
|
||||
"input",
|
||||
"logs",
|
||||
"memory_storage",
|
||||
"neo4j",
|
||||
"output",
|
||||
"outputs",
|
||||
"rag_kb_store",
|
||||
"service_data",
|
||||
"temp",
|
||||
"v15_output",
|
||||
"workspace",
|
||||
}
|
||||
|
||||
|
||||
def file_sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def is_digest_excluded(relative: str) -> bool:
|
||||
parts = Path(relative).parts
|
||||
return (
|
||||
relative == MARKER_NAME
|
||||
or not parts
|
||||
or parts[0] in MUTABLE_TOP_LEVEL_DIRS
|
||||
or "__pycache__" in parts
|
||||
or relative.endswith((".pyc", ".pyo"))
|
||||
)
|
||||
|
||||
|
||||
def tree_digest(root: Path) -> tuple[str, int]:
|
||||
digest = hashlib.sha256()
|
||||
count = 0
|
||||
for path in sorted(
|
||||
(item for item in root.rglob("*") if item.is_file()),
|
||||
key=lambda item: item.relative_to(root).as_posix(),
|
||||
):
|
||||
relative = path.relative_to(root).as_posix()
|
||||
if is_digest_excluded(relative):
|
||||
continue
|
||||
count += 1
|
||||
digest.update(relative.encode("utf-8"))
|
||||
digest.update(b"\0")
|
||||
digest.update(bytes.fromhex(file_sha256(path)))
|
||||
digest.update(b"\0")
|
||||
return digest.hexdigest(), count
|
||||
|
||||
|
||||
def audit_tree(root: Path, project: str) -> dict[str, Any]:
|
||||
errors: list[dict[str, Any]] = []
|
||||
extensions = sorted(root.rglob("*.so"))
|
||||
expected_suffix = sysconfig.get_config_var("EXT_SUFFIX") or ".so"
|
||||
hidden_non_pyinit = 0
|
||||
|
||||
if not extensions:
|
||||
errors.append({"kind": "no_extensions"})
|
||||
|
||||
for extension in extensions:
|
||||
relative = extension.relative_to(root).as_posix()
|
||||
if not extension.name.endswith(expected_suffix):
|
||||
errors.append(
|
||||
{
|
||||
"kind": "python_abi_suffix_mismatch",
|
||||
"file": relative,
|
||||
"expected_suffix": expected_suffix,
|
||||
}
|
||||
)
|
||||
try:
|
||||
info = summarize(extension, extension.read_bytes())
|
||||
except BaseException as exc:
|
||||
errors.append(
|
||||
{
|
||||
"kind": "elf_parse_failed",
|
||||
"file": relative,
|
||||
"error": f"{type(exc).__name__}: {exc}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
if info["has_symtab"]:
|
||||
errors.append({"kind": "symtab_present", "file": relative})
|
||||
if len(info["pyinit"]) != 1 or not info["all_pyinit_default"]:
|
||||
errors.append(
|
||||
{
|
||||
"kind": "invalid_pyinit",
|
||||
"file": relative,
|
||||
"pyinit": info["pyinit"],
|
||||
}
|
||||
)
|
||||
if not info["all_non_pyinit_hidden"]:
|
||||
errors.append(
|
||||
{
|
||||
"kind": "visible_non_pyinit",
|
||||
"file": relative,
|
||||
"symbols": info["non_pyinit"],
|
||||
}
|
||||
)
|
||||
hidden_non_pyinit += len(info["non_pyinit"])
|
||||
|
||||
c_files = [
|
||||
path.relative_to(root).as_posix()
|
||||
for path in root.rglob("*.c")
|
||||
if path.is_file()
|
||||
]
|
||||
if c_files:
|
||||
errors.append({"kind": "generated_c_retained", "files": c_files})
|
||||
|
||||
sensitive = []
|
||||
for path in root.rglob("*"):
|
||||
if not path.is_file():
|
||||
continue
|
||||
lowered = path.name.lower()
|
||||
if (
|
||||
lowered.endswith(".key")
|
||||
or "private_key" in lowered
|
||||
or "master_key" in lowered
|
||||
or lowered == "private_key.pass"
|
||||
):
|
||||
sensitive.append(path.relative_to(root).as_posix())
|
||||
if sensitive:
|
||||
errors.append({"kind": "sensitive_filename", "files": sensitive})
|
||||
|
||||
digest, file_count = tree_digest(root)
|
||||
return {
|
||||
"project": project,
|
||||
"tree_digest_policy": TREE_DIGEST_POLICY,
|
||||
"python_version": sys.version.split()[0],
|
||||
"extension_suffix": expected_suffix,
|
||||
"extension_count": len(extensions),
|
||||
"hidden_non_pyinit_count": hidden_non_pyinit,
|
||||
"file_count_without_marker": file_count,
|
||||
"tree_sha256": digest,
|
||||
"error_count": len(errors),
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
|
||||
def atomic_write_json(path: Path, payload: dict[str, Any]) -> None:
|
||||
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
||||
temporary.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
os.chmod(temporary, 0o640)
|
||||
os.replace(temporary, path)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--tree", type=Path, required=True)
|
||||
parser.add_argument("--project", required=True)
|
||||
mode = parser.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument("--write-marker", action="store_true")
|
||||
mode.add_argument("--require-marker", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
root = args.tree.resolve()
|
||||
if not root.is_dir():
|
||||
raise SystemExit(f"Compiled tree missing: {root}")
|
||||
|
||||
report = audit_tree(root, args.project)
|
||||
marker_path = root / MARKER_NAME
|
||||
|
||||
if args.require_marker:
|
||||
if not marker_path.is_file():
|
||||
report["errors"].append({"kind": "marker_missing"})
|
||||
else:
|
||||
marker = json.loads(marker_path.read_text(encoding="utf-8"))
|
||||
report["marker"] = marker
|
||||
for field in (
|
||||
"project",
|
||||
"tree_digest_policy",
|
||||
"python_version",
|
||||
"extension_suffix",
|
||||
"extension_count",
|
||||
"tree_sha256",
|
||||
):
|
||||
if marker.get(field) != report.get(field):
|
||||
report["errors"].append(
|
||||
{
|
||||
"kind": "marker_mismatch",
|
||||
"field": field,
|
||||
"expected": marker.get(field),
|
||||
"actual": report.get(field),
|
||||
}
|
||||
)
|
||||
|
||||
report["error_count"] = len(report["errors"])
|
||||
report["ok"] = report["error_count"] == 0
|
||||
|
||||
if args.write_marker and report["ok"]:
|
||||
marker = {
|
||||
key: report[key]
|
||||
for key in (
|
||||
"project",
|
||||
"tree_digest_policy",
|
||||
"python_version",
|
||||
"extension_suffix",
|
||||
"extension_count",
|
||||
"hidden_non_pyinit_count",
|
||||
"file_count_without_marker",
|
||||
"tree_sha256",
|
||||
)
|
||||
}
|
||||
marker["created_at"] = datetime.now().astimezone().isoformat(
|
||||
timespec="seconds"
|
||||
)
|
||||
atomic_write_json(marker_path, marker)
|
||||
report["marker_path"] = str(marker_path)
|
||||
|
||||
print(json.dumps(report, ensure_ascii=False, sort_keys=True))
|
||||
return 0 if report["ok"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Write the customer-facing manifest for the three-project offline delivery."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def tree_bytes(path: Path) -> int:
|
||||
total = 0
|
||||
for directory, _, files in os.walk(path, followlinks=True):
|
||||
for filename in files:
|
||||
total += (Path(directory) / filename).stat().st_size
|
||||
return total
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--runtime-root", required=True, type=Path)
|
||||
parser.add_argument("--output", required=True, type=Path)
|
||||
parser.add_argument("--version", required=True)
|
||||
parser.add_argument("--generation-model", required=True)
|
||||
parser.add_argument("--embed-model", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
root = args.runtime_root.resolve()
|
||||
assets = {
|
||||
"qdrant_storage": root / "qdrant" / "storage",
|
||||
"neo4j_data": root / "neo4j" / "data",
|
||||
"bge_reranker": root / "QA" / "models" / "bge-reranker-base",
|
||||
"ollama_models": root / "ollama-models",
|
||||
}
|
||||
missing = [str(path) for path in assets.values() if not path.is_dir()]
|
||||
if missing:
|
||||
raise SystemExit("Missing runtime asset directories: " + ", ".join(missing))
|
||||
|
||||
payload = {
|
||||
"release_version": args.version,
|
||||
"runtime_data_archive": f"plantai-rag-runtime-data-{args.version}.tar",
|
||||
"applications": {
|
||||
"PlantAI": {"path": "app/PlantAI_compiled", "port": 7880},
|
||||
"RAG": {"path": "app/RAG", "port": 7900},
|
||||
"QA": {"path": "app/QA", "port": 7925},
|
||||
},
|
||||
"containers": {
|
||||
"qdrant": {
|
||||
"image": "qdrant/qdrant:latest",
|
||||
"http_port": 16333,
|
||||
"grpc_port": 16334,
|
||||
"data_path": "runtime-assets/qdrant/storage",
|
||||
},
|
||||
"neo4j": {
|
||||
"image": "neo4j:5.15.0-community",
|
||||
"http_port": 17474,
|
||||
"bolt_port": 17687,
|
||||
"data_path": "runtime-assets/neo4j/data",
|
||||
},
|
||||
},
|
||||
"models": {
|
||||
"ollama_generation": args.generation_model,
|
||||
"ollama_embedding": args.embed_model,
|
||||
"ollama_path": "runtime-assets/ollama-models",
|
||||
"reranker_path": "runtime-assets/QA/models/bge-reranker-base",
|
||||
},
|
||||
"asset_bytes": {name: tree_bytes(path) for name, path in assets.items()},
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(json.dumps(payload["asset_bytes"], ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+205
@@ -0,0 +1,205 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Inspect or hide non-PyInit defined symbols in an ELF64 little-endian .dynsym."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
STN_UNDEF = 0
|
||||
STV_DEFAULT = 0
|
||||
STV_HIDDEN = 2
|
||||
|
||||
|
||||
def c_string(data: bytes | bytearray, offset: int) -> str:
|
||||
end = data.find(b"\0", offset)
|
||||
if end < 0:
|
||||
end = len(data)
|
||||
return bytes(data[offset:end]).decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def parse_sections(data: bytes | bytearray) -> tuple[list[dict[str, Any]], bytes]:
|
||||
if data[:4] != b"\x7fELF":
|
||||
raise ValueError("not an ELF file")
|
||||
if data[4] != 2:
|
||||
raise ValueError("only ELF64 is supported")
|
||||
if data[5] != 1:
|
||||
raise ValueError("only little-endian ELF is supported")
|
||||
|
||||
section_offset = struct.unpack_from("<Q", data, 40)[0]
|
||||
section_entry_size = struct.unpack_from("<H", data, 58)[0]
|
||||
section_count = struct.unpack_from("<H", data, 60)[0]
|
||||
section_name_index = struct.unpack_from("<H", data, 62)[0]
|
||||
if section_entry_size < 64 or not section_count:
|
||||
raise ValueError("invalid or missing section table")
|
||||
|
||||
sections = []
|
||||
for index in range(section_count):
|
||||
offset = section_offset + index * section_entry_size
|
||||
fields = struct.unpack_from("<IIQQQQIIQQ", data, offset)
|
||||
sections.append(
|
||||
{
|
||||
"index": index,
|
||||
"header_offset": offset,
|
||||
"name_offset": fields[0],
|
||||
"type": fields[1],
|
||||
"flags": fields[2],
|
||||
"address": fields[3],
|
||||
"offset": fields[4],
|
||||
"size": fields[5],
|
||||
"link": fields[6],
|
||||
"info": fields[7],
|
||||
"alignment": fields[8],
|
||||
"entry_size": fields[9],
|
||||
}
|
||||
)
|
||||
|
||||
if section_name_index >= len(sections):
|
||||
raise ValueError("invalid section-name string-table index")
|
||||
names_section = sections[section_name_index]
|
||||
names = bytes(
|
||||
data[
|
||||
names_section["offset"] : names_section["offset"] + names_section["size"]
|
||||
]
|
||||
)
|
||||
for section in sections:
|
||||
section["name"] = c_string(names, section["name_offset"])
|
||||
return sections, names
|
||||
|
||||
|
||||
def inspect_dynsym(data: bytes | bytearray) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
||||
sections, _ = parse_sections(data)
|
||||
dynsym = next((section for section in sections if section["name"] == ".dynsym"), None)
|
||||
if dynsym is None:
|
||||
raise ValueError("missing .dynsym")
|
||||
if dynsym["link"] >= len(sections):
|
||||
raise ValueError("invalid .dynsym string-table link")
|
||||
dynstr_section = sections[dynsym["link"]]
|
||||
dynstr = data[
|
||||
dynstr_section["offset"] : dynstr_section["offset"] + dynstr_section["size"]
|
||||
]
|
||||
entry_size = dynsym["entry_size"] or 24
|
||||
if entry_size < 24:
|
||||
raise ValueError("invalid ELF64 symbol entry size")
|
||||
|
||||
symbols = []
|
||||
count = dynsym["size"] // entry_size
|
||||
for index in range(count):
|
||||
offset = dynsym["offset"] + index * entry_size
|
||||
name_offset = struct.unpack_from("<I", data, offset)[0]
|
||||
info = data[offset + 4]
|
||||
other = data[offset + 5]
|
||||
section_index = struct.unpack_from("<H", data, offset + 6)[0]
|
||||
value, size = struct.unpack_from("<QQ", data, offset + 8)
|
||||
name = c_string(dynstr, name_offset) if name_offset < len(dynstr) else ""
|
||||
symbols.append(
|
||||
{
|
||||
"index": index,
|
||||
"entry_offset": offset,
|
||||
"name": name,
|
||||
"binding": info >> 4,
|
||||
"type": info & 0xF,
|
||||
"visibility": other & 0x3,
|
||||
"section_index": section_index,
|
||||
"value": value,
|
||||
"size": size,
|
||||
"defined": section_index != STN_UNDEF,
|
||||
}
|
||||
)
|
||||
return symbols, dynsym
|
||||
|
||||
|
||||
def summarize(path: Path, data: bytes | bytearray) -> dict[str, Any]:
|
||||
sections, _ = parse_sections(data)
|
||||
symbols, _ = inspect_dynsym(data)
|
||||
defined = [symbol for symbol in symbols if symbol["defined"] and symbol["name"]]
|
||||
pyinit = [symbol for symbol in defined if symbol["name"].startswith("PyInit_")]
|
||||
non_pyinit = [
|
||||
symbol for symbol in defined if not symbol["name"].startswith("PyInit_")
|
||||
]
|
||||
return {
|
||||
"path": str(path),
|
||||
"has_symtab": any(section["name"] == ".symtab" for section in sections),
|
||||
"defined_dynamic_count": len(defined),
|
||||
"pyinit": [
|
||||
{"name": symbol["name"], "visibility": symbol["visibility"]}
|
||||
for symbol in pyinit
|
||||
],
|
||||
"non_pyinit": [
|
||||
{"name": symbol["name"], "visibility": symbol["visibility"]}
|
||||
for symbol in non_pyinit
|
||||
],
|
||||
"all_non_pyinit_hidden": all(
|
||||
symbol["visibility"] == STV_HIDDEN for symbol in non_pyinit
|
||||
),
|
||||
"all_pyinit_default": bool(pyinit)
|
||||
and all(symbol["visibility"] == STV_DEFAULT for symbol in pyinit),
|
||||
}
|
||||
|
||||
|
||||
def hide_non_pyinit(path: Path) -> dict[str, Any]:
|
||||
original = path.read_bytes()
|
||||
data = bytearray(original)
|
||||
symbols, _ = inspect_dynsym(data)
|
||||
changed = []
|
||||
for symbol in symbols:
|
||||
if (
|
||||
symbol["defined"]
|
||||
and symbol["name"]
|
||||
and not symbol["name"].startswith("PyInit_")
|
||||
and symbol["visibility"] != STV_HIDDEN
|
||||
):
|
||||
offset = symbol["entry_offset"] + 5
|
||||
data[offset] = (data[offset] & ~0x3) | STV_HIDDEN
|
||||
changed.append(symbol["name"])
|
||||
|
||||
if changed:
|
||||
stat = path.stat()
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="wb", dir=path.parent, prefix=f".{path.name}.", delete=False
|
||||
) as handle:
|
||||
temporary = Path(handle.name)
|
||||
handle.write(data)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(temporary, stat.st_mode)
|
||||
os.replace(temporary, path)
|
||||
|
||||
result = summarize(path, data)
|
||||
result["changed"] = changed
|
||||
return result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("elf", type=Path)
|
||||
parser.add_argument("--hide-non-pyinit", action="store_true")
|
||||
parser.add_argument("--check", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
path = args.elf.resolve()
|
||||
result = (
|
||||
hide_non_pyinit(path)
|
||||
if args.hide_non_pyinit
|
||||
else summarize(path, path.read_bytes())
|
||||
)
|
||||
print(json.dumps(result, ensure_ascii=False, sort_keys=True))
|
||||
if args.check:
|
||||
valid = (
|
||||
not result["has_symtab"]
|
||||
and result["all_pyinit_default"]
|
||||
and result["all_non_pyinit_hidden"]
|
||||
and len(result["pyinit"]) == 1
|
||||
)
|
||||
return 0 if valid else 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Copy selected Ollama manifests and their referenced blobs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def model_manifest(source: Path, model: str) -> Path:
|
||||
repository, sep, tag = model.partition(":")
|
||||
if not sep:
|
||||
tag = "latest"
|
||||
return source / "manifests" / "registry.ollama.ai" / "library" / repository / tag
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--source", required=True, type=Path)
|
||||
parser.add_argument("--output", required=True, type=Path)
|
||||
parser.add_argument("--model", action="append", required=True)
|
||||
parser.add_argument("--check-only", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
source = args.source.resolve()
|
||||
output = args.output.resolve()
|
||||
copied_blobs: set[str] = set()
|
||||
summary: list[dict[str, object]] = []
|
||||
|
||||
for model in args.model:
|
||||
manifest = model_manifest(source, model)
|
||||
if not manifest.is_file():
|
||||
raise SystemExit(f"Ollama model manifest missing: {model}: {manifest}")
|
||||
payload = json.loads(manifest.read_text(encoding="utf-8"))
|
||||
digests = [payload["config"]["digest"]]
|
||||
digests.extend(layer["digest"] for layer in payload.get("layers", []))
|
||||
|
||||
if not args.check_only:
|
||||
destination = output / manifest.relative_to(source)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(manifest, destination)
|
||||
|
||||
model_bytes = 0
|
||||
for digest in digests:
|
||||
if not digest.startswith("sha256:"):
|
||||
raise SystemExit(f"Unsupported Ollama digest for {model}: {digest}")
|
||||
blob_name = digest.replace(":", "-", 1)
|
||||
blob_source = source / "blobs" / blob_name
|
||||
if not blob_source.is_file():
|
||||
raise SystemExit(f"Ollama blob missing for {model}: {blob_source}")
|
||||
model_bytes += blob_source.stat().st_size
|
||||
if args.check_only or blob_name in copied_blobs:
|
||||
continue
|
||||
blob_output = output / "blobs" / blob_name
|
||||
blob_output.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(blob_source, blob_output)
|
||||
copied_blobs.add(blob_name)
|
||||
summary.append({"model": model, "bytes": model_bytes, "blobs": len(digests)})
|
||||
|
||||
if not args.check_only:
|
||||
(output / "DELIVERED_MODELS.json").write_text(
|
||||
json.dumps(summary, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(json.dumps(summary, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+88
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
为 Cython 编译产物生成 .py stub 文件。
|
||||
|
||||
每个 stub 同时支持：
|
||||
1. 子进程调用 (python apps/cli/run_xxx.py --args)
|
||||
-> __name__ == "__main__" 分支加载 .so 并调用 main()
|
||||
2. 模块导入 (from apps.cli.run_xxx import SomeClass)
|
||||
-> 模块级 __getattr__ 懒加载 .so ，透明转发属性访问
|
||||
|
||||
用法ￚ
|
||||
python generate_stubs.py <output_dir> [<output_dir2> ...]
|
||||
"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
STUB_TEMPLATE = (
|
||||
"# Auto-generated stub — delegates to the compiled Cython extension.\n"
|
||||
"# Supports both subprocess execution and module import.\n"
|
||||
"import importlib.util as _ilu\n"
|
||||
"import sys as _sys\n"
|
||||
"from pathlib import Path as _P\n"
|
||||
"\n"
|
||||
"_stem = _P(__file__).stem\n"
|
||||
"_so = next(_P(__file__).parent.glob(_stem + '.*.so'), None)\n"
|
||||
"_cached_mod = None\n"
|
||||
"\n"
|
||||
"\n"
|
||||
"def _load_compiled():\n"
|
||||
" global _cached_mod\n"
|
||||
" if _cached_mod is None:\n"
|
||||
" if _so is None:\n"
|
||||
" raise FileNotFoundError(\n"
|
||||
" f'Compiled module not found for {__file__}'\n"
|
||||
" )\n"
|
||||
" _spec = _ilu.spec_from_file_location(_stem, _so)\n"
|
||||
" _cached_mod = _ilu.module_from_spec(_spec)\n"
|
||||
" _spec.loader.exec_module(_cached_mod)\n"
|
||||
" return _cached_mod\n"
|
||||
"\n"
|
||||
"\n"
|
||||
"def __getattr__(name: str):\n"
|
||||
" \"\"\"Transparent proxy: forward attribute access to the compiled module.\"\"\"\n"
|
||||
" return getattr(_load_compiled(), name)\n"
|
||||
"\n"
|
||||
"\n"
|
||||
"if __name__ == '__main__':\n"
|
||||
" _mod = _load_compiled()\n"
|
||||
" _sys.exit(_mod.main() if hasattr(_mod, 'main') else 0)\n"
|
||||
)
|
||||
|
||||
|
||||
def generate_stubs(output_dir: Path) -> int:
|
||||
created = 0
|
||||
for so_path in sorted(output_dir.rglob("*.so")):
|
||||
stem = so_path.name.split(".")[0]
|
||||
if stem == "__init__":
|
||||
continue
|
||||
py_path = so_path.parent / f"{stem}.py"
|
||||
if py_path.exists():
|
||||
continue
|
||||
py_path.write_text(STUB_TEMPLATE, encoding="utf-8")
|
||||
print(f" stub: {py_path.relative_to(output_dir)}")
|
||||
created += 1
|
||||
return created
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print(f"用法: {sys.argv[0]} <output_dir> [<output_dir2> ...]", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
total = 0
|
||||
for arg in sys.argv[1:]:
|
||||
d = Path(arg)
|
||||
if not d.is_dir():
|
||||
print(f"目录不存在，跳过: {d}", file=sys.stderr)
|
||||
continue
|
||||
print(f"[generate_stubs] 扫描: {d}")
|
||||
n = generate_stubs(d)
|
||||
print(f"[generate_stubs] 已生成 {n} 个 stub: {d}")
|
||||
total += n
|
||||
|
||||
print(f"[generate_stubs] 合计: {total} 个 stub")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+171
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Representative import and behavior smoke tests for hardened builds."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
PLANTAI_PROBE = r"""
|
||||
import importlib
|
||||
import json
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, sys.argv[1])
|
||||
module = importlib.import_module("gfa_core.mas.core.orchestrator")
|
||||
|
||||
class Dummy:
|
||||
def __init__(self, name="dummy"):
|
||||
self.name = name
|
||||
def __getattr__(self, name):
|
||||
return Dummy(f"{self.name}.{name}")
|
||||
def __call__(self, *args, **kwargs):
|
||||
return Dummy(f"{self.name}()")
|
||||
|
||||
instance = module.Orchestrator(Dummy("llm"), Dummy("memory"), None)
|
||||
parsed = instance._parse_intent_from_query(
|
||||
"identify the WRKY gene family in Arabidopsis"
|
||||
)
|
||||
route = instance._route_query({"query": "What is WRKY?"})
|
||||
expected = {
|
||||
"gene_family": "WRKY",
|
||||
"species": "At",
|
||||
"stage": "identify",
|
||||
"task_type": "analysis",
|
||||
}
|
||||
assert parsed == expected, (parsed, expected)
|
||||
assert route == "knowledge", route
|
||||
print("@@RESULT@@" + json.dumps({
|
||||
"module": module.__name__,
|
||||
"parse_intent": parsed,
|
||||
"route": route,
|
||||
}))
|
||||
"""
|
||||
|
||||
|
||||
RAG_PROBE = r"""
|
||||
import importlib
|
||||
import json
|
||||
import sys
|
||||
|
||||
root = sys.argv[1]
|
||||
sys.path.insert(0, root)
|
||||
sys.path.insert(0, root + "/src")
|
||||
module = importlib.import_module("src.rag.retriever")
|
||||
instance = module.RAGRetriever()
|
||||
assert instance.count_chunks() == 0
|
||||
assert instance.get_context([]) == ""
|
||||
assert instance.index_documents([]) == 0
|
||||
api_module = importlib.import_module("api.app")
|
||||
route_paths = {route.path for route in api_module.app.routes}
|
||||
assert "/health" in route_paths
|
||||
assert "/api/v1/methodology/extract-jobs" in route_paths
|
||||
print("@@RESULT@@" + json.dumps({
|
||||
"module": module.__name__,
|
||||
"api_module": api_module.__name__,
|
||||
"methodology_route": True,
|
||||
"count_chunks": 0,
|
||||
"empty_context": "",
|
||||
"empty_index_count": 0,
|
||||
}))
|
||||
"""
|
||||
|
||||
QA_PROBE = r"""
|
||||
import importlib
|
||||
import json
|
||||
import sys
|
||||
|
||||
root = sys.argv[1]
|
||||
sys.path.insert(0, root)
|
||||
module = importlib.import_module("scripts.qa_v1_1_73_server")
|
||||
route_paths = {route.path for route in module.app.routes}
|
||||
assert "/health" in route_paths
|
||||
assert "/api/v1/qa-v1-1-73/health" in route_paths
|
||||
assert "/api/v1/qa-v1-1-73/info" in route_paths
|
||||
assert "/api/v1/qa-v1-1-73/chat" in route_paths
|
||||
print("@@RESULT@@" + json.dumps({
|
||||
"module": module.__name__,
|
||||
"health_route": True,
|
||||
"info_route": True,
|
||||
"chat_route": True,
|
||||
}))
|
||||
"""
|
||||
|
||||
|
||||
def run_probe(python: str, code: str, root: Path, timeout: int) -> dict:
|
||||
environment = os.environ.copy()
|
||||
completed = subprocess.run(
|
||||
[python, "-B", "-s", "-c", code, str(root)],
|
||||
cwd=root,
|
||||
env=environment,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
)
|
||||
result = None
|
||||
for line in completed.stdout.splitlines():
|
||||
if line.startswith("@@RESULT@@"):
|
||||
result = json.loads(line[len("@@RESULT@@") :])
|
||||
if completed.returncode != 0 or result is None:
|
||||
raise RuntimeError(
|
||||
f"probe failed rc={completed.returncode}\n"
|
||||
f"stdout={completed.stdout[-3000:]}\n"
|
||||
f"stderr={completed.stderr[-3000:]}"
|
||||
)
|
||||
result["stdout_tail"] = completed.stdout[-1000:]
|
||||
return result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--plantai", type=Path)
|
||||
parser.add_argument("--rag", type=Path)
|
||||
parser.add_argument("--qa", type=Path)
|
||||
parser.add_argument("--python")
|
||||
parser.add_argument("--qa-python")
|
||||
parser.add_argument("--timeout", type=int, default=45)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.plantai is None and args.rag is None and args.qa is None:
|
||||
parser.error("at least one of --plantai, --rag or --qa is required")
|
||||
if (args.plantai is not None or args.rag is not None) and not args.python:
|
||||
parser.error("--python is required for PlantAI/RAG probes")
|
||||
if args.qa is not None and not (args.qa_python or args.python):
|
||||
parser.error("--qa-python or --python is required for the QA probe")
|
||||
|
||||
payload = {}
|
||||
if args.plantai is not None:
|
||||
payload["plantai"] = run_probe(
|
||||
args.python, PLANTAI_PROBE, args.plantai.resolve(), args.timeout
|
||||
)
|
||||
if args.rag is not None:
|
||||
payload["rag"] = run_probe(
|
||||
args.python, RAG_PROBE, args.rag.resolve(), args.timeout
|
||||
)
|
||||
if args.qa is not None:
|
||||
payload["qa"] = run_probe(
|
||||
args.qa_python or args.python,
|
||||
QA_PROBE,
|
||||
args.qa.resolve(),
|
||||
args.timeout,
|
||||
)
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(json.dumps(payload, ensure_ascii=False, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user