diff --git a/tooling/compile/compile_qa.sh b/tooling/compile/compile_qa.sh new file mode 100755 index 0000000..bbc6c8f --- /dev/null +++ b/tooling/compile/compile_qa.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# Compile the QA V1.1.73 runtime surface into stripped Cython extensions. +set -euo pipefail +umask 027 + +SOURCE_ROOT="${1:?Usage: $0 [python]}" +OUTPUT_DIR="${2:?Usage: $0 [python]}" +PYTHON="${3:-python3}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VISIBILITY_TOOL="${ELF_VISIBILITY_TOOL:-$SCRIPT_DIR/tools/elf_visibility.py}" +ENTRY_SCRIPT="scripts/qa_v1_1_73_server.py" + +SOURCE_ROOT="$(realpath "$SOURCE_ROOT")" +OUTPUT_DIR="$(realpath -m "$OUTPUT_DIR")" + +[[ -d "$SOURCE_ROOT" ]] || { echo "QA source directory missing: $SOURCE_ROOT" >&2; exit 2; } +[[ -x "$PYTHON" ]] || { echo "Python is not executable: $PYTHON" >&2; exit 2; } +[[ -f "$VISIBILITY_TOOL" ]] || { echo "ELF visibility tool missing: $VISIBILITY_TOOL" >&2; exit 2; } +for command_name in gcc strip rsync; do command -v "$command_name" >/dev/null; done + +case "$OUTPUT_DIR" in + "/"|"$(realpath -m "$HOME")"|"$SOURCE_ROOT") + echo "Unsafe output directory: $OUTPUT_DIR" >&2 + exit 2 + ;; +esac +[[ ! -e "$OUTPUT_DIR" ]] || { + echo "Output already exists; refusing to overwrite: $OUTPUT_DIR" >&2 + exit 2 +} + +mkdir -p \ + "$OUTPUT_DIR/scripts" \ + "$OUTPUT_DIR/src/rag/qa_v1_1_73" \ + "$OUTPUT_DIR/src/rag/retrieval" \ + "$OUTPUT_DIR/src/rag/dual_kb_taxonomy_rag" \ + "$OUTPUT_DIR/configs/rag" + +echo "[QA 1/6] Copying the audited runtime surface" +install -m 0750 "$SOURCE_ROOT/$ENTRY_SCRIPT" "$OUTPUT_DIR/$ENTRY_SCRIPT" +rsync -a --include='*.py' --exclude='*' \ + "$SOURCE_ROOT/src/rag/qa_v1_1_73/" "$OUTPUT_DIR/src/rag/qa_v1_1_73/" +rsync -a --include='*.py' --exclude='*' \ + "$SOURCE_ROOT/src/rag/retrieval/" "$OUTPUT_DIR/src/rag/retrieval/" +install -m 0640 \ + "$SOURCE_ROOT/src/rag/dual_kb_taxonomy_rag/entity_alias.py" \ + "$OUTPUT_DIR/src/rag/dual_kb_taxonomy_rag/entity_alias.py" +# 2026-08-31 主项目新增:normalized_release.py(backend_guard 校验 + 图适配器) +[[ -f "$SOURCE_ROOT/src/rag/normalized_release.py" ]] && install -m 0640 \ + "$SOURCE_ROOT/src/rag/normalized_release.py" \ + "$OUTPUT_DIR/src/rag/normalized_release.py" +for relative in \ + configs/rag/entity_alias.yaml \ + configs/rag/relation_taxonomy.yaml \ + pyproject.toml \ + requirements.txt \ + requirements-rag.txt +do + if [[ -f "$SOURCE_ROOT/$relative" ]]; then + install -D -m 0640 "$SOURCE_ROOT/$relative" "$OUTPUT_DIR/$relative" + fi +done + +# Namespace/package markers are intentionally left as minimal Python files. +touch \ + "$OUTPUT_DIR/scripts/__init__.py" \ + "$OUTPUT_DIR/src/__init__.py" \ + "$OUTPUT_DIR/src/rag/__init__.py" \ + "$OUTPUT_DIR/src/rag/dual_kb_taxonomy_rag/__init__.py" + +echo "[QA 2/6] Discovering Python modules" +PY_FILES=() +while IFS= read -r -d '' path; do PY_FILES+=("$path"); done \ + < <(find "$OUTPUT_DIR" -type f -name '*.py' \ + -not -path "*/$ENTRY_SCRIPT" \ + -not -name '__init__.py' -print0) +TOTAL="${#PY_FILES[@]}" +(( TOTAL > 0 )) || { echo "No QA modules selected for compilation" >&2; exit 3; } + +INCLUDE_PY="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("INCLUDEPY") or "")')" +EXT_SUFFIX="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX") or ".so")')" +[[ -n "$INCLUDE_PY" ]] || { echo "Python INCLUDEPY is empty" >&2; exit 3; } + +echo "[QA 3/6] Cython preflight ($TOTAL modules)" +COUNT=0 +CYTHON_FAILURES=0 +for py_file in "${PY_FILES[@]}"; do + COUNT=$((COUNT + 1)) + directory="$(dirname "$py_file")" + module="$(basename "$py_file" .py)" + c_file="$directory/$module.c" + echo " [$COUNT/$TOTAL] ${py_file#"$OUTPUT_DIR"/}" + if ! "$PYTHON" -m cython -3 \ + --directive always_allow_keywords=true,annotation_typing=False \ + --no-docstrings "$py_file" -o "$c_file"; then + CYTHON_FAILURES=$((CYTHON_FAILURES + 1)) + fi +done +(( CYTHON_FAILURES == 0 )) || { + echo "QA Cython preflight failed: modules=$CYTHON_FAILURES" >&2 + exit 3 +} + +echo "[QA 4/6] GCC hidden visibility + strip ($TOTAL modules)" +COUNT=0 +for py_file in "${PY_FILES[@]}"; do + COUNT=$((COUNT + 1)) + directory="$(dirname "$py_file")" + module="$(basename "$py_file" .py)" + c_file="$directory/$module.c" + so_file="$directory/$module$EXT_SUFFIX" + echo " [$COUNT/$TOTAL] ${py_file#"$OUTPUT_DIR"/}" + gcc -shared -pthread -fPIC -O2 -fvisibility=hidden \ + -I"$INCLUDE_PY" -o "$so_file" "$c_file" + strip --strip-unneeded "$so_file" + "$PYTHON" -B "$VISIBILITY_TOOL" \ + --hide-non-pyinit --check "$so_file" >/dev/null + rm -f -- "$c_file" "$py_file" +done + +echo "[QA 5/6] Restoring package markers" +while IFS= read -r -d '' directory; do + if find "$directory" -maxdepth 1 -type f \( -name '*.so' -o -name '*.py' \) -print -quit | grep -q .; then + [[ -f "$directory/__init__.py" ]] || touch "$directory/__init__.py" + fi +done < <(find "$OUTPUT_DIR" -type d -print0) + +echo "[QA 6/6] Final checks" +SO_COUNT="$(find "$OUTPUT_DIR" -type f -name '*.so' | wc -l)" +[[ "$SO_COUNT" -eq "$TOTAL" ]] || { + echo "QA count mismatch: selected=$TOTAL produced=$SO_COUNT" >&2 + exit 4 +} +if find "$OUTPUT_DIR" -type f -name '*.c' -print -quit | grep -q .; then + echo "Generated C source remains in output" >&2 + exit 4 +fi + +echo "QA compile+harden complete: modules=$SO_COUNT output=$OUTPUT_DIR" diff --git a/tooling/compile/tools/audit_compiled_tree.py b/tooling/compile/tools/audit_compiled_tree.py new file mode 100755 index 0000000..ae9bbd5 --- /dev/null +++ b/tooling/compile/tools/audit_compiled_tree.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +"""Audit a compiled tree and bind it to a tamper-evident build marker.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import sys +import sysconfig +from datetime import datetime +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from elf_visibility import summarize # noqa: E402 + + +MARKER_NAME = ".hardened-build.json" +TREE_DIGEST_POLICY = "immutable-runtime-v2" +MUTABLE_TOP_LEVEL_DIRS = { + "cost_logs", + "data", + "input", + "logs", + "memory_storage", + "neo4j", + "output", + "outputs", + "rag_kb_store", + "service_data", + "temp", + "v15_output", + "workspace", +} + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def is_digest_excluded(relative: str) -> bool: + parts = Path(relative).parts + return ( + relative == MARKER_NAME + or not parts + or parts[0] in MUTABLE_TOP_LEVEL_DIRS + or "__pycache__" in parts + or relative.endswith((".pyc", ".pyo")) + ) + + +def tree_digest(root: Path) -> tuple[str, int]: + digest = hashlib.sha256() + count = 0 + for path in sorted( + (item for item in root.rglob("*") if item.is_file()), + key=lambda item: item.relative_to(root).as_posix(), + ): + relative = path.relative_to(root).as_posix() + if is_digest_excluded(relative): + continue + count += 1 + digest.update(relative.encode("utf-8")) + digest.update(b"\0") + digest.update(bytes.fromhex(file_sha256(path))) + digest.update(b"\0") + return digest.hexdigest(), count + + +def audit_tree(root: Path, project: str) -> dict[str, Any]: + errors: list[dict[str, Any]] = [] + extensions = sorted(root.rglob("*.so")) + expected_suffix = sysconfig.get_config_var("EXT_SUFFIX") or ".so" + hidden_non_pyinit = 0 + + if not extensions: + errors.append({"kind": "no_extensions"}) + + for extension in extensions: + relative = extension.relative_to(root).as_posix() + if not extension.name.endswith(expected_suffix): + errors.append( + { + "kind": "python_abi_suffix_mismatch", + "file": relative, + "expected_suffix": expected_suffix, + } + ) + try: + info = summarize(extension, extension.read_bytes()) + except BaseException as exc: + errors.append( + { + "kind": "elf_parse_failed", + "file": relative, + "error": f"{type(exc).__name__}: {exc}", + } + ) + continue + if info["has_symtab"]: + errors.append({"kind": "symtab_present", "file": relative}) + if len(info["pyinit"]) != 1 or not info["all_pyinit_default"]: + errors.append( + { + "kind": "invalid_pyinit", + "file": relative, + "pyinit": info["pyinit"], + } + ) + if not info["all_non_pyinit_hidden"]: + errors.append( + { + "kind": "visible_non_pyinit", + "file": relative, + "symbols": info["non_pyinit"], + } + ) + hidden_non_pyinit += len(info["non_pyinit"]) + + c_files = [ + path.relative_to(root).as_posix() + for path in root.rglob("*.c") + if path.is_file() + ] + if c_files: + errors.append({"kind": "generated_c_retained", "files": c_files}) + + sensitive = [] + for path in root.rglob("*"): + if not path.is_file(): + continue + lowered = path.name.lower() + if ( + lowered.endswith(".key") + or "private_key" in lowered + or "master_key" in lowered + or lowered == "private_key.pass" + ): + sensitive.append(path.relative_to(root).as_posix()) + if sensitive: + errors.append({"kind": "sensitive_filename", "files": sensitive}) + + digest, file_count = tree_digest(root) + return { + "project": project, + "tree_digest_policy": TREE_DIGEST_POLICY, + "python_version": sys.version.split()[0], + "extension_suffix": expected_suffix, + "extension_count": len(extensions), + "hidden_non_pyinit_count": hidden_non_pyinit, + "file_count_without_marker": file_count, + "tree_sha256": digest, + "error_count": len(errors), + "errors": errors, + } + + +def atomic_write_json(path: Path, payload: dict[str, Any]) -> None: + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + temporary.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + os.chmod(temporary, 0o640) + os.replace(temporary, path) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--tree", type=Path, required=True) + parser.add_argument("--project", required=True) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--write-marker", action="store_true") + mode.add_argument("--require-marker", action="store_true") + args = parser.parse_args() + + root = args.tree.resolve() + if not root.is_dir(): + raise SystemExit(f"Compiled tree missing: {root}") + + report = audit_tree(root, args.project) + marker_path = root / MARKER_NAME + + if args.require_marker: + if not marker_path.is_file(): + report["errors"].append({"kind": "marker_missing"}) + else: + marker = json.loads(marker_path.read_text(encoding="utf-8")) + report["marker"] = marker + for field in ( + "project", + "tree_digest_policy", + "python_version", + "extension_suffix", + "extension_count", + "tree_sha256", + ): + if marker.get(field) != report.get(field): + report["errors"].append( + { + "kind": "marker_mismatch", + "field": field, + "expected": marker.get(field), + "actual": report.get(field), + } + ) + + report["error_count"] = len(report["errors"]) + report["ok"] = report["error_count"] == 0 + + if args.write_marker and report["ok"]: + marker = { + key: report[key] + for key in ( + "project", + "tree_digest_policy", + "python_version", + "extension_suffix", + "extension_count", + "hidden_non_pyinit_count", + "file_count_without_marker", + "tree_sha256", + ) + } + marker["created_at"] = datetime.now().astimezone().isoformat( + timespec="seconds" + ) + atomic_write_json(marker_path, marker) + report["marker_path"] = str(marker_path) + + print(json.dumps(report, ensure_ascii=False, sort_keys=True)) + return 0 if report["ok"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tooling/compile/tools/build_delivery_manifest.py b/tooling/compile/tools/build_delivery_manifest.py new file mode 100755 index 0000000..150c9db --- /dev/null +++ b/tooling/compile/tools/build_delivery_manifest.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Write the customer-facing manifest for the three-project offline delivery.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + + +def tree_bytes(path: Path) -> int: + total = 0 + for directory, _, files in os.walk(path, followlinks=True): + for filename in files: + total += (Path(directory) / filename).stat().st_size + return total + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--runtime-root", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--version", required=True) + parser.add_argument("--generation-model", required=True) + parser.add_argument("--embed-model", required=True) + args = parser.parse_args() + + root = args.runtime_root.resolve() + assets = { + "qdrant_storage": root / "qdrant" / "storage", + "neo4j_data": root / "neo4j" / "data", + "bge_reranker": root / "QA" / "models" / "bge-reranker-base", + "ollama_models": root / "ollama-models", + } + missing = [str(path) for path in assets.values() if not path.is_dir()] + if missing: + raise SystemExit("Missing runtime asset directories: " + ", ".join(missing)) + + payload = { + "release_version": args.version, + "runtime_data_archive": f"plantai-rag-runtime-data-{args.version}.tar", + "applications": { + "PlantAI": {"path": "app/PlantAI_compiled", "port": 7880}, + "RAG": {"path": "app/RAG", "port": 7900}, + "QA": {"path": "app/QA", "port": 7925}, + }, + "containers": { + "qdrant": { + "image": "qdrant/qdrant:latest", + "http_port": 16333, + "grpc_port": 16334, + "data_path": "runtime-assets/qdrant/storage", + }, + "neo4j": { + "image": "neo4j:5.15.0-community", + "http_port": 17474, + "bolt_port": 17687, + "data_path": "runtime-assets/neo4j/data", + }, + }, + "models": { + "ollama_generation": args.generation_model, + "ollama_embedding": args.embed_model, + "ollama_path": "runtime-assets/ollama-models", + "reranker_path": "runtime-assets/QA/models/bge-reranker-base", + }, + "asset_bytes": {name: tree_bytes(path) for name, path in assets.items()}, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + print(json.dumps(payload["asset_bytes"], ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tooling/compile/tools/elf_visibility.py b/tooling/compile/tools/elf_visibility.py new file mode 100755 index 0000000..1a009fe --- /dev/null +++ b/tooling/compile/tools/elf_visibility.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Inspect or hide non-PyInit defined symbols in an ELF64 little-endian .dynsym.""" + +from __future__ import annotations + +import argparse +import json +import os +import struct +import tempfile +from pathlib import Path +from typing import Any + + +STN_UNDEF = 0 +STV_DEFAULT = 0 +STV_HIDDEN = 2 + + +def c_string(data: bytes | bytearray, offset: int) -> str: + end = data.find(b"\0", offset) + if end < 0: + end = len(data) + return bytes(data[offset:end]).decode("utf-8", errors="replace") + + +def parse_sections(data: bytes | bytearray) -> tuple[list[dict[str, Any]], bytes]: + if data[:4] != b"\x7fELF": + raise ValueError("not an ELF file") + if data[4] != 2: + raise ValueError("only ELF64 is supported") + if data[5] != 1: + raise ValueError("only little-endian ELF is supported") + + section_offset = struct.unpack_from("= len(sections): + raise ValueError("invalid section-name string-table index") + names_section = sections[section_name_index] + names = bytes( + data[ + names_section["offset"] : names_section["offset"] + names_section["size"] + ] + ) + for section in sections: + section["name"] = c_string(names, section["name_offset"]) + return sections, names + + +def inspect_dynsym(data: bytes | bytearray) -> tuple[list[dict[str, Any]], dict[str, Any]]: + sections, _ = parse_sections(data) + dynsym = next((section for section in sections if section["name"] == ".dynsym"), None) + if dynsym is None: + raise ValueError("missing .dynsym") + if dynsym["link"] >= len(sections): + raise ValueError("invalid .dynsym string-table link") + dynstr_section = sections[dynsym["link"]] + dynstr = data[ + dynstr_section["offset"] : dynstr_section["offset"] + dynstr_section["size"] + ] + entry_size = dynsym["entry_size"] or 24 + if entry_size < 24: + raise ValueError("invalid ELF64 symbol entry size") + + symbols = [] + count = dynsym["size"] // entry_size + for index in range(count): + offset = dynsym["offset"] + index * entry_size + name_offset = struct.unpack_from("> 4, + "type": info & 0xF, + "visibility": other & 0x3, + "section_index": section_index, + "value": value, + "size": size, + "defined": section_index != STN_UNDEF, + } + ) + return symbols, dynsym + + +def summarize(path: Path, data: bytes | bytearray) -> dict[str, Any]: + sections, _ = parse_sections(data) + symbols, _ = inspect_dynsym(data) + defined = [symbol for symbol in symbols if symbol["defined"] and symbol["name"]] + pyinit = [symbol for symbol in defined if symbol["name"].startswith("PyInit_")] + non_pyinit = [ + symbol for symbol in defined if not symbol["name"].startswith("PyInit_") + ] + return { + "path": str(path), + "has_symtab": any(section["name"] == ".symtab" for section in sections), + "defined_dynamic_count": len(defined), + "pyinit": [ + {"name": symbol["name"], "visibility": symbol["visibility"]} + for symbol in pyinit + ], + "non_pyinit": [ + {"name": symbol["name"], "visibility": symbol["visibility"]} + for symbol in non_pyinit + ], + "all_non_pyinit_hidden": all( + symbol["visibility"] == STV_HIDDEN for symbol in non_pyinit + ), + "all_pyinit_default": bool(pyinit) + and all(symbol["visibility"] == STV_DEFAULT for symbol in pyinit), + } + + +def hide_non_pyinit(path: Path) -> dict[str, Any]: + original = path.read_bytes() + data = bytearray(original) + symbols, _ = inspect_dynsym(data) + changed = [] + for symbol in symbols: + if ( + symbol["defined"] + and symbol["name"] + and not symbol["name"].startswith("PyInit_") + and symbol["visibility"] != STV_HIDDEN + ): + offset = symbol["entry_offset"] + 5 + data[offset] = (data[offset] & ~0x3) | STV_HIDDEN + changed.append(symbol["name"]) + + if changed: + stat = path.stat() + with tempfile.NamedTemporaryFile( + mode="wb", dir=path.parent, prefix=f".{path.name}.", delete=False + ) as handle: + temporary = Path(handle.name) + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + os.chmod(temporary, stat.st_mode) + os.replace(temporary, path) + + result = summarize(path, data) + result["changed"] = changed + return result + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("elf", type=Path) + parser.add_argument("--hide-non-pyinit", action="store_true") + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + + path = args.elf.resolve() + result = ( + hide_non_pyinit(path) + if args.hide_non_pyinit + else summarize(path, path.read_bytes()) + ) + print(json.dumps(result, ensure_ascii=False, sort_keys=True)) + if args.check: + valid = ( + not result["has_symtab"] + and result["all_pyinit_default"] + and result["all_non_pyinit_hidden"] + and len(result["pyinit"]) == 1 + ) + return 0 if valid else 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tooling/compile/tools/export_ollama_models.py b/tooling/compile/tools/export_ollama_models.py new file mode 100755 index 0000000..eee9ab8 --- /dev/null +++ b/tooling/compile/tools/export_ollama_models.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Copy selected Ollama manifests and their referenced blobs.""" + +from __future__ import annotations + +import argparse +import json +import shutil +from pathlib import Path + + +def model_manifest(source: Path, model: str) -> Path: + repository, sep, tag = model.partition(":") + if not sep: + tag = "latest" + return source / "manifests" / "registry.ollama.ai" / "library" / repository / tag + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--source", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--model", action="append", required=True) + parser.add_argument("--check-only", action="store_true") + args = parser.parse_args() + + source = args.source.resolve() + output = args.output.resolve() + copied_blobs: set[str] = set() + summary: list[dict[str, object]] = [] + + for model in args.model: + manifest = model_manifest(source, model) + if not manifest.is_file(): + raise SystemExit(f"Ollama model manifest missing: {model}: {manifest}") + payload = json.loads(manifest.read_text(encoding="utf-8")) + digests = [payload["config"]["digest"]] + digests.extend(layer["digest"] for layer in payload.get("layers", [])) + + if not args.check_only: + destination = output / manifest.relative_to(source) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(manifest, destination) + + model_bytes = 0 + for digest in digests: + if not digest.startswith("sha256:"): + raise SystemExit(f"Unsupported Ollama digest for {model}: {digest}") + blob_name = digest.replace(":", "-", 1) + blob_source = source / "blobs" / blob_name + if not blob_source.is_file(): + raise SystemExit(f"Ollama blob missing for {model}: {blob_source}") + model_bytes += blob_source.stat().st_size + if args.check_only or blob_name in copied_blobs: + continue + blob_output = output / "blobs" / blob_name + blob_output.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(blob_source, blob_output) + copied_blobs.add(blob_name) + summary.append({"model": model, "bytes": model_bytes, "blobs": len(digests)}) + + if not args.check_only: + (output / "DELIVERED_MODELS.json").write_text( + json.dumps(summary, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + print(json.dumps(summary, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tooling/compile/tools/generate_stubs.py b/tooling/compile/tools/generate_stubs.py new file mode 100755 index 0000000..7bc5a4c --- /dev/null +++ b/tooling/compile/tools/generate_stubs.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +""" +为 Cython 编译产物生成 .py stub 文件。 + +每个 stub 同时支持： + 1. 子进程调用 (python apps/cli/run_xxx.py --args) + -> __name__ == "__main__" 分支加载 .so 并调用 main() + 2. 模块导入 (from apps.cli.run_xxx import SomeClass) + -> 模块级 __getattr__ 懒加载 .so ，透明转发属性访问 + +用法ￚ + python generate_stubs.py [ ...] +""" +import sys +from pathlib import Path + +STUB_TEMPLATE = ( + "# Auto-generated stub — delegates to the compiled Cython extension.\n" + "# Supports both subprocess execution and module import.\n" + "import importlib.util as _ilu\n" + "import sys as _sys\n" + "from pathlib import Path as _P\n" + "\n" + "_stem = _P(__file__).stem\n" + "_so = next(_P(__file__).parent.glob(_stem + '.*.so'), None)\n" + "_cached_mod = None\n" + "\n" + "\n" + "def _load_compiled():\n" + " global _cached_mod\n" + " if _cached_mod is None:\n" + " if _so is None:\n" + " raise FileNotFoundError(\n" + " f'Compiled module not found for {__file__}'\n" + " )\n" + " _spec = _ilu.spec_from_file_location(_stem, _so)\n" + " _cached_mod = _ilu.module_from_spec(_spec)\n" + " _spec.loader.exec_module(_cached_mod)\n" + " return _cached_mod\n" + "\n" + "\n" + "def __getattr__(name: str):\n" + " \"\"\"Transparent proxy: forward attribute access to the compiled module.\"\"\"\n" + " return getattr(_load_compiled(), name)\n" + "\n" + "\n" + "if __name__ == '__main__':\n" + " _mod = _load_compiled()\n" + " _sys.exit(_mod.main() if hasattr(_mod, 'main') else 0)\n" +) + + +def generate_stubs(output_dir: Path) -> int: + created = 0 + for so_path in sorted(output_dir.rglob("*.so")): + stem = so_path.name.split(".")[0] + if stem == "__init__": + continue + py_path = so_path.parent / f"{stem}.py" + if py_path.exists(): + continue + py_path.write_text(STUB_TEMPLATE, encoding="utf-8") + print(f" stub: {py_path.relative_to(output_dir)}") + created += 1 + return created + + +def main(): + if len(sys.argv) < 2: + print(f"用法: {sys.argv[0]} [ ...]", file=sys.stderr) + sys.exit(1) + + total = 0 + for arg in sys.argv[1:]: + d = Path(arg) + if not d.is_dir(): + print(f"目录不存在，跳过: {d}", file=sys.stderr) + continue + print(f"[generate_stubs] 扫描: {d}") + n = generate_stubs(d) + print(f"[generate_stubs] 已生成 {n} 个 stub: {d}") + total += n + + print(f"[generate_stubs] 合计: {total} 个 stub") + + +if __name__ == "__main__": + main() diff --git a/tooling/compile/tools/smoke_dual_build.py b/tooling/compile/tools/smoke_dual_build.py new file mode 100755 index 0000000..81dbf3e --- /dev/null +++ b/tooling/compile/tools/smoke_dual_build.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Representative import and behavior smoke tests for hardened builds.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from pathlib import Path + + +PLANTAI_PROBE = r""" +import importlib +import json +import sys + +sys.path.insert(0, sys.argv[1]) +module = importlib.import_module("gfa_core.mas.core.orchestrator") + +class Dummy: + def __init__(self, name="dummy"): + self.name = name + def __getattr__(self, name): + return Dummy(f"{self.name}.{name}") + def __call__(self, *args, **kwargs): + return Dummy(f"{self.name}()") + +instance = module.Orchestrator(Dummy("llm"), Dummy("memory"), None) +parsed = instance._parse_intent_from_query( + "identify the WRKY gene family in Arabidopsis" +) +route = instance._route_query({"query": "What is WRKY?"}) +expected = { + "gene_family": "WRKY", + "species": "At", + "stage": "identify", + "task_type": "analysis", +} +assert parsed == expected, (parsed, expected) +assert route == "knowledge", route +print("@@RESULT@@" + json.dumps({ + "module": module.__name__, + "parse_intent": parsed, + "route": route, +})) +""" + + +RAG_PROBE = r""" +import importlib +import json +import sys + +root = sys.argv[1] +sys.path.insert(0, root) +sys.path.insert(0, root + "/src") +module = importlib.import_module("src.rag.retriever") +instance = module.RAGRetriever() +assert instance.count_chunks() == 0 +assert instance.get_context([]) == "" +assert instance.index_documents([]) == 0 +api_module = importlib.import_module("api.app") +route_paths = {route.path for route in api_module.app.routes} +assert "/health" in route_paths +assert "/api/v1/methodology/extract-jobs" in route_paths +print("@@RESULT@@" + json.dumps({ + "module": module.__name__, + "api_module": api_module.__name__, + "methodology_route": True, + "count_chunks": 0, + "empty_context": "", + "empty_index_count": 0, +})) +""" + +QA_PROBE = r""" +import importlib +import json +import sys + +root = sys.argv[1] +sys.path.insert(0, root) +module = importlib.import_module("scripts.qa_v1_1_73_server") +route_paths = {route.path for route in module.app.routes} +assert "/health" in route_paths +assert "/api/v1/qa-v1-1-73/health" in route_paths +assert "/api/v1/qa-v1-1-73/info" in route_paths +assert "/api/v1/qa-v1-1-73/chat" in route_paths +print("@@RESULT@@" + json.dumps({ + "module": module.__name__, + "health_route": True, + "info_route": True, + "chat_route": True, +})) +""" + + +def run_probe(python: str, code: str, root: Path, timeout: int) -> dict: + environment = os.environ.copy() + completed = subprocess.run( + [python, "-B", "-s", "-c", code, str(root)], + cwd=root, + env=environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + check=False, + ) + result = None + for line in completed.stdout.splitlines(): + if line.startswith("@@RESULT@@"): + result = json.loads(line[len("@@RESULT@@") :]) + if completed.returncode != 0 or result is None: + raise RuntimeError( + f"probe failed rc={completed.returncode}\n" + f"stdout={completed.stdout[-3000:]}\n" + f"stderr={completed.stderr[-3000:]}" + ) + result["stdout_tail"] = completed.stdout[-1000:] + return result + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--plantai", type=Path) + parser.add_argument("--rag", type=Path) + parser.add_argument("--qa", type=Path) + parser.add_argument("--python") + parser.add_argument("--qa-python") + parser.add_argument("--timeout", type=int, default=45) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + + if args.plantai is None and args.rag is None and args.qa is None: + parser.error("at least one of --plantai, --rag or --qa is required") + if (args.plantai is not None or args.rag is not None) and not args.python: + parser.error("--python is required for PlantAI/RAG probes") + if args.qa is not None and not (args.qa_python or args.python): + parser.error("--qa-python or --python is required for the QA probe") + + payload = {} + if args.plantai is not None: + payload["plantai"] = run_probe( + args.python, PLANTAI_PROBE, args.plantai.resolve(), args.timeout + ) + if args.rag is not None: + payload["rag"] = run_probe( + args.python, RAG_PROBE, args.rag.resolve(), args.timeout + ) + if args.qa is not None: + payload["qa"] = run_probe( + args.qa_python or args.python, + QA_PROBE, + args.qa.resolve(), + args.timeout, + ) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + print(json.dumps(payload, ensure_ascii=False, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())