fix: stabilize Router model error handling

This commit is contained in:
potaser
2026-08-22 00:47:11 +08:00
parent 8985f5cbdd
commit 052b0c1d5d
4 changed files with 424 additions and 10 deletions
+11
View File
@@ -62,12 +62,23 @@ DEFAULT_AGENT_LANG = "zh"
if PLANTAI_RAG_ROOT.exists() and str(PLANTAI_RAG_ROOT) not in sys.path:
sys.path.insert(0, str(PLANTAI_RAG_ROOT))
_deepseek_api_key_before_rag_import = os.environ.get("DEEPSEEK_API_KEY")
_plantai_new_api_bundle_configured = all(
str(os.environ.get(name) or "").strip()
for name in ("NEW_API_KEY", "NEW_API_BASE_URL", "NEW_API_MODEL")
)
try:
from src.rag.unified.bbav5_methodology_config import build_bbav5_methodology_config
METHODOLOGY_CONFIG_AVAILABLE = True
except Exception as e:
METHODOLOGY_CONFIG_AVAILABLE = False
print(f"⚠️ 方法学配置模块加载失败: {e}")
finally:
if _plantai_new_api_bundle_configured:
if _deepseek_api_key_before_rag_import is None:
os.environ.pop("DEEPSEEK_API_KEY", None)
else:
os.environ["DEEPSEEK_API_KEY"] = _deepseek_api_key_before_rag_import
try:
from gfa_core.mas.agents.interpretation_evidence import build_evidence_report
+134 -10
View File
@@ -1,5 +1,7 @@
import asyncio
import json
import hashlib
import logging
import os
import re
from collections import Counter
@@ -20,6 +22,64 @@ from gfa_core.mas.workflow_catalog import (
from gfa_core.mas.workflow_models import CurrentRecommendation, GoalLedger
logger = logging.getLogger(__name__)
_ROUTER_ERROR_TEXT_LIMIT = 1000
_ROUTER_MODEL_FAILURE_MESSAGE_PREFIX = "Router 调用 DeepSeek 失败("
_ROUTER_CREDENTIAL_PATTERN = re.compile(
r"(?i)\b(api[_-]?key|authorization|(?:access|refresh)[_-]?token|token|secret)"
r"\b\s*[:=]\s*[^,;}\]\r\n]+"
)
def _sanitize_router_error_text(value: Any) -> str:
text = str(value or "").strip()
if not text:
return ""
return _ROUTER_CREDENTIAL_PATTERN.sub(r"\1=<REDACTED>", text)[
:_ROUTER_ERROR_TEXT_LIMIT
]
class RouterModelCallError(RuntimeError):
"""Allow-listed details from a failed Router model request."""
def __init__(self, cause: Exception, *, attempts: int, model: str) -> None:
self.status_code = getattr(cause, "status_code", None)
self.attempts = attempts
self.model = _sanitize_router_error_text(model) or "unknown"
self.request_id = _sanitize_router_error_text(
getattr(cause, "request_id", "")
)
body = getattr(cause, "body", None)
error: Mapping[str, Any] = {}
if isinstance(body, Mapping):
nested_error = body.get("error")
error = nested_error if isinstance(nested_error, Mapping) else body
self.error_code = _sanitize_router_error_text(error.get("code"))
self.upstream_message = _sanitize_router_error_text(error.get("message"))
if not self.upstream_message:
self.upstream_message = _sanitize_router_error_text(str(cause))
if not self.upstream_message:
self.upstream_message = type(cause).__name__
super().__init__(self.user_message())
def user_message(self) -> str:
details = []
if self.status_code is not None:
details.append(f"HTTP {self.status_code}")
details.append(f"模型 {self.model}")
details.append(f"已尝试 {self.attempts} 次")
if self.error_code:
details.append(f"错误码 {self.error_code}")
if self.request_id:
details.append(f"Request ID {self.request_id}")
return f"Router 调用 DeepSeek 失败({','.join(details)}):{self.upstream_message}"
InteractionKind = Literal[
"supplement",
"correction",
@@ -561,6 +621,7 @@ class RouterRuntime:
) -> None:
self.llm_manager = llm_manager
self.tools = tools
self._last_model_call_error: RouterModelCallError | None = None
if max_iterations is None:
raw_budget = os.environ.get("PLANTAI_MAIN_AGENT_MAX_TOOL_CALLS", "10")
try:
@@ -577,6 +638,50 @@ class RouterRuntime:
else:
self.max_iterations = min(max(0, max_iterations), 5)
async def _call_router_model(
self,
messages: list[LLMMessage],
**kwargs: Any,
) -> Any:
"""Call the Router model with bounded retries for transient failures."""
max_attempts = 3
retryable_statuses = {402, 408, 409, 429, 500, 502, 503, 504}
for attempt in range(1, max_attempts + 1):
try:
return await self.llm_manager.acall(
messages,
provider="deepseek",
allow_fallback=False,
**kwargs,
)
except Exception as exc:
status = getattr(exc, "status_code", None)
retryable = status is None or status in retryable_statuses
logger.warning(
"Router model call failed: attempt=%s/%s exception=%s status=%s retryable=%s",
attempt,
max_attempts,
type(exc).__name__,
status,
retryable,
)
if not retryable or attempt >= max_attempts:
failure = RouterModelCallError(
exc,
attempts=attempt,
model=str(
getattr(
self.llm_manager,
"_default_model",
"deepseek-v4-flash",
)
or "deepseek-v4-flash"
),
)
logger.error("Router model unavailable: %s", failure.user_message())
raise failure from exc
await asyncio.sleep(0.5 * (2 ** (attempt - 1)))
@staticmethod
def _action_for_route(
decision: RouteDecision,
@@ -1126,15 +1231,16 @@ class RouterRuntime:
calls = 0
for attempt in range(min(2, self.max_iterations)):
try:
response = await self.llm_manager.acall(
response = await self._call_router_model(
route_messages,
provider="deepseek",
allow_fallback=False,
tools=[],
tool_choice="none",
temperature=0,
response_format={"type": "json_object"},
)
except RouterModelCallError as exc:
self._last_model_call_error = exc
return None, calls + 1, []
except Exception:
return None, calls + 1, []
calls += 1
@@ -1181,6 +1287,7 @@ class RouterRuntime:
goal_ledger: GoalLedger | None = None,
language: str = "zh",
) -> RouterOutcome:
self._last_model_call_error = None
ui_action = self._config_dialog_ui_action(query)
if ui_action is not None:
return RouterOutcome(
@@ -1276,7 +1383,11 @@ class RouterRuntime:
if not decision_trace:
return RouterOutcome(
status="agent_unavailable",
message="The Router model is currently unavailable.",
message=(
self._last_model_call_error.user_message()
if self._last_model_call_error is not None
else "The Router model is currently unavailable."
),
)
if decision_calls < 2:
return RouterOutcome(
@@ -1522,10 +1633,8 @@ class RouterRuntime:
]
force_final = not remaining_needs or recognized_intent is not None
try:
response = await self.llm_manager.acall(
response = await self._call_router_model(
messages,
provider="deepseek",
allow_fallback=False,
tools=[] if force_final else available_definitions,
tool_choice=(
"none" if force_final else "required"
@@ -1537,6 +1646,12 @@ class RouterRuntime:
else None
),
)
except RouterModelCallError as exc:
return RouterOutcome(
status="agent_unavailable",
message=exc.user_message(),
tool_trace=tool_trace,
)
except Exception:
return RouterOutcome(
status="agent_unavailable",
@@ -2271,8 +2386,12 @@ class RouterRuntime:
role = item.get("role")
if role not in {"user", "assistant"}:
continue
content = item.get("content", "")
normalized.append(LLMMessage(role=role, content=str(content or "")))
content = str(item.get("content", "") or "")
if role == "assistant" and content.startswith(
_ROUTER_MODEL_FAILURE_MESSAGE_PREFIX
):
continue
normalized.append(LLMMessage(role=role, content=content))
route_context = json.dumps(
{
@@ -2338,8 +2457,13 @@ class RouterRuntime:
role = item.get("role")
if role not in {"user", "assistant"}:
continue
content = str(item.get("content", "") or "")
if role == "assistant" and content.startswith(
_ROUTER_MODEL_FAILURE_MESSAGE_PREFIX
):
continue
normalized.append(
LLMMessage(role=role, content=str(item.get("content", "") or ""))
LLMMessage(role=role, content=content)
)
context = f"\nCurrent session_id={session_id!r}; current job_id={job_id!r}."
if goal_ledger is not None:
@@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import json
from types import SimpleNamespace
from typing import Any
import pytest
@@ -269,3 +270,203 @@ def test_router_response_language_does_not_rewrite_business_query() -> None:
assert messages[-1].content == query
assert "Respond to the user in English." in messages[0].content
def test_router_generated_model_failure_is_not_replayed_as_history() -> None:
failure_message = (
"Router 调用 DeepSeek 失败(HTTP 402,模型 deepseek-v4-flash,"
"已尝试 3 次,错误码 invalid_request_error):Insufficient Balance"
)
history = [
{"role": "user", "content": "hello"},
{"role": "assistant", "content": failure_message},
{"role": "assistant", "content": "正常历史回复"},
]
decision = RouteDecision(goal="general", needs=[], subject="hello")
route_messages = RouterRuntime._route_messages(
query="hello",
history=history,
session_id="router-history-session",
job_id="job-router-history",
goal_ledger=None,
)
response_messages = RouterRuntime._initial_messages(
query="hello",
history=history,
session_id="router-history-session",
job_id="job-router-history",
decision=decision,
goal_ledger=None,
)
for messages in (route_messages, response_messages):
contents = [message.content for message in messages]
assert failure_message not in contents
assert "正常历史回复" in contents
def test_route_decision_retries_transient_model_failures(
monkeypatch: pytest.MonkeyPatch,
) -> None:
class TransientPaymentGatewayFailure(RuntimeError):
status_code = 402
class TransientLLM:
calls = 0
async def acall(self, *_: Any, **__: Any) -> Any:
self.calls += 1
if self.calls < 3:
raise TransientPaymentGatewayFailure("temporary billing gateway failure")
return SimpleNamespace(
content=(
'{"goal":"new_analysis","needs":["analysis_intent"],'
'"subject":"执行数据准备","analysis_request_kind":"execution"}'
)
)
async def no_sleep(_: float) -> None:
return None
llm = TransientLLM()
runtime = RouterRuntime(llm, _PassThroughTools(), max_iterations=5) # type: ignore[arg-type]
monkeypatch.setattr(asyncio, "sleep", no_sleep)
decision, calls, trace = asyncio.run(
runtime._decide_route(
query="执行数据准备",
history=[],
session_id="retry-session",
job_id="job-retry",
goal_ledger=None,
)
)
assert llm.calls == 3
assert calls == 1
assert trace == []
assert decision is not None
assert decision.goal == "new_analysis"
def test_route_decision_does_not_retry_authentication_failure() -> None:
class AuthenticationFailure(RuntimeError):
status_code = 401
class UnauthorizedLLM:
calls = 0
async def acall(self, *_: Any, **__: Any) -> Any:
self.calls += 1
raise AuthenticationFailure("unauthorized")
llm = UnauthorizedLLM()
runtime = RouterRuntime(llm, _PassThroughTools(), max_iterations=5) # type: ignore[arg-type]
decision, calls, trace = asyncio.run(
runtime._decide_route(
query="执行数据准备",
history=[],
session_id="auth-session",
job_id="job-auth",
goal_ledger=None,
)
)
assert llm.calls == 1
assert decision is None
assert calls == 1
assert trace == []
def test_route_failure_exposes_sanitized_upstream_reason(
monkeypatch: pytest.MonkeyPatch,
) -> None:
class StructuredPaymentGatewayFailure(RuntimeError):
status_code = 402
request_id = "req-router-402"
body = {
"error": {
"code": "request_policy_rejected",
"message": (
"the concrete upstream reason; "
"Authorization: Bearer must-not-leak"
),
"api_key": "must-not-leak",
}
}
class FailingLLM:
_default_model = "deepseek-v4-flash"
calls = 0
async def acall(self, *_: Any, **__: Any) -> Any:
self.calls += 1
raise StructuredPaymentGatewayFailure("generic SDK error")
async def no_sleep(_: float) -> None:
return None
llm = FailingLLM()
runtime = RouterRuntime(llm, _PassThroughTools(), max_iterations=5) # type: ignore[arg-type]
monkeypatch.setattr(asyncio, "sleep", no_sleep)
outcome = asyncio.run(
runtime.run(
query="执行数据准备",
history=[],
session_id="failure-detail-session",
job_id="job-failure-detail",
goal_ledger=None,
)
)
assert llm.calls == 3
assert outcome.status == "agent_unavailable"
assert "HTTP 402" in outcome.message
assert "request_policy_rejected" in outcome.message
assert "the concrete upstream reason" in outcome.message
assert "deepseek-v4-flash" in outcome.message
assert "3 次" in outcome.message
assert "req-router-402" in outcome.message
assert "must-not-leak" not in outcome.message
def test_route_failure_reads_flat_openai_error_body(
monkeypatch: pytest.MonkeyPatch,
) -> None:
class FlatOpenAIError(RuntimeError):
status_code = 402
body = {
"code": "invalid_request_error",
"message": "Insufficient Balance",
"type": "unknown_error",
}
class FailingLLM:
_default_model = "deepseek-v4-flash"
async def acall(self, *_: Any, **__: Any) -> Any:
raise FlatOpenAIError("serialized SDK fallback")
async def no_sleep(_: float) -> None:
return None
runtime = RouterRuntime(FailingLLM(), _PassThroughTools(), max_iterations=5) # type: ignore[arg-type]
monkeypatch.setattr(asyncio, "sleep", no_sleep)
outcome = asyncio.run(
runtime.run(
query="执行数据准备",
history=[],
session_id="flat-openai-error-session",
job_id="job-flat-openai-error",
goal_ledger=None,
)
)
assert outcome.status == "agent_unavailable"
assert "错误码 invalid_request_error" in outcome.message
assert outcome.message.endswith("Insufficient Balance")
assert "serialized SDK fallback" not in outcome.message
@@ -0,0 +1,78 @@
from __future__ import annotations
import os
from pathlib import Path
import subprocess
import sys
PROJECT_ROOT = Path(__file__).resolve().parents[2]
def test_web_import_keeps_plantai_new_api_bundle_when_rag_import_sets_deepseek_key(
tmp_path: Path,
) -> None:
rag_root = tmp_path / "rag"
module_dir = rag_root / "src" / "rag" / "unified"
module_dir.mkdir(parents=True)
for package_dir in (rag_root / "src", rag_root / "src" / "rag", module_dir):
(package_dir / "__init__.py").write_text("", encoding="utf-8")
(module_dir / "bbav5_methodology_config.py").write_text(
(
"import os\n"
"os.environ.setdefault('DEEPSEEK_API_KEY', 'external-stale-key')\n"
"def build_bbav5_methodology_config(*args, **kwargs):\n"
" return {}\n"
),
encoding="utf-8",
)
runtime_root = tmp_path / "runtime"
runtime_root.mkdir()
env = os.environ.copy()
for name in (
"DEEPSEEK_API_KEY",
"deepseek_api_key",
"NEW_API_KEY",
"NEW_API_BASE_URL",
"NEW_API_MODEL",
):
env.pop(name, None)
env.update(
{
"NEW_API_KEY": "plantai-current-key",
"NEW_API_BASE_URL": "https://api.deepseek.com",
"NEW_API_MODEL": "deepseek-v4-flash",
"PLANTAI_RAG_ROOT": str(rag_root),
"GF_MAS_ROOT": str(tmp_path / "missing-gf-mas"),
"BBAV5_PROJECT_ROOT": str(runtime_root),
"PLANTAI_PROJECT_ROOT": str(runtime_root),
}
)
result = subprocess.run(
[
sys.executable,
"-c",
(
"import os\n"
"from pathlib import Path\n"
f"path = Path({str(PROJECT_ROOT / 'apps/web/vue3/main.py')!r})\n"
"source = path.read_text(encoding='utf-8')\n"
"marker = 'try:\\n from gfa_core.mas.agents.interpretation_evidence'\n"
"prefix, separator, _ = source.partition(marker)\n"
"assert separator\n"
"namespace = {'__file__': str(path), '__name__': 'web_import_prefix'}\n"
"exec(compile(prefix, str(path), 'exec'), namespace)\n"
"assert 'DEEPSEEK_API_KEY' not in os.environ\n"
),
],
cwd=PROJECT_ROOT,
env=env,
capture_output=True,
text=True,
timeout=120,
check=False,
)
assert result.returncode == 0, result.stdout + result.stderr