fix: stabilize Router model error handling
This commit is contained in:
potaser
@@ -1,5 +1,7 @@
|
||||
import asyncio
|
||||
import json
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from collections import Counter
|
||||
@@ -20,6 +22,64 @@ from gfa_core.mas.workflow_catalog import (
|
||||
from gfa_core.mas.workflow_models import CurrentRecommendation, GoalLedger
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_ROUTER_ERROR_TEXT_LIMIT = 1000
|
||||
_ROUTER_MODEL_FAILURE_MESSAGE_PREFIX = "Router 调用 DeepSeek 失败（"
|
||||
_ROUTER_CREDENTIAL_PATTERN = re.compile(
|
||||
r"(?i)\b(api[_-]?key|authorization|(?:access|refresh)[_-]?token|token|secret)"
|
||||
r"\b\s*[:=]\s*[^,;}\]\r\n]+"
|
||||
)
|
||||
|
||||
|
||||
def _sanitize_router_error_text(value: Any) -> str:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
return _ROUTER_CREDENTIAL_PATTERN.sub(r"\1=<REDACTED>", text)[
|
||||
:_ROUTER_ERROR_TEXT_LIMIT
|
||||
]
|
||||
|
||||
|
||||
class RouterModelCallError(RuntimeError):
|
||||
"""Allow-listed details from a failed Router model request."""
|
||||
|
||||
def __init__(self, cause: Exception, *, attempts: int, model: str) -> None:
|
||||
self.status_code = getattr(cause, "status_code", None)
|
||||
self.attempts = attempts
|
||||
self.model = _sanitize_router_error_text(model) or "unknown"
|
||||
self.request_id = _sanitize_router_error_text(
|
||||
getattr(cause, "request_id", "")
|
||||
)
|
||||
|
||||
body = getattr(cause, "body", None)
|
||||
error: Mapping[str, Any] = {}
|
||||
if isinstance(body, Mapping):
|
||||
nested_error = body.get("error")
|
||||
error = nested_error if isinstance(nested_error, Mapping) else body
|
||||
self.error_code = _sanitize_router_error_text(error.get("code"))
|
||||
self.upstream_message = _sanitize_router_error_text(error.get("message"))
|
||||
if not self.upstream_message:
|
||||
self.upstream_message = _sanitize_router_error_text(str(cause))
|
||||
if not self.upstream_message:
|
||||
self.upstream_message = type(cause).__name__
|
||||
|
||||
super().__init__(self.user_message())
|
||||
|
||||
def user_message(self) -> str:
|
||||
details = []
|
||||
if self.status_code is not None:
|
||||
details.append(f"HTTP {self.status_code}")
|
||||
details.append(f"模型 {self.model}")
|
||||
details.append(f"已尝试 {self.attempts} 次")
|
||||
if self.error_code:
|
||||
details.append(f"错误码 {self.error_code}")
|
||||
if self.request_id:
|
||||
details.append(f"Request ID {self.request_id}")
|
||||
return f"Router 调用 DeepSeek 失败（{'，'.join(details)}）：{self.upstream_message}"
|
||||
|
||||
|
||||
InteractionKind = Literal[
|
||||
"supplement",
|
||||
"correction",
|
||||
@@ -561,6 +621,7 @@ class RouterRuntime:
|
||||
) -> None:
|
||||
self.llm_manager = llm_manager
|
||||
self.tools = tools
|
||||
self._last_model_call_error: RouterModelCallError | None = None
|
||||
if max_iterations is None:
|
||||
raw_budget = os.environ.get("PLANTAI_MAIN_AGENT_MAX_TOOL_CALLS", "10")
|
||||
try:
|
||||
@@ -577,6 +638,50 @@ class RouterRuntime:
|
||||
else:
|
||||
self.max_iterations = min(max(0, max_iterations), 5)
|
||||
|
||||
async def _call_router_model(
|
||||
self,
|
||||
messages: list[LLMMessage],
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
"""Call the Router model with bounded retries for transient failures."""
|
||||
max_attempts = 3
|
||||
retryable_statuses = {402, 408, 409, 429, 500, 502, 503, 504}
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
return await self.llm_manager.acall(
|
||||
messages,
|
||||
provider="deepseek",
|
||||
allow_fallback=False,
|
||||
**kwargs,
|
||||
)
|
||||
except Exception as exc:
|
||||
status = getattr(exc, "status_code", None)
|
||||
retryable = status is None or status in retryable_statuses
|
||||
logger.warning(
|
||||
"Router model call failed: attempt=%s/%s exception=%s status=%s retryable=%s",
|
||||
attempt,
|
||||
max_attempts,
|
||||
type(exc).__name__,
|
||||
status,
|
||||
retryable,
|
||||
)
|
||||
if not retryable or attempt >= max_attempts:
|
||||
failure = RouterModelCallError(
|
||||
exc,
|
||||
attempts=attempt,
|
||||
model=str(
|
||||
getattr(
|
||||
self.llm_manager,
|
||||
"_default_model",
|
||||
"deepseek-v4-flash",
|
||||
)
|
||||
or "deepseek-v4-flash"
|
||||
),
|
||||
)
|
||||
logger.error("Router model unavailable: %s", failure.user_message())
|
||||
raise failure from exc
|
||||
await asyncio.sleep(0.5 * (2 ** (attempt - 1)))
|
||||
|
||||
@staticmethod
|
||||
def _action_for_route(
|
||||
decision: RouteDecision,
|
||||
@@ -1126,15 +1231,16 @@ class RouterRuntime:
|
||||
calls = 0
|
||||
for attempt in range(min(2, self.max_iterations)):
|
||||
try:
|
||||
response = await self.llm_manager.acall(
|
||||
response = await self._call_router_model(
|
||||
route_messages,
|
||||
provider="deepseek",
|
||||
allow_fallback=False,
|
||||
tools=[],
|
||||
tool_choice="none",
|
||||
temperature=0,
|
||||
response_format={"type": "json_object"},
|
||||
)
|
||||
except RouterModelCallError as exc:
|
||||
self._last_model_call_error = exc
|
||||
return None, calls + 1, []
|
||||
except Exception:
|
||||
return None, calls + 1, []
|
||||
calls += 1
|
||||
@@ -1181,6 +1287,7 @@ class RouterRuntime:
|
||||
goal_ledger: GoalLedger | None = None,
|
||||
language: str = "zh",
|
||||
) -> RouterOutcome:
|
||||
self._last_model_call_error = None
|
||||
ui_action = self._config_dialog_ui_action(query)
|
||||
if ui_action is not None:
|
||||
return RouterOutcome(
|
||||
@@ -1276,7 +1383,11 @@ class RouterRuntime:
|
||||
if not decision_trace:
|
||||
return RouterOutcome(
|
||||
status="agent_unavailable",
|
||||
message="The Router model is currently unavailable.",
|
||||
message=(
|
||||
self._last_model_call_error.user_message()
|
||||
if self._last_model_call_error is not None
|
||||
else "The Router model is currently unavailable."
|
||||
),
|
||||
)
|
||||
if decision_calls < 2:
|
||||
return RouterOutcome(
|
||||
@@ -1522,10 +1633,8 @@ class RouterRuntime:
|
||||
]
|
||||
force_final = not remaining_needs or recognized_intent is not None
|
||||
try:
|
||||
response = await self.llm_manager.acall(
|
||||
response = await self._call_router_model(
|
||||
messages,
|
||||
provider="deepseek",
|
||||
allow_fallback=False,
|
||||
tools=[] if force_final else available_definitions,
|
||||
tool_choice=(
|
||||
"none" if force_final else "required"
|
||||
@@ -1537,6 +1646,12 @@ class RouterRuntime:
|
||||
else None
|
||||
),
|
||||
)
|
||||
except RouterModelCallError as exc:
|
||||
return RouterOutcome(
|
||||
status="agent_unavailable",
|
||||
message=exc.user_message(),
|
||||
tool_trace=tool_trace,
|
||||
)
|
||||
except Exception:
|
||||
return RouterOutcome(
|
||||
status="agent_unavailable",
|
||||
@@ -2271,8 +2386,12 @@ class RouterRuntime:
|
||||
role = item.get("role")
|
||||
if role not in {"user", "assistant"}:
|
||||
continue
|
||||
content = item.get("content", "")
|
||||
normalized.append(LLMMessage(role=role, content=str(content or "")))
|
||||
content = str(item.get("content", "") or "")
|
||||
if role == "assistant" and content.startswith(
|
||||
_ROUTER_MODEL_FAILURE_MESSAGE_PREFIX
|
||||
):
|
||||
continue
|
||||
normalized.append(LLMMessage(role=role, content=content))
|
||||
|
||||
route_context = json.dumps(
|
||||
{
|
||||
@@ -2338,8 +2457,13 @@ class RouterRuntime:
|
||||
role = item.get("role")
|
||||
if role not in {"user", "assistant"}:
|
||||
continue
|
||||
content = str(item.get("content", "") or "")
|
||||
if role == "assistant" and content.startswith(
|
||||
_ROUTER_MODEL_FAILURE_MESSAGE_PREFIX
|
||||
):
|
||||
continue
|
||||
normalized.append(
|
||||
LLMMessage(role=role, content=str(item.get("content", "") or ""))
|
||||
LLMMessage(role=role, content=content)
|
||||
)
|
||||
context = f"\nCurrent session_id={session_id!r}; current job_id={job_id!r}."
|
||||
if goal_ledger is not None:
|
||||
|
||||
Reference in New Issue
Block a user