feat(agent): implement LLM call timeout and transient error handling

- Introduced a configurable LLM call timeout with a default value, allowing for better control over LLM call durations.
- Added logic to retry transient errors (e.g., timeouts, rate limits) up to a specified maximum number of retries, improving robustness in error handling.
- Implemented parameter casting for tool arguments to ensure correct types are used, addressing common LLM quirks.
- Enhanced tool execution error messages with guidance for retrying with different approaches.
- Added validation for configuration values to prevent runtime errors.
This commit is contained in:
wizardchen
2026-03-19 20:39:38 +08:00
committed by lyingbug
parent 00ac14aabb
commit e936e0b347
9 changed files with 408 additions and 49 deletions
+63 -3
View File
@@ -21,11 +21,45 @@ import (
)
const (
// llmPerCallTimeout is the maximum time allowed for a single LLM call (stream initiation + full response).
// defaultLLMCallTimeout is the default maximum time allowed for a single LLM call.
// This prevents a single slow call from consuming the entire pipeline deadline.
llmPerCallTimeout = 120 * time.Second
// Can be overridden via AgentConfig.LLMCallTimeout.
defaultLLMCallTimeout = 120 * time.Second
// maxLLMRetries is the maximum number of retries for transient LLM errors.
maxLLMRetries = 2
)
// transientErrorMarkers are substrings that indicate a transient (retryable) error.
var transientErrorMarkers = []string{
"429", "rate limit",
"500", "502", "503", "504",
"overloaded", "timeout", "timed out",
"connection", "server error", "temporarily unavailable",
}
// isTransientError checks whether an error is likely transient and worth retrying.
func isTransientError(err error) bool {
if err == nil {
return false
}
errStr := strings.ToLower(err.Error())
for _, marker := range transientErrorMarkers {
if strings.Contains(errStr, marker) {
return true
}
}
return false
}
// getLLMCallTimeout returns the configured LLM call timeout, falling back to default.
func (e *AgentEngine) getLLMCallTimeout() time.Duration {
if e.config.LLMCallTimeout > 0 {
return time.Duration(e.config.LLMCallTimeout) * time.Second
}
return defaultLLMCallTimeout
}
// generateEventID generates a unique event ID with type suffix for better traceability
func generateEventID(suffix string) string {
return fmt.Sprintf("%s-%s", uuid.New().String()[:8], suffix)
@@ -305,6 +339,20 @@ func (e *AgentEngine) executeLoop(
"tool_cnt": len(tools),
})
response, err := e.streamThinkingToEventBus(ctx, messages, tools, state.CurrentRound, sessionID)
if err != nil && isTransientError(err) {
// Retry transient errors (timeout, rate limit, server errors) up to maxLLMRetries times
for retry := 1; retry <= maxLLMRetries; retry++ {
retryDelay := time.Duration(retry) * time.Second
logger.Warnf(ctx, "[Agent][Round-%d] LLM transient error (attempt %d/%d), retrying in %v: %v",
state.CurrentRound+1, retry, maxLLMRetries, retryDelay, err)
time.Sleep(retryDelay)
response, err = e.streamThinkingToEventBus(ctx, messages, tools, state.CurrentRound, sessionID)
if err == nil || !isTransientError(err) {
break
}
}
}
if err != nil {
logger.Errorf(ctx, "[Agent][Round-%d] LLM call failed: %v", state.CurrentRound+1, err)
common.PipelineError(ctx, "Agent", "think_failed", map[string]interface{}{
@@ -450,6 +498,18 @@ func (e *AgentEngine) executeLoop(
if err := json.Unmarshal([]byte(tc.Function.Arguments), &args); err != nil {
logger.Errorf(ctx, "[Agent][Round-%d][Tool-%d/%d] Failed to parse tool arguments: %v",
state.CurrentRound+1, i+1, len(response.ToolCalls), err)
// Record parse failure as a tool result so the LLM can see the error and adapt
parseFailCall := types.ToolCall{
ID: tc.ID,
Name: tc.Function.Name,
Args: map[string]any{"_raw": tc.Function.Arguments},
Result: &types.ToolResult{
Success: false,
Error: fmt.Sprintf("Failed to parse tool arguments: %v", err) +
"\n\n[Analyze the error above and try a different approach.]",
},
}
step.ToolCalls = append(step.ToolCalls, parseFailCall)
continue
}
@@ -790,7 +850,7 @@ func (e *AgentEngine) streamLLMToEventBus(
// guaranteed time window even when the parent context's deadline is almost
// exhausted after previous iterations. The shorter of the two deadlines wins,
// so the parent pipeline timeout is still respected.
llmCtx, llmCancel := context.WithTimeout(ctx, llmPerCallTimeout)
llmCtx, llmCancel := context.WithTimeout(ctx, e.getLLMCallTimeout())
defer llmCancel()
stream, err := e.chatModel.ChatStream(llmCtx, messages, opts)