Agent Error Recovery

Design fault-tolerant agent systems that recover gracefully from failures.

When to Use

• Building production-grade agent systems

• Agents need to handle API failures

• Implementing autonomous error recovery

• Designing resilient multi-agent workflows

• Setting up monitoring and alerting

Error Classification

enum ErrorCategory { // Transient - retry likely to succeed RATE_LIMIT = 'rate_limit', TIMEOUT = 'timeout', NETWORK = 'network', SERVICE_UNAVAILABLE = 'service_unavailable',

// Recoverable - different approach may work INVALID_INPUT = 'invalid_input', CONTEXT_OVERFLOW = 'context_overflow', TOOL_FAILURE = 'tool_failure',

// Terminal - cannot proceed AUTHENTICATION = 'authentication', AUTHORIZATION = 'authorization', NOT_FOUND = 'not_found', VALIDATION = 'validation',

// Unknown UNKNOWN = 'unknown' }

interface AgentError { category: ErrorCategory; code: string; message: string; retryable: boolean; context: Record<string, unknown>; timestamp: Date; stackTrace?: string; }

function classifyError(error: Error): AgentError { // Rate limits if (error.message.includes('429') || error.message.includes('rate limit')) { return { category: ErrorCategory.RATE_LIMIT, code: 'RATE_LIMITED', message: error.message, retryable: true, context: { waitTime: extractWaitTime(error) }, timestamp: new Date() }; }

// Timeouts if (error.message.includes('timeout') || error.message.includes('ETIMEDOUT')) { return { category: ErrorCategory.TIMEOUT, code: 'TIMEOUT', message: error.message, retryable: true, context: {}, timestamp: new Date() }; }

// Context overflow if (error.message.includes('context length') || error.message.includes('too long')) { return { category: ErrorCategory.CONTEXT_OVERFLOW, code: 'CONTEXT_OVERFLOW', message: error.message, retryable: true, // Can retry with truncated context context: {}, timestamp: new Date() }; }

// Default return { category: ErrorCategory.UNKNOWN, code: 'UNKNOWN', message: error.message, retryable: false, context: {}, timestamp: new Date(), stackTrace: error.stack }; }

Recovery Strategies

Strategy 1: Retry with Backoff

interface RetryConfig { maxAttempts: number; initialDelayMs: number; maxDelayMs: number; backoffMultiplier: number; jitterMs: number; }

async function retryWithBackoff<T>( operation: () => Promise<T>, config: RetryConfig ): Promise<T> { let lastError: Error; let delay = config.initialDelayMs;

for (let attempt = 1; attempt <= config.maxAttempts; attempt++) { try { return await operation(); } catch (error) { lastError = error as Error; const classified = classifyError(lastError);

  // Don't retry non-retryable errors
  if (!classified.retryable) {
    throw lastError;
  }

  // Last attempt - throw
  if (attempt === config.maxAttempts) {
    throw lastError;
  }

  // Calculate delay with jitter
  const jitter = Math.random() * config.jitterMs;
  const waitTime = Math.min(delay + jitter, config.maxDelayMs);

  console.log(`Attempt ${attempt} failed, retrying in ${waitTime}ms...`);
  await sleep(waitTime);

  // Increase delay for next attempt
  delay *= config.backoffMultiplier;
}

}

throw lastError!; }

Strategy 2: Circuit Breaker

enum CircuitState { CLOSED = 'closed', // Normal operation OPEN = 'open', // Failing, reject requests HALF_OPEN = 'half_open' // Testing if recovered }

class CircuitBreaker { private state = CircuitState.CLOSED; private failures = 0; private lastFailure?: Date; private successCount = 0;

constructor( private config: { failureThreshold: number; resetTimeoutMs: number; successThreshold: number; } ) {}

async execute<T>(operation: () => Promise<T>): Promise<T> { // Check if circuit should transition this.checkState();

if (this.state === CircuitState.OPEN) {
  throw new Error('Circuit breaker is OPEN');
}

try {
  const result = await operation();
  this.onSuccess();
  return result;
} catch (error) {
  this.onFailure();
  throw error;
}

}

private checkState(): void { if (this.state === CircuitState.OPEN) { const elapsed = Date.now() - this.lastFailure!.getTime(); if (elapsed >= this.config.resetTimeoutMs) { this.state = CircuitState.HALF_OPEN; this.successCount = 0; } } }

private onSuccess(): void { if (this.state === CircuitState.HALF_OPEN) { this.successCount++; if (this.successCount >= this.config.successThreshold) { this.state = CircuitState.CLOSED; this.failures = 0; } } else { this.failures = 0; } }

private onFailure(): void { this.failures++; this.lastFailure = new Date();

if (this.failures >= this.config.failureThreshold) {
  this.state = CircuitState.OPEN;
}

} }

Strategy 3: Fallback Chain

interface FallbackOption<T> { name: string; execute: () => Promise<T>; isApplicable: (error: AgentError) => boolean; }

async function executeWithFallbacks<T>( primary: () => Promise<T>, fallbacks: FallbackOption<T>[] ): Promise<T> { try { return await primary(); } catch (error) { const classified = classifyError(error as Error);

for (const fallback of fallbacks) {
  if (fallback.isApplicable(classified)) {
    console.log(`Primary failed, trying fallback: ${fallback.name}`);
    try {
      return await fallback.execute();
    } catch (fallbackError) {
      console.log(`Fallback ${fallback.name} also failed`);
      continue;
    }
  }
}

// All fallbacks failed
throw error;

} }

// Example usage const result = await executeWithFallbacks( () => callPrimaryAPI(), [ { name: 'backup_api', execute: () => callBackupAPI(), isApplicable: (e) => e.category === ErrorCategory.SERVICE_UNAVAILABLE }, { name: 'cached_response', execute: () => getCachedResponse(), isApplicable: (e) => e.category === ErrorCategory.TIMEOUT }, { name: 'simplified_request', execute: () => callWithReducedParams(), isApplicable: (e) => e.category === ErrorCategory.CONTEXT_OVERFLOW } ] );

Strategy 4: Self-Healing Agent

class SelfHealingAgent { async execute(task: Task): Promise<Result> { let attempt = 0; const maxAttempts = 3;

while (attempt < maxAttempts) {
  attempt++;

  try {
    return await this.runTask(task);
  } catch (error) {
    const classified = classifyError(error as Error);

    // Can we heal?
    const healingAction = this.determineHealingAction(classified);

    if (!healingAction) {
      throw error;
    }

    console.log(`Attempting self-healing: ${healingAction.description}`);

    // Execute healing
    await healingAction.execute();

    // Modify task if needed
    task = healingAction.modifyTask?.(task) || task;
  }
}

throw new Error('Max healing attempts exceeded');

}

private determineHealingAction(error: AgentError): HealingAction | null { switch (error.category) { case ErrorCategory.CONTEXT_OVERFLOW: return { description: 'Truncating context to fit limits', execute: async () => {}, modifyTask: (task) => ({ ...task, context: this.truncateContext(task.context) }) };

  case ErrorCategory.TOOL_FAILURE:
    return {
      description: 'Switching to alternative tool',
      execute: async () => {
        this.toolRouter.excludeTool(error.context.toolName as string);
      }
    };

  case ErrorCategory.RATE_LIMIT:
    return {
      description: `Waiting ${error.context.waitTime}ms for rate limit`,
      execute: async () => {
        await sleep(error.context.waitTime as number);
      }
    };

  default:
    return null;
}

} }

Error Recovery Workflow

┌─────────────────────────────────────────────────────────────┐ │ Error Occurs │ └─────────────────────────────────────────────────────────────┘ │ ▼ ┌─────────────────────────────────────────────────────────────┐ │ Classify Error │ └─────────────────────────────────────────────────────────────┘ │ ┌─────────────────┼─────────────────┐ ▼ ▼ ▼ ┌─────────┐ ┌─────────┐ ┌─────────┐ │Transient│ │Recoverable │Terminal │ └────┬────┘ └────┬────┘ └────┬────┘ │ │ │ ▼ ▼ ▼ ┌─────────┐ ┌─────────┐ ┌─────────┐ │ Retry │ │ Try │ │ Log & │ │ w/Backoff │ Fallback│ │ Alert │ └────┬────┘ └────┬────┘ └────┬────┘ │ │ │ ▼ ▼ ▼ ┌─────────────────────────────────────────┐ │ Success or Escalate │ └─────────────────────────────────────────┘

Monitoring and Alerting

interface ErrorMetrics { totalErrors: number; errorsByCategory: Map<ErrorCategory, number>; errorRate: number; // errors per minute recoveryRate: number; // successful recoveries mttr: number; // mean time to recover (ms) }

class ErrorMonitor { private errors: AgentError[] = []; private recoveries: { error: AgentError; recoveredAt: Date }[] = [];

recordError(error: AgentError): void { this.errors.push(error); this.checkAlerts(); }

recordRecovery(error: AgentError): void { this.recoveries.push({ error, recoveredAt: new Date() }); }

private checkAlerts(): void { const recentErrors = this.getRecentErrors(60000); // Last minute

// High error rate alert
if (recentErrors.length > 10) {
  this.sendAlert({
    severity: 'high',
    message: `High error rate: ${recentErrors.length} errors in last minute`,
    errors: recentErrors
  });
}

// Repeated same error alert
const errorCounts = new Map<string, number>();
for (const e of recentErrors) {
  const key = `${e.category}:${e.code}`;
  errorCounts.set(key, (errorCounts.get(key) || 0) + 1);
}

for (const [key, count] of errorCounts) {
  if (count >= 5) {
    this.sendAlert({
      severity: 'medium',
      message: `Repeated error: ${key} occurred ${count} times`,
      errors: recentErrors.filter(e => `${e.category}:${e.code}` === key)
    });
  }
}

}

getMetrics(): ErrorMetrics { const window = 5 * 60 * 1000; // 5 minutes const recent = this.getRecentErrors(window);

const byCategory = new Map<ErrorCategory, number>();
for (const e of recent) {
  byCategory.set(e.category, (byCategory.get(e.category) || 0) + 1);
}

const recentRecoveries = this.recoveries.filter(
  r => Date.now() - r.recoveredAt.getTime() < window
);

const recoveryTimes = recentRecoveries.map(
  r => r.recoveredAt.getTime() - r.error.timestamp.getTime()
);

return {
  totalErrors: recent.length,
  errorsByCategory: byCategory,
  errorRate: recent.length / (window / 60000),
  recoveryRate: recentRecoveries.length / Math.max(recent.length, 1),
  mttr: recoveryTimes.length > 0
    ? recoveryTimes.reduce((a, b) => a + b, 0) / recoveryTimes.length
    : 0
};

} }

Best Practices

• Classify all errors - Know what you're dealing with

• Don't retry everything - Some errors won't recover

• Use exponential backoff - Avoid hammering failing services

• Set circuit breakers - Protect downstream systems

• Log everything - Debugging is hard without logs

• Have fallbacks - Always have a Plan B

• Alert on patterns - Single errors may be noise, patterns matter

• Test failure scenarios - Chaos engineering