《互联网产品的 AI 架构·推理性能与成本工程》12.3 一次成本失控的复盘

// file: src/perf/incident.ts
// tier: L1
// verify: node --test test/perf/incident.test.ts

import { AiError } from "../core/errors.ts";

export interface IncidentPhase {
  minute: number; label: string; requests: number;
  attemptsPerRequest: number;
  /** 每次尝试的平均成本，整数微元。 */
  microsPerAttempt: number;
}

export interface IncidentBaseline { attemptsPerRequest: number; microsPerAttempt: number; }

export interface IncidentPoint {
  minute: number; label: string; requests: number; attempts: number;
  costMicros: number; cumulativeMicros: number; budgetUtilization: number; overBudget: boolean;
}

export interface IncidentAnalysis {
  points: IncidentPoint[];
  totalRequests: number; totalAttempts: number; totalMicros: number; baselineMicros: number;
  /** 总成本相对基线的倍数，拆成尝试放大与单价放大两个因子。 */
  multiplier: number; attemptsMultiplier: number; priceMultiplier: number;
  peakMinute: number; peakMicros: number;
  budgetBreachMinute: number | null; budget: number;
}

function fail(message: string): never {
  throw new AiError("invalid-request", `incident: ${message}`);
}

function finite(value: number, field: string, min = 0): number {
  if (!Number.isFinite(value) || value < min) fail(`${field} must be a finite number >= ${min}`);
  return value;
}

function integer(value: number, field: string, min = 0): number {
  if (!Number.isInteger(value) || value < min) fail(`${field} must be an integer >= ${min}`);
  return value;
}

/**
 * 把事故时间线折算成成本曲线。成本倍数拆成“尝试放大”和“单价放大”，
 * 是为了回答复盘中最常见的问题：到底是重试多了，还是模型贵了。
 */
export function analyzeIncident(
  phases: readonly IncidentPhase[], baseline: IncidentBaseline, budgetMicros: number,
): IncidentAnalysis {
  if (phases.length === 0) fail("phases must not be empty");
  finite(budgetMicros, "budgetMicros");
  finite(baseline.attemptsPerRequest, "baseline.attemptsPerRequest");
  finite(baseline.microsPerAttempt, "baseline.microsPerAttempt");
  if (baseline.attemptsPerRequest <= 0 || baseline.microsPerAttempt <= 0) fail("baseline values must be > 0");

  const ordered = [...phases].sort((left, right) => left.minute - right.minute);
  const points: IncidentPoint[] = [];
  let cumulative = 0; let totalRequests = 0; let totalAttempts = 0; let totalMicros = 0;
  let peakMinute = ordered[0].minute; let peakMicros = -1; let budgetBreachMinute: number | null = null;

  for (const phase of ordered) {
    integer(phase.minute, "phase.minute");
    integer(phase.requests, "phase.requests");
    finite(phase.attemptsPerRequest, "phase.attemptsPerRequest");
    integer(phase.microsPerAttempt, "phase.microsPerAttempt");
    const attempts = Math.round(phase.requests * phase.attemptsPerRequest);
    const costMicros = attempts * phase.microsPerAttempt;
    cumulative += costMicros;
    totalRequests += phase.requests;
    totalAttempts += attempts;
    totalMicros += costMicros;
    if (costMicros > peakMicros) { peakMicros = costMicros; peakMinute = phase.minute; }
    if (budgetBreachMinute === null && cumulative > budgetMicros) budgetBreachMinute = phase.minute;
    points.push({
      minute: phase.minute, label: phase.label, requests: phase.requests, attempts, costMicros,
      cumulativeMicros: cumulative, budgetUtilization: cumulative / budgetMicros,
      overBudget: cumulative > budgetMicros,
    });
  }

  const baselineAttempts = totalRequests * baseline.attemptsPerRequest;
  const baselineMicros = baselineAttempts * baseline.microsPerAttempt;
  const averagePrice = totalAttempts === 0 ? 0 : totalMicros / totalAttempts;
  return {
    points, totalRequests, totalAttempts, totalMicros, baselineMicros,
    multiplier: totalMicros / baselineMicros,
    attemptsMultiplier: totalAttempts / baselineAttempts,
    priceMultiplier: averagePrice / baseline.microsPerAttempt,
    peakMinute, peakMicros, budgetBreachMinute, budget: budgetMicros,
  };
}

export interface PostmortemAction {
  action: string; owner: string; due: string; type: "prevent" | "detect" | "mitigate";
}

export interface IncidentPostmortem {
  title: string; severity: "sev1" | "sev2" | "sev3";
  detectedAt: string; resolvedAt: string;
  impact: { requests: number; costMicros: number; costMultiplier: number; userVisible: string; };
  timeline: { at: string; event: string; evidence: string; }[];
  rootCause: string; contributing: string[]; detectionGap: string; responseGap: string;
  actions: PostmortemAction[];
}

/** 复盘不是自由作文：缺时间线、缺根因、缺防复发动作，都不算完成。 */
export function validatePostmortem(postmortem: IncidentPostmortem): string[] {
  const issues: string[] = [];
  if (postmortem.title.trim() === "") issues.push("title is empty");
  if (postmortem.rootCause.trim() === "") issues.push("rootCause is empty");
  if (postmortem.detectionGap.trim() === "") issues.push("detectionGap is empty");
  if (postmortem.responseGap.trim() === "") issues.push("responseGap is empty");
  if (postmortem.contributing.length === 0) issues.push("contributing is empty");
  if (postmortem.timeline.length < 3) issues.push("timeline needs at least 3 entries");
  if (postmortem.actions.length < 3) issues.push("actions need at least 3 entries");
  if (!postmortem.actions.some((action) => action.type === "prevent")) {
    issues.push("at least one prevent action is required");
  }
  if (postmortem.impact.costMultiplier < 1) issues.push("costMultiplier must be >= 1");
  if (postmortem.impact.userVisible.trim() === "") issues.push("impact.userVisible is empty");
  if (postmortem.detectedAt > postmortem.resolvedAt) issues.push("detectedAt is after resolvedAt");
  postmortem.timeline.forEach((entry, index) => {
    if (entry.event.trim() === "" || entry.evidence.trim() === "") {
      issues.push(`timeline[${index}] needs event and evidence`);
    }
    if (index > 0 && postmortem.timeline[index - 1].at > entry.at) {
      issues.push(`timeline[${index}] is out of order`);
    }
  });
  postmortem.actions.forEach((action, index) => {
    if (action.action.trim() === "" || action.owner.trim() === "" || action.due.trim() === "") {
      issues.push(`actions[${index}] needs action, owner and due`);
    }
  });
  return issues;
}

export function buildPostmortem(postmortem: IncidentPostmortem): IncidentPostmortem {
  const issues = validatePostmortem(postmortem);
  if (issues.length > 0) throw new AiError("invalid-request", `postmortem: ${issues.join("; ")}`);
  return postmortem;
}