《互联网产品的 AI 架构·推理性能与成本工程》13.1 推理成本基线

// file: src/perf/baseline.ts
// tier: L1
// verify: node --test test/perf/baseline.test.ts

import { AiError } from "../core/errors.ts";
import { percentile } from "../observe/baseline.ts";

export type BaselineOutcome = "ok" | "cache-hit" | "degraded" | "error";
export type ModelTier = "frontier" | "standard" | "economy";

/** 一条结算记录，把第 1、8、9、11 章关心的字段放在同一行。 */
export interface BaselineRecord {
  requestId: string;
  capability: string;
  tier: ModelTier;
  outcome: BaselineOutcome;
  ttftMs: number;
  e2eMs: number;
  inputTokens: number;
  outputTokens: number;
  costMicros: number;
  /** 输入 token 中命中缓存的比例，0 到 1。 */
  cachedInputShare: number;
  /** 第 11 章的评测分，0 到 1；失败请求记 0。 */
  qualityScore: number;
}

/** 阈值是基线报告的一部分，必须和样本一起版本化。 */
export interface BaselineTargets {
  minSamples: number;
  maxErrorRate: number;
  maxDegradeRate: number;
  minCacheHitRate: number;
  maxP95E2eMs: number;
  maxAvgCostMicros: number;
  minQualityMean: number;
}

export const DEFAULT_TARGETS: BaselineTargets = {
  minSamples: 200,
  maxErrorRate: 0.02,
  maxDegradeRate: 0.05,
  minCacheHitRate: 0.15,
  maxP95E2eMs: 900,
  maxAvgCostMicros: 130,
  minQualityMean: 0.8,
};

export interface BaselineSlice {
  samples: number;
  ok: number;
  cacheHits: number;
  degraded: number;
  errors: number;
  successRate: number;
  errorRate: number;
  degradedRate: number;
  cacheHitRate: number;
  ttftP95Ms: number;
  e2eP95Ms: number;
  avgCostMicros: number;
  totalCostMicros: number;
  avgInputTokens: number;
  avgOutputTokens: number;
  avgCachedInputShare: number;
  qualityMean: number;
}

export type FindingSeverity = "blocker" | "warning";

export interface BaselineFinding {
  id: string;
  severity: FindingSeverity;
  metric: string;
  observed: string;
  target: string;
  action: string;
}

export interface BaselineReport {
  overall: BaselineSlice;
  byCapability: Record<string, BaselineSlice>;
  byTier: Record<string, BaselineSlice>;
  findings: BaselineFinding[];
  verdict: "pass" | "conditional" | "fail";
  targets: BaselineTargets;
}

const OUTCOMES = new Set<string>(["ok", "cache-hit", "degraded", "error"]);
const TIERS = new Set<string>(["frontier", "standard", "economy"]);

function fail(message: string): never {
  throw new AiError("invalid-request", `perf-baseline: ${message}`);
}

function ratio(numerator: number, denominator: number): number {
  return denominator === 0 ? 0 : numerator / denominator;
}

function mean(values: readonly number[]): number {
  return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length;
}

function validate(record: BaselineRecord): void {
  if (record.requestId.trim() === "" || record.capability.trim() === "") {
    fail("requestId 与 capability 不能为空");
  }
  if (!OUTCOMES.has(record.outcome)) fail(`${record.requestId}: 未知 outcome ${String(record.outcome)}`);
  if (!TIERS.has(record.tier)) fail(`${record.requestId}: 未知 tier ${String(record.tier)}`);
  for (const field of ["ttftMs", "e2eMs", "inputTokens", "outputTokens", "costMicros"] as const) {
    const value = record[field];
    if (!Number.isFinite(value) || value < 0) fail(`${record.requestId}: ${field} 必须是非负有限数`);
  }
  if (!Number.isInteger(record.costMicros)) fail(`${record.requestId}: costMicros 必须是整数微元`);
  if (record.e2eMs < record.ttftMs) fail(`${record.requestId}: e2eMs 不能小于 ttftMs`);
  for (const field of ["cachedInputShare", "qualityScore"] as const) {
    const value = record[field];
    if (!Number.isFinite(value) || value < 0 || value > 1) fail(`${record.requestId}: ${field} 必须落在 [0, 1]`);
  }
}

/** 一个切片就是同一口径下的一组样本；整体、能力、模型档位都复用它。 */
export function summarizeSlice(records: readonly BaselineRecord[]): BaselineSlice {
  if (records.length === 0) fail("切片没有样本");
  const count = (outcome: BaselineOutcome): number =>
    records.filter((record) => record.outcome === outcome).length;
  const ok = count("ok");
  const cacheHits = count("cache-hit");
  const degraded = count("degraded");
  const errors = count("error");
  const totalCostMicros = records.reduce((sum, record) => sum + record.costMicros, 0);

  return {
    samples: records.length,
    ok,
    cacheHits,
    degraded,
    errors,
    successRate: ratio(ok + cacheHits + degraded, records.length),
    errorRate: ratio(errors, records.length),
    degradedRate: ratio(degraded, records.length),
    cacheHitRate: ratio(cacheHits, records.length),
    ttftP95Ms: percentile(records.map((record) => record.ttftMs), 0.95),
    e2eP95Ms: percentile(records.map((record) => record.e2eMs), 0.95),
    avgCostMicros: ratio(totalCostMicros, records.length),
    totalCostMicros,
    avgInputTokens: mean(records.map((record) => record.inputTokens)),
    avgOutputTokens: mean(records.map((record) => record.outputTokens)),
    avgCachedInputShare: mean(records.map((record) => record.cachedInputShare)),
    qualityMean: mean(records.map((record) => record.qualityScore)),
  };
}

function groupBy(
  records: readonly BaselineRecord[],
  key: (record: BaselineRecord) => string,
): Record<string, BaselineSlice> {
  const groups = new Map<string, BaselineRecord[]>();
  for (const record of records) {
    const bucket = groups.get(key(record)) ?? [];
    bucket.push(record);
    groups.set(key(record), bucket);
  }
  const slices: Record<string, BaselineSlice> = {};
  for (const [name, bucket] of groups) slices[name] = summarizeSlice(bucket);
  return slices;
}

function percent(value: number): string {
  return `${(value * 100).toFixed(2)}%`;
}

function finding(
  id: string,
  severity: FindingSeverity,
  metric: string,
  observed: string,
  target: string,
  action: string,
): BaselineFinding {
  return { id, severity, metric, observed, target, action };
}

/**
 * 基线报告把四项口径合成一张表：延迟看 P95，成本看均值，
 * 质量看评测均值，样本量决定结论能不能成立。
 */
export function buildBaseline(
  records: readonly BaselineRecord[],
  targets: BaselineTargets = DEFAULT_TARGETS,
): BaselineReport {
  if (records.length === 0) fail("没有样本，无法建立基线");
  for (const record of records) validate(record);
  const overall = summarizeSlice(records);
  const byCapability = groupBy(records, (record) => record.capability);
  const byTier = groupBy(records, (record) => record.tier);
  const findings: BaselineFinding[] = [];
  const add = (id: string, severity: FindingSeverity, metric: string,
    observed: string, target: string, action: string): void => {
    findings.push(finding(id, severity, metric, observed, target, action));
  };

  if (overall.samples < targets.minSamples) {
    add("sample", "blocker", "sample-size", `${overall.samples} 条`,
      `>= ${targets.minSamples} 条`, "扩大确定性样本后再引用结论");
  }
  if (overall.errorRate > targets.maxErrorRate) {
    add("error", "blocker", "error-rate", percent(overall.errorRate),
      `<= ${percent(targets.maxErrorRate)}`, "按错误类别拆分失败并先消不可重试错误");
  }
  if (overall.e2eP95Ms > targets.maxP95E2eMs) {
    add("latency", "blocker", "e2e-p95", `${overall.e2eP95Ms}ms`,
      `<= ${targets.maxP95E2eMs}ms`, "拆段定位排队、预填充与解码，再决定优化杠杆");
  }
  if (overall.qualityMean < targets.minQualityMean) {
    add("quality", "blocker", "quality-mean", overall.qualityMean.toFixed(3),
      `>= ${targets.minQualityMean}`, "回到评测集确认质量回退来自模型、提示还是压缩");
  }
  if (overall.degradedRate > targets.maxDegradeRate) {
    add("degrade", "warning", "degrade-rate", percent(overall.degradedRate),
      `<= ${percent(targets.maxDegradeRate)}`, "检查主模型健康度，确认降级没有变成默认体验");
  }
  if (overall.cacheHitRate < targets.minCacheHitRate) {
    add("cache", "warning", "cache-hit-rate", percent(overall.cacheHitRate),
      `>= ${percent(targets.minCacheHitRate)}`, "先核对缓存键与提示版本，再决定是否投入缓存容量");
  }
  if (overall.avgCostMicros > targets.maxAvgCostMicros) {
    add("cost", "warning", "avg-cost", `${overall.avgCostMicros.toFixed(2)} 微元`,
      `<= ${targets.maxAvgCostMicros} 微元`, "按能力与模型档位归因，优先处理单位成本最高的切片");
  }
  for (const [name, slice] of Object.entries(byCapability)) {
    if (slice.e2eP95Ms > targets.maxP95E2eMs) {
      add(`latency:${name}`, "warning", "capability-e2e-p95", `${slice.e2eP95Ms}ms`,
        `<= ${targets.maxP95E2eMs}ms`, `单独拆 ${name} 的慢请求，能力级长尾会被整体 P95 掩盖`);
    }
    if (slice.qualityMean < targets.minQualityMean) {
      add(`quality:${name}`, "warning", "capability-quality", slice.qualityMean.toFixed(3),
        `>= ${targets.minQualityMean}`, `回到 ${name} 的评测集，确认回退来自模型档位还是提示变更`);
    }
  }

  const order: Record<FindingSeverity, number> = { blocker: 0, warning: 1 };
  findings.sort((left, right) => order[left.severity] - order[right.severity] || left.id.localeCompare(right.id));
  const verdict = findings.some((item) => item.severity === "blocker")
    ? "fail"
    : findings.length > 0 ? "conditional" : "pass";

  return { overall, byCapability, byTier, findings, verdict, targets };
}

node --test test/perf/baseline.test.ts

ℹ tests 7
ℹ pass 7
ℹ fail 0