《互联网产品的 AI 架构·模型接入与能力边界》13.3 基线度量与风险清单

// file: src/observe/baseline.ts
// tier: L1
// verify: node --test test/observe/baseline.test.ts

import { AiError } from "../core/errors.ts";

export type SampleOutcome = "ok" | "cache-hit" | "degraded" | "error";

/** 一条样本就是一次请求的结算行，不要在统计阶段再回查原始日志。 */
export interface BaselineSample {
  requestId: string; capability: string; outcome: SampleOutcome;
  latencyMs: number; costMicros: number; cacheHit: boolean; degraded: boolean;
}

export interface BaselineThresholds {
  latencyBudgetMs: number; costCeilingMicros: number; minCacheHitRate: number;
  maxErrorRate: number; maxDegradeRate: number; minSampleSize: number;
}

/** 阈值是基线的一部分，写死在代码里等于让数字失去出处。 */
export const DEFAULT_THRESHOLDS: BaselineThresholds = {
  latencyBudgetMs: 800, costCeilingMicros: 200, minCacheHitRate: 0.2,
  maxErrorRate: 0.01, maxDegradeRate: 0.05, minSampleSize: 200,
};

export interface CapabilityBaseline {
  samples: number; successRate: number; p95LatencyMs: number;
  avgCostMicros: number; cacheHitRate: number; degradedRate: number;
}

export interface BaselineMetrics {
  samples: number; ok: number; cacheHits: number; degraded: number; errors: number;
  successRate: number; errorRate: number; degradedRate: number; cacheHitRate: number;
  avgLatencyMs: number; p95LatencyMs: number;
  totalCostMicros: number; avgCostMicros: number; overLatency: number; overCost: number;
  byCapability: Record<string, CapabilityBaseline>;
}

const OUTCOMES = new Set<string>(["ok", "cache-hit", "degraded", "error"]);

function fail(message: string): never {
  throw new AiError("invalid-request", `baseline: ${message}`);
}

function validateSample(sample: BaselineSample): void {
  if (sample.requestId.trim() === "" || sample.capability.trim() === "") fail("requestId 与 capability 不能为空");
  if (!OUTCOMES.has(sample.outcome)) fail(`未知 outcome ${String(sample.outcome)}`);
  if (!Number.isFinite(sample.latencyMs) || sample.latencyMs < 0) fail(`${sample.requestId}: latencyMs 必须是非负有限数`);
  if (!Number.isInteger(sample.costMicros) || sample.costMicros < 0) fail(`${sample.requestId}: costMicros 必须是非负整数`);
}

/** 最近秩法：第 ceil(p*n) 个值，结果可被逐条样本复算。 */
export function percentile(values: readonly number[], ratio: number): number {
  if (!(ratio > 0 && ratio <= 1)) fail("ratio 必须落在 (0, 1]");
  if (values.length === 0) return 0;
  const sorted = [...values].sort((left, right) => left - right);
  const rank = Math.ceil(ratio * sorted.length) - 1;
  return sorted[Math.max(0, Math.min(sorted.length - 1, rank))];
}

function rate(numerator: number, denominator: number): number {
  return denominator === 0 ? 0 : numerator / denominator;
}

function summarize(samples: readonly BaselineSample[], thresholds: BaselineThresholds): Omit<BaselineMetrics, "byCapability"> {
  const latencies = samples.map((sample) => sample.latencyMs);
  const count = (outcome: SampleOutcome): number => samples.filter((sample) => sample.outcome === outcome).length;
  const ok = count("ok");
  const cacheHits = count("cache-hit");
  const degraded = count("degraded");
  const errors = count("error");
  const totalCostMicros = samples.reduce((sum, sample) => sum + sample.costMicros, 0);
  return {
    samples: samples.length, ok, cacheHits, degraded, errors,
    successRate: rate(ok + cacheHits + degraded, samples.length),
    errorRate: rate(errors, samples.length),
    degradedRate: rate(degraded, samples.length),
    cacheHitRate: rate(cacheHits, samples.length),
    avgLatencyMs: rate(latencies.reduce((sum, value) => sum + value, 0), samples.length),
    p95LatencyMs: percentile(latencies, 0.95),
    totalCostMicros,
    avgCostMicros: rate(totalCostMicros, samples.length),
    overLatency: samples.filter((sample) => sample.latencyMs > thresholds.latencyBudgetMs).length,
    overCost: samples.filter((sample) => sample.costMicros > thresholds.costCeilingMicros).length,
  };
}

/** 成功率把降级与缓存命中都算作“交付了结果”，错误才算失败。 */
export function computeBaseline(
  samples: readonly BaselineSample[],
  thresholds: BaselineThresholds = DEFAULT_THRESHOLDS,
): BaselineMetrics {
  if (samples.length === 0) fail("没有样本，无法建立基线");
  for (const sample of samples) validateSample(sample);
  const overall = summarize(samples, thresholds);

  const groups = new Map<string, BaselineSample[]>();
  for (const sample of samples) {
    const bucket = groups.get(sample.capability) ?? [];
    bucket.push(sample);
    groups.set(sample.capability, bucket);
  }
  const byCapability: Record<string, CapabilityBaseline> = {};
  for (const [capability, bucket] of groups) {
    const data = summarize(bucket, thresholds);
    byCapability[capability] = {
      samples: data.samples, successRate: data.successRate, p95LatencyMs: data.p95LatencyMs,
      avgCostMicros: data.avgCostMicros, cacheHitRate: data.cacheHitRate, degradedRate: data.degradedRate,
    };
  }
  return { ...overall, byCapability };
}

export type RiskCategory = "availability" | "latency" | "cost" | "degrade" | "cache" | "evidence";

export interface BaselineRisk {
  id: string; category: RiskCategory; title: string;
  impact: number; probability: number; score: number; evidence: string; action: string;
}

function percent(value: number): string {
  return `${(value * 100).toFixed(2)}%`;
}

/** 把观测比例映射到 1..5 的概率档，阈值来自基线而不是主观打分。 */
function band(value: number, tolerance: number): number {
  if (value <= 0) return 1;
  const ratio = value / Math.max(tolerance, 1e-9);
  if (ratio >= 2) return 5;
  if (ratio >= 1) return 4;
  if (ratio >= 0.5) return 3;
  return 2;
}

function risk(
  id: string, category: RiskCategory, title: string,
  impact: number, probability: number, evidence: string, action: string,
): BaselineRisk {
  return { id, category, title, impact, probability, score: impact * probability, evidence, action };
}

/** 风险清单只收录有样本证据的项，按 影响×概率 从高到低排序。 */
export function buildRiskList(
  metrics: BaselineMetrics,
  thresholds: BaselineThresholds = DEFAULT_THRESHOLDS,
): BaselineRisk[] {
  const risks: BaselineRisk[] = [];
  const total = metrics.samples;
  if (metrics.errors > 0) {
    risks.push(risk("availability", "availability", "请求失败率超出可接受范围", 5,
      band(metrics.errorRate, thresholds.maxErrorRate),
      `失败 ${metrics.errors}/${total}，错误率 ${percent(metrics.errorRate)}`,
      "按错误类别拆分失败，先消除不可重试错误"));
  }
  if (metrics.degraded > 0) {
    risks.push(risk("degrade", "degrade", "降级路径被频繁触发", 4,
      band(metrics.degradedRate, thresholds.maxDegradeRate),
      `降级 ${metrics.degraded}/${total}，降级率 ${percent(metrics.degradedRate)}`,
      "检查主模型健康度与降级链，确认降级没有变成默认体验"));
  }
  if (metrics.overLatency > 0 || metrics.p95LatencyMs > thresholds.latencyBudgetMs) {
    risks.push(risk("latency", "latency", "P95 延迟超出交互预算", 4,
      band(rate(metrics.overLatency, total), 0.05),
      `P95 ${metrics.p95LatencyMs}ms，超预算 ${metrics.overLatency}/${total}`,
      "拆段定位慢在排队、调用还是后处理，再决定换模型或加缓存"));
  }
  if (metrics.overCost > 0 || metrics.avgCostMicros > thresholds.costCeilingMicros) {
    risks.push(risk("cost", "cost", "单次成本超出预算上限", 4,
      band(rate(metrics.overCost, total), 0.05),
      `平均 ${metrics.avgCostMicros.toFixed(2)} 微元/次，超上限 ${metrics.overCost}/${total}`,
      "检查输入长度与重试次数，必要时下调模型档位"));
  }
  if (metrics.cacheHitRate < thresholds.minCacheHitRate) {
    const shortfall = thresholds.minCacheHitRate - metrics.cacheHitRate;
    risks.push(risk("cache", "cache", "缓存命中率低于基线", 3,
      band(shortfall, thresholds.minCacheHitRate * 0.5),
      `命中率 ${percent(metrics.cacheHitRate)}，下限 ${percent(thresholds.minCacheHitRate)}`,
      "核对缓存键是否包含易变字段，并确认提示版本没有频繁变更"));
  }
  if (total < thresholds.minSampleSize) {
    risks.push(risk("evidence", "evidence", "样本量不足以支撑结论", 2, 5,
      `样本 ${total} 条，最低 ${thresholds.minSampleSize} 条`,
      "扩大确定性样本集后再决定是否进入下一卷"));
  }
  return risks.sort((left, right) =>
    right.score - left.score || right.impact - left.impact || left.id.localeCompare(right.id));
}

export interface ReadinessCondition { id: string; met: boolean; detail: string }
export interface VolumeReadiness { ready: boolean; conditions: ReadinessCondition[]; blockers: string[] }

/** 进入卷二前，基线必须同时满足样本量、成功率、延迟、成本、缓存、降级与风险七项。 */
export function assessVolumeReadiness(
  metrics: BaselineMetrics,
  risks: readonly BaselineRisk[],
  thresholds: BaselineThresholds = DEFAULT_THRESHOLDS,
): VolumeReadiness {
  const critical = risks.filter((item) => item.score >= 20);
  const conditions: ReadinessCondition[] = [
    { id: "sample", met: metrics.samples >= thresholds.minSampleSize, detail: `${metrics.samples}/${thresholds.minSampleSize} 条样本` },
    { id: "success", met: metrics.successRate >= 1 - thresholds.maxErrorRate, detail: `成功率 ${percent(metrics.successRate)}` },
    { id: "latency", met: metrics.p95LatencyMs <= thresholds.latencyBudgetMs, detail: `P95 ${metrics.p95LatencyMs}ms / ${thresholds.latencyBudgetMs}ms` },
    { id: "cost", met: metrics.avgCostMicros <= thresholds.costCeilingMicros, detail: `平均 ${metrics.avgCostMicros.toFixed(2)} / ${thresholds.costCeilingMicros} 微元` },
    { id: "cache", met: metrics.cacheHitRate >= thresholds.minCacheHitRate, detail: `命中率 ${percent(metrics.cacheHitRate)}` },
    { id: "degrade", met: metrics.degradedRate <= thresholds.maxDegradeRate, detail: `降级率 ${percent(metrics.degradedRate)}` },
    { id: "risk", met: critical.length === 0, detail: critical.length === 0 ? "无高分风险" : `高分风险 ${critical.map((item) => item.id).join(",")}` },
  ];
  const blockers = conditions.filter((condition) => !condition.met).map((condition) => condition.id);
  return { ready: blockers.length === 0, conditions, blockers };
}

node --test test/observe/baseline.test.ts

ℹ tests 7
ℹ pass 7
ℹ fail 0