《互联网产品的 AI 架构·推理性能与成本工程》10.3 结果解读与回归风险

// file: src/perf/experiment-analysis.ts
// tier: L2
// verify: node --test test/perf/experiment-analysis.test.ts

import { AiError } from "../core/errors.ts";
import { normalCdf, tCritical95 } from "./experiment.ts";
import type { MetricSpec, PairedSample } from "./experiment.ts";

export interface ArmSummary {
  n: number;
  mean: number;
  sd: number;
  se: number;
}

function fail(message: string): never {
  throw new AiError("invalid-request", `experiment-analysis: ${message}`);
}

function mean(values: readonly number[]): number {
  return values.reduce((sum, value) => sum + value, 0) / values.length;
}

function sampleSd(values: readonly number[]): number {
  if (values.length < 2) return 0;
  const center = mean(values);
  const variance = values.reduce((sum, value) => sum + (value - center) ** 2, 0) / (values.length - 1);
  return Math.sqrt(variance);
}

export function summarizeArm(values: readonly number[]): ArmSummary {
  if (values.length === 0) fail("汇总至少需要 1 个值");
  const sd = sampleSd(values);
  return { n: values.length, mean: mean(values), sd, se: sd / Math.sqrt(values.length) };
}

export interface PairedAnalysis {
  metric: string;
  n: number;
  baseline: ArmSummary;
  treatment: ArmSummary;
  meanDiff: number;
  sdDiff: number;
  ci95: [number, number];
  tStat: number;
  pValue: number;
  relativeChange: number;
  cohensDz: number;
  /** 按指标方向归一后的改善量：正数表示处理组更好。 */
  improvement: number;
  significant: boolean;
}

/** 逐条配对后再做差，而不是拿两个组的均值直接相减。配对消掉样本难度差异。 */
export function analyzePaired(samples: readonly PairedSample[], spec: MetricSpec, alpha = 0.05): PairedAnalysis {
  if (samples.length < 2) fail("配对分析至少需要 2 条样本");
  for (const sample of samples) {
    if (!Number.isFinite(sample.baseline) || !Number.isFinite(sample.treatment)) fail(`样本 ${sample.id} 的数值必须是有限数`);
  }
  const baseline = summarizeArm(samples.map((sample) => sample.baseline));
  const treatment = summarizeArm(samples.map((sample) => sample.treatment));
  const diffs = samples.map((sample) => sample.treatment - sample.baseline);
  const meanDiff = mean(diffs);
  const sdDiff = sampleSd(diffs);
  const seDiff = sdDiff / Math.sqrt(samples.length);
  const crit = tCritical95(samples.length - 1);
  const ci95: [number, number] = seDiff === 0 ? [meanDiff, meanDiff] : [meanDiff - crit * seDiff, meanDiff + crit * seDiff];
  const tStat = seDiff === 0 ? (meanDiff === 0 ? 0 : Number.POSITIVE_INFINITY * Math.sign(meanDiff)) : meanDiff / seDiff;
  const pValue = seDiff === 0 ? (meanDiff === 0 ? 1 : 0) : 2 * (1 - normalCdf(Math.abs(tStat)));
  const improvement = spec.direction === "lower-is-better" ? -meanDiff : meanDiff;
  return {
    metric: spec.name, n: samples.length, baseline, treatment, meanDiff, sdDiff, ci95, tStat, pValue,
    relativeChange: baseline.mean === 0 ? 0 : meanDiff / Math.abs(baseline.mean),
    cohensDz: sdDiff === 0 ? 0 : meanDiff / sdDiff,
    improvement,
    significant: pValue < alpha && (ci95[0] > 0 || ci95[1] < 0),
  };
}

export interface SegmentObservation {
  segment: string;
  baselineN: number;
  baselineMean: number;
  treatmentN: number;
  treatmentMean: number;
}

export interface SegmentDelta {
  segment: string;
  baselineN: number;
  treatmentN: number;
  delta: number;
  weight: number;
}

export interface SimpsonReport {
  aggregateBaseline: number;
  aggregateTreatment: number;
  aggregateDelta: number;
  withinSegmentDelta: number;
  signFlip: boolean;
  flippedSegments: string[];
  segments: SegmentDelta[];
}

function weightedMean(values: readonly number[], weights: readonly number[]): number {
  const total = weights.reduce((sum, weight) => sum + weight, 0);
  if (total === 0) return 0;
  return values.reduce((sum, value, index) => sum + value * weights[index], 0) / total;
}

/** 整体差值用两组各自样本量加权，段内差值用每段总样本量加权；反号即构成悖论。 */
export function detectSimpson(segments: readonly SegmentObservation[]): SimpsonReport {
  if (segments.length === 0) fail("辛普森检查至少需要 1 个分段");
  for (const item of segments) {
    if (item.baselineN < 0 || item.treatmentN < 0) fail(`${item.segment}: 样本量不能为负`);
    if (!Number.isFinite(item.baselineMean) || !Number.isFinite(item.treatmentMean)) fail(`${item.segment}: 均值必须是有限数`);
  }
  const aggregateBaseline = weightedMean(segments.map((item) => item.baselineMean), segments.map((item) => item.baselineN));
  const aggregateTreatment = weightedMean(segments.map((item) => item.treatmentMean), segments.map((item) => item.treatmentN));
  const totalN = segments.reduce((sum, item) => sum + item.baselineN + item.treatmentN, 0);
  const details: SegmentDelta[] = segments.map((item) => ({
    segment: item.segment, baselineN: item.baselineN, treatmentN: item.treatmentN,
    delta: item.treatmentMean - item.baselineMean,
    weight: totalN === 0 ? 0 : (item.baselineN + item.treatmentN) / totalN,
  }));
  const withinSegmentDelta = details.reduce((sum, item) => sum + item.delta * item.weight, 0);
  const aggregateDelta = aggregateTreatment - aggregateBaseline;
  const signFlip = aggregateDelta * withinSegmentDelta < 0 && Math.abs(aggregateDelta) > 1e-9 && Math.abs(withinSegmentDelta) > 1e-9;
  return {
    aggregateBaseline, aggregateTreatment, aggregateDelta, withinSegmentDelta, signFlip,
    flippedSegments: signFlip
      ? details.filter((item) => item.delta * aggregateDelta < 0 && item.weight >= 0.05).map((item) => item.segment)
      : [],
    segments: details,
  };
}

export interface GuardrailResult {
  metric: string;
  /** 处理组相对对照组的原始差值。 */
  delta: number;
  allowed: number;
  passed: boolean;
}

/** 护栏按“回退量”判断：成本看是否涨，质量看是否跌，方向不能共用一套符号。 */
export function checkGuardrail(analysis: PairedAnalysis, spec: MetricSpec): GuardrailResult {
  const regression = spec.direction === "higher-is-better" ? -analysis.meanDiff : analysis.meanDiff;
  return { metric: spec.name, delta: analysis.meanDiff, allowed: spec.nonInferiorityMargin, passed: regression <= spec.nonInferiorityMargin };
}

export interface DecisionPolicy {
  alpha: number;
  /** 相对改善低于该比例时，不值得为迁移与回滚风险买单。 */
  minRelativeEffect: number;
}

export interface ExperimentDecision {
  ship: boolean;
  reasons: string[];
}

export function decideExperiment(primary: PairedAnalysis, guardrails: readonly GuardrailResult[], policy: DecisionPolicy): ExperimentDecision {
  const reasons: string[] = [];
  if (!primary.significant) reasons.push("主指标未达到统计显著");
  if (primary.improvement <= 0) reasons.push("主指标没有改善");
  if (Math.abs(primary.relativeChange) < policy.minRelativeEffect) {
    reasons.push(`相对变化 ${(primary.relativeChange * 100).toFixed(2)}% 低于门槛 ${(policy.minRelativeEffect * 100).toFixed(2)}%`);
  }
  for (const guardrail of guardrails) {
    if (!guardrail.passed) reasons.push(`护栏 ${guardrail.metric} 回退 ${guardrail.delta.toFixed(4)} 超过允许 ${guardrail.allowed}`);
  }
  return { ship: reasons.length === 0, reasons };
}

export interface ShareObservation {
  segment: string;
  /** 实验样本中该段占比。 */
  experimentShare: number;
  /** 生产流量中该段占比。 */
  productionShare: number;
  /** 处理组减对照组；负数表示处理组更差。 */
  delta: number;
}

export interface ExtrapolationReport {
  /** 两组占比的总变差距离，0 表示构成完全一致。 */
  shareGap: number;
  risk: "low" | "medium" | "high";
  underRepresentedHarmful: string[];
  reasons: string[];
}

/** 外推风险 = 段构成差多少 + 是否漏掉受损段；只报“代表性良好”无法指导发布。 */
export function extrapolationRisk(items: readonly ShareObservation[]): ExtrapolationReport {
  if (items.length === 0) fail("外推检查至少需要 1 个分段");
  for (const item of items) {
    if (item.experimentShare < 0 || item.productionShare < 0) fail(`${item.segment}: 占比不能为负`);
  }
  const shareGap = 0.5 * items.reduce((sum, item) => sum + Math.abs(item.experimentShare - item.productionShare), 0);
  const underRepresentedHarmful = items
    .filter((item) => item.delta < 0 && item.productionShare - item.experimentShare > 0.05)
    .map((item) => item.segment);
  const harmfulProduction = items.filter((item) => item.delta < 0).reduce((sum, item) => sum + item.productionShare, 0);
  const reasons: string[] = [];
  if (shareGap > 0.15) reasons.push(`段构成差异 ${(shareGap * 100).toFixed(1)}% 超过 15%`);
  if (underRepresentedHarmful.length > 0) reasons.push(`受损段被低采样：${underRepresentedHarmful.join("、")}`);
  if (harmfulProduction > 0.2) reasons.push(`生产流量中受损段占 ${(harmfulProduction * 100).toFixed(1)}%`);
  return { shareGap, risk: reasons.length >= 2 ? "high" : reasons.length === 1 ? "medium" : "low", underRepresentedHarmful, reasons };
}

node --test test/perf/experiment-analysis.test.ts

ℹ tests 7
ℹ pass 7
ℹ fail 0
ℹ duration_ms 154.132792