《互联网产品的 AI 架构·推理性能与成本工程》10.1 基线、假设与实验设计

n = 2 × (z(1−α/2) + z(power))² × σ² / Δ²

// file: src/perf/experiment.ts
// tier: L2
// verify: node --test test/perf/experiment.test.ts

import { AiError } from "../core/errors.ts";

export type MetricKind = "costMicros" | "latencyMs" | "qualityScore";
export type MetricDirection = "lower-is-better" | "higher-is-better";
export type AssignmentUnit = "request" | "item" | "sample";

/** 指标口径固定三件事：单位、方向、非劣性边界。缺一项就会在评审时各说各话。 */
export interface MetricSpec {
  name: string;
  kind: MetricKind;
  direction: MetricDirection;
  /** 允许处理组相对对照组回退的上限；主指标通常为 0。 */
  nonInferiorityMargin: number;
}

export interface ExperimentDesign {
  id: string;
  hypothesis: string;
  unit: AssignmentUnit;
  baselineArm: string;
  treatmentArm: string;
  /** 处理组相对对照组只允许改变一个因子。 */
  changedFactors: string[];
  primary: MetricSpec;
  guardrails: MetricSpec[];
  /** 每组目标样本量；等分分配时即每组样本数。 */
  sampleSize: number;
  /** 统计窗口，必须覆盖至少一个完整业务周期。 */
  windowMinutes: number;
  allocationRatio: number;
  assignment: "hash" | "alternate" | "none";
}

export interface DesignIssue {
  code: string;
  message: string;
}

function fail(message: string): never {
  throw new AiError("invalid-request", `experiment: ${message}`);
}

/** 设计门禁只拦结构性问题；多因子、无分配、缺质量护栏都会在这里暴露。 */
export function checkDesign(design: ExperimentDesign): DesignIssue[] {
  const issues: DesignIssue[] = [];
  if (design.changedFactors.length !== 1) {
    issues.push({ code: "factor-count", message: `单变量实验必须恰好改变 1 个因子，当前 ${design.changedFactors.length} 个` });
  }
  if (design.changedFactors.some((factor) => factor.trim() === "")) {
    issues.push({ code: "empty-factor", message: "因子名不能为空" });
  }
  if (design.assignment === "none") {
    issues.push({ code: "no-assignment", message: "没有随机或交替分配，无法排除顺序、时段与选择偏差" });
  }
  if (!Number.isInteger(design.sampleSize) || design.sampleSize <= 0) {
    issues.push({ code: "sample-size", message: "每组样本量必须是正整数" });
  }
  if (!Number.isFinite(design.windowMinutes) || design.windowMinutes <= 0) {
    issues.push({ code: "window", message: "统计窗口必须是正数分钟" });
  }
  if (!(design.allocationRatio > 0 && design.allocationRatio < 1)) {
    issues.push({ code: "allocation", message: "流量比例必须落在 (0, 1)" });
  } else if (Math.abs(design.allocationRatio - 0.5) > 0.1) {
    issues.push({ code: "unbalanced", message: `处理组流量 ${design.allocationRatio}，两组样本量相差超过 20%，方差估计会失真` });
  }
  if (!design.guardrails.some((spec) => spec.kind === "qualityScore")) {
    issues.push({ code: "no-quality-guardrail", message: "缺少质量护栏；只看成本与延迟会把质量回退放行" });
  }
  if (design.primary.nonInferiorityMargin < 0) {
    issues.push({ code: "negative-margin", message: "非劣性边界不能为负" });
  }
  return issues;
}

/** Acklam 有理逼近；误差远小于实验本身的抽样误差。 */
export function normalQuantile(p: number): number {
  if (!(p > 0 && p < 1)) fail("概率必须落在 (0, 1)");
  const a = [-39.69683028665376, 220.9460984245205, -275.9285104469687, 138.357751867269, -30.66479806614716, 2.506628277459239];
  const b = [-54.47609879822406, 161.5858368580409, -155.6989798598866, 66.80131188771972, -13.28068155288572];
  const c = [-0.007784894002430293, -0.3223964580411365, -2.400758277161838, -2.549732539343734, 4.374664141464968, 2.938163982698783];
  const d = [0.007784695709041462, 0.3224671290700398, 2.445134137142996, 3.754408661907416];
  const low = 0.02425;
  if (p < low) {
    const q = Math.sqrt(-2 * Math.log(p));
    return (((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) / ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1);
  }
  if (p > 1 - low) {
    const q = Math.sqrt(-2 * Math.log(1 - p));
    return -(((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) / ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1);
  }
  const q = p - 0.5;
  const r = q * q;
  return (((((a[0] * r + a[1]) * r + a[2]) * r + a[3]) * r + a[4]) * r + a[5]) * q / (((((b[0] * r + b[1]) * r + b[2]) * r + b[3]) * r + b[4]) * r + 1);
}

function erf(x: number): number {
  const sign = x < 0 ? -1 : 1;
  const ax = Math.abs(x);
  const t = 1 / (1 + 0.3275911 * ax);
  const poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t;
  return sign * (1 - poly * Math.exp(-ax * ax));
}

export function normalCdf(z: number): number {
  return 0.5 * (1 + erf(z / Math.SQRT2));
}

const T95 = [12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.160, 2.145, 2.131, 2.120, 2.110, 2.101, 2.093, 2.086, 2.080, 2.074, 2.069, 2.064, 2.060, 2.056, 2.052, 2.048, 2.045, 2.042];

export function tCritical95(df: number): number {
  if (!Number.isInteger(df) || df < 1) fail("自由度必须是正整数");
  return df <= 30 ? T95[df - 1] : 1.96;
}

/**
 * 两组等分、双侧检验所需样本量：n = 2 × (z(1−α/2) + z(power))² × σ² / Δ²。
 * Δ 是期望检测到的最小差值，不是事后观测到的差值。
 */
export function requiredSampleSize(effect: number, sd: number, alpha = 0.05, power = 0.8): number {
  if (!(effect > 0)) fail("effect 必须是正数");
  if (!(sd > 0)) fail("sd 必须是正数");
  if (!(alpha > 0 && alpha < 1)) fail("alpha 必须落在 (0, 1)");
  if (!(power > 0 && power < 1)) fail("power 必须落在 (0, 1)");
  const z = normalQuantile(1 - alpha / 2) + normalQuantile(power);
  return Math.ceil((2 * z * z * sd * sd) / (effect * effect));
}

/** 给定样本量能稳定检出的最小效应，用来区分“没效果”和“没样本”。 */
export function minimumDetectableEffect(sd: number, sampleSize: number, alpha = 0.05, power = 0.8): number {
  if (!(sd > 0)) fail("sd 必须是正数");
  if (!Number.isInteger(sampleSize) || sampleSize < 2) fail("sampleSize 必须是 >= 2 的整数");
  const z = normalQuantile(1 - alpha / 2) + normalQuantile(power);
  return z * sd * Math.sqrt(2 / sampleSize);
}

export interface PairedSample {
  id: string;
  baseline: number;
  treatment: number;
}

node --test test/perf/experiment.test.ts

ℹ tests 6
ℹ pass 6
ℹ fail 0
ℹ duration_ms 113.920083