《互联网产品的 AI 架构·平台化与治理》5.2 自动化回归与门禁

// file: src/gov/eval-gates.ts
// tier: L2
// verify: node --test test/gov/eval-platform.test.ts

import { createHash } from "node:crypto";

/** 三级门禁：提交级拦单点改动，发布级拦能力版本，换代级拦模型整体切换。 */
export type GateLevel = "commit" | "release" | "model-swap";
export type Direction = "higher" | "lower";
export interface GateRule {
  metric: string; direction: Direction;
  /** higher 用绝对差，lower 用相对比例；两者都只允许「变差」。 */
  maxRegression: number; minSample: number; hard: boolean;
}
export interface GatePolicy { level: GateLevel; rules: readonly GateRule[] }
export const GATE_POLICIES: Record<GateLevel, GatePolicy> = {
  commit: {
    level: "commit",
    rules: [
      { metric: "task.success", direction: "higher", maxRegression: 0.005, minSample: 200, hard: false },
      { metric: "delivery.cost", direction: "lower", maxRegression: 0.02, minSample: 200, hard: false },
    ],
  },
  release: {
    level: "release",
    rules: [
      { metric: "task.success", direction: "higher", maxRegression: 0.003, minSample: 1_000, hard: false },
      { metric: "delivery.cost", direction: "lower", maxRegression: 0.02, minSample: 1_000, hard: false },
      { metric: "latency.p95", direction: "lower", maxRegression: 0.05, minSample: 1_000, hard: false },
      { metric: "safety.violation", direction: "lower", maxRegression: 0, minSample: 1_000, hard: true },
    ],
  },
  "model-swap": {
    level: "model-swap",
    rules: [
      { metric: "task.success", direction: "higher", maxRegression: 0.002, minSample: 2_000, hard: false },
      { metric: "delivery.cost", direction: "lower", maxRegression: 0.01, minSample: 2_000, hard: false },
      { metric: "latency.p95", direction: "lower", maxRegression: 0.03, minSample: 2_000, hard: false },
      { metric: "safety.violation", direction: "lower", maxRegression: 0, minSample: 2_000, hard: true },
    ],
  },
};
export interface MetricSample { metric: string; baseline: number; candidate: number; sampleSize: number }
export interface GateEvidence {
  level: GateLevel; runId: string; evalSetId: string; evalSetVersion: string;
  samples: readonly MetricSample[]; createdAt: string;
}
export type CheckCode = "pass" | "regression" | "missing-metric" | "insufficient-sample" | "exempted";
export interface GateCheckResult {
  metric: string; code: CheckCode; baseline: number; candidate: number;
  delta: number; sampleSize: number; requiredSample: number; hard: boolean;
}
export interface GateResult {
  level: GateLevel; runId: string; passed: boolean;
  checks: GateCheckResult[]; failed: string[];
}
function delta(rule: GateRule, sample: MetricSample): number {
  if (rule.direction === "higher") return sample.candidate - sample.baseline;
  return sample.baseline === 0 ? 0 : sample.candidate / sample.baseline - 1;
}
function withinTolerance(rule: GateRule, sample: MetricSample): boolean {
  const change = delta(rule, sample);
  return rule.direction === "higher" ? change >= -rule.maxRegression : change <= rule.maxRegression;
}

/** 缺指标与样本不足都按失败处理：没有证据不等于没有退化。 */
export function evaluateGate(evidence: GateEvidence, policy: GatePolicy = GATE_POLICIES[evidence.level]): GateResult {
  const checks: GateCheckResult[] = [];
  for (const rule of policy.rules) {
    const sample = evidence.samples.find((item) => item.metric === rule.metric);
    if (sample === undefined) {
      checks.push({
        metric: rule.metric, code: "missing-metric", baseline: 0, candidate: 0,
        delta: 0, sampleSize: 0, requiredSample: rule.minSample, hard: rule.hard,
      });
      continue;
    }
    const base = {
      metric: rule.metric, baseline: sample.baseline, candidate: sample.candidate,
      delta: Math.round(delta(rule, sample) * 10_000) / 10_000,
      sampleSize: sample.sampleSize, requiredSample: rule.minSample, hard: rule.hard,
    };
    if (sample.sampleSize < rule.minSample) checks.push({ ...base, code: "insufficient-sample" });
    else if (!withinTolerance(rule, sample)) checks.push({ ...base, code: "regression" });
    else checks.push({ ...base, code: "pass" });
  }
  const failed = checks.filter((item) => item.code !== "pass").map((item) => item.metric);
  return { level: evidence.level, runId: evidence.runId, passed: failed.length === 0, checks, failed };
}
export interface GateExemption {
  id: string; level: GateLevel; metric: string; owner: string; reason: string;
  createdAt: string; expiresAt: string; evidenceRef: string;
}
export interface ExemptionIssue { id: string; code: string; detail: string }
const MAX_EXEMPT_DAYS: Record<GateLevel, number> = { commit: 3, release: 14, "model-swap": 0 };
const EVIDENCE_REF = /^EV-(COMMIT|RELEASE|SWAP)-[0-9a-f]{8}$/;
const DAY = 86_400_000;
function dayDiff(from: string, to: string): number {
  return Math.round((Date.parse(`${to}T00:00:00Z`) - Date.parse(`${from}T00:00:00Z`)) / DAY);
}
export function validateExemptions(
  exemptions: readonly GateExemption[],
  policy: Record<GateLevel, GatePolicy> = GATE_POLICIES,
): ExemptionIssue[] {
  const issues: ExemptionIssue[] = [];
  const active = new Set<string>();
  for (const item of exemptions) {
    const rule = policy[item.level].rules.find((entry) => entry.metric === item.metric);
    if (rule === undefined) {
      issues.push({ id: item.id, code: "unknown-metric", detail: `${item.level} 没有指标 ${item.metric}` });
      continue;
    }
    if (rule.hard) issues.push({ id: item.id, code: "hard-metric", detail: `${item.metric} 是硬指标，不允许豁免` });
    if (MAX_EXEMPT_DAYS[item.level] === 0) {
      issues.push({ id: item.id, code: "level-not-exemptible", detail: "模型换代级不允许豁免" });
    }
    if (item.owner.trim() === "") issues.push({ id: item.id, code: "missing-owner", detail: "豁免必须有责任人" });
    if ([...item.reason].length < 20) {
      issues.push({ id: item.id, code: "reason-too-short", detail: "豁免理由少于 20 个字符" });
    }
    if (!EVIDENCE_REF.test(item.evidenceRef)) {
      issues.push({ id: item.id, code: "invalid-evidence", detail: "豁免必须引用证据包编号" });
    }
    const window = dayDiff(item.createdAt, item.expiresAt);
    if (window <= 0 || window > MAX_EXEMPT_DAYS[item.level]) {
      issues.push({
        id: item.id, code: "invalid-window",
        detail: `豁免窗口 ${window} 天，${item.level} 最多 ${MAX_EXEMPT_DAYS[item.level]} 天`,
      });
    }
    const key = `${item.level}:${item.metric}`;
    if (active.has(key)) {
      issues.push({ id: item.id, code: "duplicate-active", detail: `${key} 同时存在多条有效豁免` });
    }
    active.add(key);
  }
  return issues;
}
export interface ExemptionRecord {
  id: string; level: GateLevel; metric: string; owner: string;
  expiresAt: string; applied: boolean; reason: string;
}
export interface GateDecision {
  passed: boolean; blocked: string[]; exempted: string[];
  records: ExemptionRecord[]; ignored: ExemptionRecord[];
}
function isActive(item: GateExemption, now: string): boolean {
  return item.createdAt <= now && now < item.expiresAt;
}

/** 只有「回归」可以豁免；缺指标、样本不足与硬指标都必须保持阻塞。 */
export function applyExemptions(
  result: GateResult,
  exemptions: readonly GateExemption[],
  now: string,
): GateDecision {
  const records: ExemptionRecord[] = [];
  const ignored: ExemptionRecord[] = [];
  const exempted: string[] = [];
  for (const check of result.checks) {
    if (check.code !== "regression" || check.hard) continue;
    const found = exemptions.find(
      (item) => item.level === result.level && item.metric === check.metric && isActive(item, now),
    );
    if (found === undefined) continue;
    exempted.push(check.metric);
    check.code = "exempted";
    records.push({
      id: found.id, level: found.level, metric: found.metric, owner: found.owner,
      expiresAt: found.expiresAt, applied: true, reason: found.reason,
    });
  }
  for (const item of exemptions) {
    if (records.some((record) => record.id === item.id)) continue;
    if (item.level !== result.level || !result.failed.includes(item.metric) || !isActive(item, now)) {
      ignored.push({
        id: item.id, level: item.level, metric: item.metric, owner: item.owner,
        expiresAt: item.expiresAt, applied: false, reason: "未命中失败指标、级别不符或不在有效期",
      });
    }
  }
  const blocked = result.checks
    .filter((item) => item.code !== "pass" && item.code !== "exempted")
    .map((item) => item.metric);
  return { passed: blocked.length === 0, blocked, exempted, records, ignored };
}
export interface EvidenceBundle {
  id: string; level: GateLevel; runId: string; evalSet: string; digest: string;
  createdAt: string; retainUntil: string; retentionDays: number;
}
const LEVEL_CODE: Record<GateLevel, string> = { commit: "COMMIT", release: "RELEASE", "model-swap": "SWAP" };
export const RETENTION_DAYS: Record<GateLevel, number> = { commit: 30, release: 180, "model-swap": 730 };

/** 证据包只摘要判定输入，不保存原始样本；摘要按指标名排序，保证同样输入同样编号。 */
export function buildEvidence(evidence: GateEvidence): EvidenceBundle {
  const canonical = [...evidence.samples]
    .sort((left, right) => left.metric.localeCompare(right.metric))
    .map((item) => `${item.metric}:${item.baseline}:${item.candidate}:${item.sampleSize}`)
    .join("|");
  const digest = createHash("sha256")
    .update(`${evidence.evalSetId}@${evidence.evalSetVersion}|${canonical}`)
    .digest("hex");
  const retentionDays = RETENTION_DAYS[evidence.level];
  return {
    id: `EV-${LEVEL_CODE[evidence.level]}-${digest.slice(0, 8)}`,
    level: evidence.level,
    runId: evidence.runId,
    evalSet: `${evidence.evalSetId}@${evidence.evalSetVersion}`,
    digest,
    createdAt: evidence.createdAt,
    retainUntil: new Date(Date.parse(`${evidence.createdAt}T00:00:00Z`) + retentionDays * DAY)
      .toISOString().slice(0, 10),
    retentionDays,
  };
}
export interface EvidenceAudit { id: string; ageDays: number; daysRemaining: number; expired: boolean }
export function auditEvidence(bundle: EvidenceBundle, today: string): EvidenceAudit {
  const ageDays = Math.max(0, dayDiff(bundle.createdAt, today));
  return {
    id: bundle.id,
    ageDays,
    daysRemaining: Math.max(0, bundle.retentionDays - ageDays),
    expired: ageDays > bundle.retentionDays,
  };
}
export interface ReleaseDecision {
  evidence: EvidenceBundle; gate: GateResult; decision: GateDecision;
  audit: EvidenceAudit; pass: boolean;
}

/** 一次判定同时产出结论与证据包；没有证据包的放行不算放行。 */
export function decideRelease(
  evidence: GateEvidence,
  exemptions: readonly GateExemption[],
  today: string,
): ReleaseDecision {
  const gate = evaluateGate(evidence);
  const decision = applyExemptions(gate, exemptions, today);
  const bundle = buildEvidence(evidence);
  return { evidence: bundle, gate, decision, audit: auditEvidence(bundle, today), pass: decision.passed };
}