《互联网产品的 AI 架构·平台化与治理》5.3 人工评审与仲裁

// file: src/gov/eval-review.ts
// tier: L2
// verify: node --test test/gov/eval-platform.test.ts

import { createHash } from "node:crypto";

/** 人工评审的四个动作：分层抽样、双盲派单、分歧仲裁、评审员校准。 */
export type Risk = "high" | "normal" | "low";
export interface ReviewCase {
  id: string;
  capability: string;
  risk: Risk;
  author: string;
}
export interface ReviewPlanOptions {
  /** 常规样本的抽检率，万分比。 */
  rateBp: number;
  seed: string;
  /** 高风险全检是默认值，显式传入更低的值需要治理委员会批准。 */
  highRiskRateBp?: number;
}
export interface Stratum {
  risk: Risk;
  population: number;
  target: number;
  selected: number;
}
export interface ReviewPlan {
  selected: string[];
  strata: Stratum[];
  rateBp: number;
  seed: string;
}
function rank(seed: string, value: string): string {
  return createHash("sha256").update(`${seed}:${value}`).digest("hex");
}
function targetFor(risk: Risk, population: number, rateBp: number, highRiskRateBp: number): number {
  if (risk === "high") return Math.ceil((population * highRiskRateBp) / 10_000);
  const effective = risk === "low" ? Math.ceil(rateBp / 2) : rateBp;
  return Math.ceil((population * effective) / 10_000);
}

/** 高风险层默认全检；排序只依赖种子与 case id，同一输入总能得到同一份名单。 */
export function planReview(
  cases: readonly ReviewCase[],
  options: ReviewPlanOptions,
): ReviewPlan {
  const highRiskRateBp = options.highRiskRateBp ?? 10_000;
  const risks: readonly Risk[] = ["high", "normal", "low"];
  const strata: Stratum[] = [];
  const selected: string[] = [];
  for (const risk of risks) {
    const pool = cases.filter((item) => item.risk === risk).sort((left, right) => {
      const a = rank(options.seed, left.id);
      const b = rank(options.seed, right.id);
      return a === b ? left.id.localeCompare(right.id) : a.localeCompare(b);
    });
    const target = Math.min(pool.length, targetFor(risk, pool.length, options.rateBp, highRiskRateBp));
    const picked = pool.slice(0, target).map((item) => item.id);
    selected.push(...picked);
    strata.push({ risk, population: pool.length, target, selected: picked.length });
  }
  return { selected: selected.sort(), strata, rateBp: options.rateBp, seed: options.seed };
}
export interface Reviewer {
  id: string;
  role: "reviewer" | "arbiter";
  capabilities: readonly string[];
  /** 与该评审员存在利益冲突的作者 id。 */
  conflicts: readonly string[];
  /** 最近一次校准的通过率，万分比；未校准按 0 处理。 */
  agreementBp: number;
}
export interface CaseAssignment { caseId: string; reviewers: [string, string] }
export type AssignmentCode =
  | "insufficient-reviewers" | "capability-miss" | "conflict" | "uncalibrated";
export interface AssignmentIssue { caseId: string; code: AssignmentCode; detail: string }
export interface AssignmentPlan {
  assignments: CaseAssignment[];
  issues: AssignmentIssue[];
  blind: true;
}
const MIN_REVIEWER_BP = 8_000;

/** 作者不可评审自己的样本，两名评审员互相看不到对方身份与结论。 */
export function assignDoubleBlind(
  cases: readonly ReviewCase[],
  reviewers: readonly Reviewer[],
  seed: string,
): AssignmentPlan {
  const assignments: CaseAssignment[] = [];
  const issues: AssignmentIssue[] = [];
  for (const item of cases) {
    const eligible = reviewers.filter((reviewer) =>
      reviewer.role === "reviewer"
      && reviewer.capabilities.includes(item.capability)
      && !reviewer.conflicts.includes(item.author)
      && reviewer.id !== item.author);
    const calibrated = eligible.filter((reviewer) => reviewer.agreementBp >= MIN_REVIEWER_BP);
    if (calibrated.length < 2) {
      const code: AssignmentCode = eligible.length < 2 ? "insufficient-reviewers" : "uncalibrated";
      issues.push({
        caseId: item.id, code,
        detail: `可用评审员 ${calibrated.length} 名，至少需要 2 名`,
      });
      continue;
    }
    const ordered = calibrated
      .map((reviewer) => ({ id: reviewer.id, key: rank(`${seed}:${item.id}`, reviewer.id) }))
      .sort((left, right) => (left.key === right.key ? left.id.localeCompare(right.id) : left.key.localeCompare(right.key)));
    assignments.push({ caseId: item.id, reviewers: [ordered[0].id, ordered[1].id] });
  }
  return { assignments, issues, blind: true };
}
export type ReviewLabel = "pass" | "fail" | "escalate";
export interface ReviewVerdict {
  caseId: string;
  reviewerId: string;
  label: ReviewLabel;
  /** 0–100 的自评把握度，只用于排序与审计，不直接决定结论。 */
  confidence: number;
}
export type ResolutionStatus = "settled" | "needs-arbiter" | "invalid";
export interface CaseResolution {
  caseId: string;
  status: ResolutionStatus;
  label?: ReviewLabel;
  reason: string;
}

/** 两位评审员结论一致才算落槌；任何分歧或升级都必须进入仲裁。 */
export function resolveCase(verdicts: readonly ReviewVerdict[]): CaseResolution {
  const caseId = verdicts[0]?.caseId ?? "unknown";
  const reviewers = new Set(verdicts.map((item) => item.reviewerId));
  if (verdicts.length !== 2 || reviewers.size !== 2) {
    return { caseId, status: "invalid", reason: `需要两位不同评审员的结论，实际 ${verdicts.length} 条` };
  }
  if (verdicts.some((item) => item.confidence < 0 || item.confidence > 100)) {
    return { caseId, status: "invalid", reason: "把握度必须在 0–100 之间" };
  }
  const [left, right] = verdicts;
  if (left.label === right.label) {
    if (left.label === "escalate") {
      return { caseId, status: "needs-arbiter", reason: "两位评审员都要求升级" };
    }
    return { caseId, status: "settled", label: left.label, reason: "两位评审员结论一致" };
  }
  return { caseId, status: "needs-arbiter", reason: `结论分歧：${left.label} / ${right.label}` };
}
export interface ArbiterChoice { arbiterId: string | null; reason: string }

/** 仲裁人不能是本案评审员，也不得与作者存在利益冲突；同分时按 id 排序。 */
export function assignArbiter(
  reviewCase: ReviewCase,
  verdicts: readonly ReviewVerdict[],
  arbiters: readonly Reviewer[],
  seed: string,
): ArbiterChoice {
  const used = new Set(verdicts.map((item) => item.reviewerId));
  const eligible = arbiters.filter((reviewer) =>
    reviewer.role === "arbiter"
    && reviewer.capabilities.includes(reviewCase.capability)
    && !reviewer.conflicts.includes(reviewCase.author)
    && !used.has(reviewer.id));
  if (eligible.length === 0) return { arbiterId: null, reason: "没有符合条件的仲裁人" };
  const ordered = eligible
    .map((reviewer) => ({
      id: reviewer.id,
      agreement: reviewer.agreementBp,
      key: rank(`${seed}:${reviewCase.id}:arbiter`, reviewer.id),
    }))
    .sort((left, right) => {
      if (left.agreement !== right.agreement) return right.agreement - left.agreement;
      return left.key === right.key ? left.id.localeCompare(right.id) : left.key.localeCompare(right.key);
    });
  return { arbiterId: ordered[0].id, reason: `校准通过率 ${ordered[0].agreement}bp，最高优先级` };
}
export interface GoldCase { caseId: string; expected: "pass" | "fail" }
export type CalibrationStatus = "calibrated" | "probation" | "blocked";
export interface Calibration {
  reviewerId: string;
  reviewed: number;
  agreed: number;
  agreementBp: number;
  falsePass: number;
  falseFail: number;
  status: CalibrationStatus;
}

/** 校准只看金标样本；漏放（false pass）比误拦（false fail）更严重。 */
export function calibrateReviewer(
  reviewerId: string,
  verdicts: readonly ReviewVerdict[],
  gold: readonly GoldCase[],
): Calibration {
  const goldMap = new Map(gold.map((item) => [item.caseId, item.expected]));
  const own = verdicts.filter((item) => item.reviewerId === reviewerId && goldMap.has(item.caseId));
  let agreed = 0;
  let falsePass = 0;
  let falseFail = 0;
  for (const item of own) {
    const expected = goldMap.get(item.caseId);
    if (item.label === expected) agreed += 1;
    else if (item.label === "pass") falsePass += 1;
    else if (item.label === "fail") falseFail += 1;
  }
  const agreementBp = own.length === 0 ? 0 : Math.round((agreed * 10_000) / own.length);
  const status: CalibrationStatus =
    own.length < 20 ? "blocked"
      : agreementBp >= 9_000 && falsePass === 0 ? "calibrated"
        : agreementBp >= 8_000 ? "probation" : "blocked";
  return { reviewerId, reviewed: own.length, agreed, agreementBp, falsePass, falseFail, status };
}