《互联网产品的 AI 架构·模型接入与能力边界》9.3 命中率的度量与陷阱

// file: src/cache/metrics.ts
// tier: L1
// verify: node --test test/cache/metrics.test.ts

import type { CapabilityId, TenantId } from "../core/types.ts";

export type LookupOutcome = "exact" | "semantic" | "miss";

export interface MetricKey {
  capability: CapabilityId;
  tenantId: TenantId;
}

export interface MetricBucket {
  lookups: number;
  exact: number;
  semantic: number;
  misses: number;
  semanticRejected: number;
  semanticFalsePositives: number;
  writes: number;
  invalidations: number;
  churn: number;
  similaritySum: number;
}

export interface HitRate {
  lookups: number;
  exact: number;
  semantic: number;
  misses: number;
  hitRate: number;
  exactRate: number;
  semanticRate: number;
  /** 语义命中里经用户反馈确认正确的比例。 */
  semanticPrecision: number;
  /** 扣掉已知误命中后的命中率。 */
  adjustedHitRate: number;
  /** 写入后很快被失效的比例，反映缓存污染。 */
  churnRate: number;
}

export interface MetricsSnapshot {
  overall: HitRate;
  byCapability: Record<string, HitRate>;
  byTenant: Record<string, HitRate>;
  byCapabilityTenant: Record<string, HitRate>;
  semanticRejected: number;
  semanticFalsePositives: number;
  averageSemanticSimilarity: number;
}

export interface MetricsOptions {
  now?: () => number;
  /** 写入后多久内被失效算作一次抖动，默认 60 秒。 */
  pollutionWindowMs?: number;
}

function emptyBucket(): MetricBucket {
  return {
    lookups: 0,
    exact: 0,
    semantic: 0,
    misses: 0,
    semanticRejected: 0,
    semanticFalsePositives: 0,
    writes: 0,
    invalidations: 0,
    churn: 0,
    similaritySum: 0,
  };
}

function bucketKey(key: MetricKey): string {
  return `${key.tenantId}\u0000${key.capability}`;
}

function merge(target: MetricBucket, source: MetricBucket): void {
  target.lookups += source.lookups;
  target.exact += source.exact;
  target.semantic += source.semantic;
  target.misses += source.misses;
  target.semanticRejected += source.semanticRejected;
  target.semanticFalsePositives += source.semanticFalsePositives;
  target.writes += source.writes;
  target.invalidations += source.invalidations;
  target.churn += source.churn;
  target.similaritySum += source.similaritySum;
}

function toRate(bucket: MetricBucket): HitRate {
  const hits = bucket.exact + bucket.semantic;
  const semanticCorrect = Math.max(0, bucket.semantic - bucket.semanticFalsePositives);
  return {
    lookups: bucket.lookups,
    exact: bucket.exact,
    semantic: bucket.semantic,
    misses: bucket.misses,
    hitRate: bucket.lookups === 0 ? 0 : hits / bucket.lookups,
    exactRate: bucket.lookups === 0 ? 0 : bucket.exact / bucket.lookups,
    semanticRate: bucket.lookups === 0 ? 0 : bucket.semantic / bucket.lookups,
    semanticPrecision: bucket.semantic === 0 ? 1 : semanticCorrect / bucket.semantic,
    adjustedHitRate:
      bucket.lookups === 0 ? 0 : Math.max(0, bucket.exact + semanticCorrect) / bucket.lookups,
    churnRate: bucket.writes === 0 ? 0 : bucket.churn / bucket.writes,
  };
}

export class HitRateMetrics {
  private buckets = new Map<string, MetricBucket>();
  private writeTimes = new Map<string, number>();
  private now: () => number;
  private pollutionWindowMs: number;

  constructor(options: MetricsOptions = {}) {
    this.now = options.now ?? (() => Date.now());
    this.pollutionWindowMs = options.pollutionWindowMs ?? 60_000;
  }

  recordLookup(key: MetricKey, outcome: LookupOutcome, similarity = 0): void {
    const bucket = this.bucket(key);
    bucket.lookups += 1;
    if (outcome === "exact") {
      bucket.exact += 1;
    } else if (outcome === "semantic") {
      bucket.semantic += 1;
      bucket.similaritySum += similarity;
    } else {
      bucket.misses += 1;
    }
  }

  /** 语义层评估过但被阈值或守卫拒绝，计入候选拒绝数。 */
  recordSemanticRejected(key: MetricKey): void {
    this.bucket(key).semanticRejected += 1;
  }

  recordSemanticFalsePositive(key: MetricKey): void {
    this.bucket(key).semanticFalsePositives += 1;
  }

  recordWrite(key: MetricKey, cacheKey: string): void {
    this.bucket(key).writes += 1;
    this.writeTimes.set(cacheKey, this.now());
  }

  recordInvalidation(key: MetricKey, cacheKey: string): void {
    const bucket = this.bucket(key);
    bucket.invalidations += 1;
    const writtenAt = this.writeTimes.get(cacheKey);
    if (writtenAt !== undefined) {
      this.writeTimes.delete(cacheKey);
      if (this.now() - writtenAt <= this.pollutionWindowMs) {
        bucket.churn += 1;
      }
    }
  }

  reset(): void {
    this.buckets.clear();
    this.writeTimes.clear();
  }

  snapshot(): MetricsSnapshot {
    const overall = emptyBucket();
    const byCapability = new Map<string, MetricBucket>();
    const byTenant = new Map<string, MetricBucket>();
    const byCapabilityTenant: Record<string, HitRate> = {};

    for (const bucket of this.buckets.values()) {
      merge(overall, bucket);
    }

    for (const [rawKey, bucket] of this.buckets) {
      const [tenantId, capability] = rawKey.split("\u0000");
      const capBucket = byCapability.get(capability) ?? emptyBucket();
      merge(capBucket, bucket);
      byCapability.set(capability, capBucket);

      const tenantBucket = byTenant.get(tenantId) ?? emptyBucket();
      merge(tenantBucket, bucket);
      byTenant.set(tenantId, tenantBucket);

      byCapabilityTenant[`${tenantId}/${capability}`] = toRate(bucket);
    }

    return {
      overall: toRate(overall),
      byCapability: Object.fromEntries(
        [...byCapability].map(([capability, bucket]) => [capability, toRate(bucket)]),
      ),
      byTenant: Object.fromEntries(
        [...byTenant].map(([tenantId, bucket]) => [tenantId, toRate(bucket)]),
      ),
      byCapabilityTenant,
      semanticRejected: overall.semanticRejected,
      semanticFalsePositives: overall.semanticFalsePositives,
      averageSemanticSimilarity:
        overall.semantic === 0 ? 0 : overall.similaritySum / overall.semantic,
    };
  }

  private bucket(key: MetricKey): MetricBucket {
    const rawKey = bucketKey(key);
    let bucket = this.buckets.get(rawKey);
    if (!bucket) {
      bucket = emptyBucket();
      this.buckets.set(rawKey, bucket);
    }
    return bucket;
  }
}