《互联网产品的客户端架构·规模化与架构演进》7.2 采样、脱敏与成本

// file: src/observability/sampling-cost.ts
// tier: L1
// verify: node --test test/observability/sampling-cost.test.ts

import { Buffer } from "node:buffer";

export type Severity = "debug" | "info" | "warn" | "error" | "fatal";

export type TelemetryEvent = {
  id: string;
  name: string;
  severity: Severity;
  timestamp: number;
  attributes?: Record<string, string>;
};

export type SampleDecision = { keep: boolean; rate: number; reason: "always" | "rate" };
export type SamplingReport = {
  total: number;
  kept: number;
  dropped: number;
  bySeverity: Record<Severity, { kept: number; dropped: number }>;
};
export type CostConfig = { pricePerGb: number; scaleFactor?: number };
export type CostEstimate = { events: number; bytes: number; gigabytes: number; cost: number };
export type RetentionPolicy = Record<Severity, number>;
export type RetentionPartition = { kept: TelemetryEvent[]; expired: TelemetryEvent[] };

export const SEVERITIES: Severity[] = ["debug", "info", "warn", "error", "fatal"];

export const DEFAULT_RATES: Record<Severity, number> = {
  debug: 0.01,
  info: 0.1,
  warn: 0.5,
  error: 1,
  fatal: 1,
};

export const DEFAULT_RETENTION_DAYS: RetentionPolicy = {
  debug: 3,
  info: 14,
  warn: 30,
  error: 90,
  fatal: 180,
};

export const DAY_MS = 24 * 60 * 60 * 1000;
export const REDACTED = "[REDACTED]";
export const SENSITIVE_KEYS = ["email", "phone", "mobile", "idcard", "id_card", "address", "token", "password"];

const PII_PATTERNS: RegExp[] = [
  /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g,
  /(?<!\d)\d{17}[\dXx](?!\d)/g,
  /(?<!\d)1[3-9]\d{9}(?!\d)/g,
  /(?<!\d)(?:\d{1,3}\.){3}\d{1,3}(?!\d)/g,
];

export function stableHash(input: string): number {
  let hash = 0x811c9dc5;
  for (let i = 0; i < input.length; i += 1) {
    hash ^= input.charCodeAt(i);
    hash = Math.imul(hash, 0x01000193) >>> 0;
  }
  return hash >>> 0;
}

export function sampleDecision(
  key: string,
  severity: Severity,
  rates: Record<Severity, number> = DEFAULT_RATES,
): SampleDecision {
  const rate = rates[severity];
  if (rate >= 1) {
    return { keep: true, rate: 1, reason: "always" };
  }
  if (rate <= 0) {
    return { keep: false, rate: 0, reason: "always" };
  }
  return { keep: stableHash(key) % 10000 < Math.round(rate * 10000), rate, reason: "rate" };
}

export function keepEvent(event: TelemetryEvent, rates: Record<Severity, number> = DEFAULT_RATES): boolean {
  return sampleDecision(event.id, event.severity, rates).keep;
}

export function samplingReport(
  events: TelemetryEvent[],
  rates: Record<Severity, number> = DEFAULT_RATES,
): SamplingReport {
  const bySeverity = {} as Record<Severity, { kept: number; dropped: number }>;
  for (const severity of SEVERITIES) {
    bySeverity[severity] = { kept: 0, dropped: 0 };
  }
  let kept = 0;
  let dropped = 0;
  for (const event of events) {
    if (keepEvent(event, rates)) {
      bySeverity[event.severity].kept += 1;
      kept += 1;
    } else {
      bySeverity[event.severity].dropped += 1;
      dropped += 1;
    }
  }
  return { total: events.length, kept, dropped, bySeverity };
}

export function redactText(text: string): string {
  let output = text;
  for (const pattern of PII_PATTERNS) {
    output = output.replace(pattern, REDACTED);
  }
  return output;
}

function sensitiveKey(key: string): boolean {
  const normalized = key.toLowerCase().replace(/[^a-z0-9_]/g, "");
  return SENSITIVE_KEYS.some((candidate) => normalized.includes(candidate));
}

export function redactAttributes(attributes: Record<string, string>): Record<string, string> {
  const output: Record<string, string> = {};
  for (const key of Object.keys(attributes)) {
    output[key] = sensitiveKey(key) ? REDACTED : redactText(attributes[key]);
  }
  return output;
}

export function redactEvent(event: TelemetryEvent): TelemetryEvent {
  const copy: TelemetryEvent = {
    id: event.id,
    name: redactText(event.name),
    severity: event.severity,
    timestamp: event.timestamp,
  };
  if (event.attributes) {
    copy.attributes = redactAttributes(event.attributes);
  }
  return copy;
}

export function estimateBytes(event: TelemetryEvent): number {
  return Buffer.byteLength(JSON.stringify(event), "utf8");
}

export function estimateCost(events: TelemetryEvent[], config: CostConfig): CostEstimate {
  const bytes = events.reduce((sum, event) => sum + estimateBytes(event), 0);
  const scaledBytes = bytes * (config.scaleFactor ?? 1);
  const gigabytes = scaledBytes / (1024 * 1024 * 1024);
  return { events: events.length, bytes: scaledBytes, gigabytes, cost: gigabytes * config.pricePerGb };
}

export function retentionDays(severity: Severity, policy: RetentionPolicy = DEFAULT_RETENTION_DAYS): number {
  return policy[severity];
}

export function expiresAt(event: TelemetryEvent, policy: RetentionPolicy = DEFAULT_RETENTION_DAYS): number {
  return event.timestamp + retentionDays(event.severity, policy) * DAY_MS;
}

export function isExpired(event: TelemetryEvent, now: number, policy: RetentionPolicy = DEFAULT_RETENTION_DAYS): boolean {
  return now >= expiresAt(event, policy);
}

export function partitionByRetention(
  events: TelemetryEvent[],
  now: number,
  policy: RetentionPolicy = DEFAULT_RETENTION_DAYS,
): RetentionPartition {
  const kept: TelemetryEvent[] = [];
  const expired: TelemetryEvent[] = [];
  for (const event of events) {
    (isExpired(event, now, policy) ? expired : kept).push(event);
  }
  return { kept, expired };
}

export function bytesBySeverity(events: TelemetryEvent[]): Record<Severity, number> {
  const out = {} as Record<Severity, number>;
  for (const severity of SEVERITIES) {
    out[severity] = 0;
  }
  for (const event of events) {
    out[event.severity] += estimateBytes(event);
  }
  return out;
}

✔ 默认采样率随等级单调不减 (0.575958ms)
✔ error 与 fatal 全量保留 (0.402875ms)
✔ 同一 key 的决策稳定 (0.142583ms)
✔ stableHash 对同一输入返回同一结果 (0.072625ms)
✔ debug 采样率约为百分之一 (1.22175ms)
✔ 采样率为零时全部丢弃 (0.093041ms)
✔ 采样率为一时全部保留且标记 always (0.055542ms)
✔ 采样统计守恒且按等级分桶 (0.114666ms)
✔ 邮箱与手机号被脱敏 (0.266042ms)
✔ 身份证号被脱敏 (0.108209ms)
✔ IPv4 地址被脱敏 (0.059458ms)
✔ 敏感键名直接整体脱敏 (0.109041ms)
✔ 脱敏不修改原对象 (0.246875ms)
✔ redactEvent 同时清理名称与属性 (0.064291ms)
✔ 字节估算为正且随字段增长 (0.086667ms)
✔ 成本估算按缩放因子线性增长 (0.084875ms)
✔ 成本估算与千兆字节换算一致 (0.051166ms)
✔ 默认保留天数按等级递减 (0.049125ms)
✔ 到期时间按事件时间加保留天数 (0.047666ms)
✔ 到期判定在边界时刻生效 (0.04925ms)
✔ 保留策略拆分保留与过期事件 (0.1325ms)
✔ 自定义保留策略覆盖默认值 (0.043583ms)
✔ 按等级统计字节数覆盖全部等级 (0.065666ms)
ℹ tests 23
ℹ suites 0
ℹ pass 23
ℹ fail 0
ℹ cancelled 0
ℹ skipped 0
ℹ todo 0
ℹ duration_ms 136.694416