《互联网产品的 AI 架构·推理性能与成本工程》3.2 前缀缓存与提示结构

// file: src/perf/prefix.ts
// tier: L2
// verify: node --test test/perf/prefix.test.ts

import { createHash } from "node:crypto";

import { AiError } from "../core/errors.ts";
import { estimateTokens } from "../prompt/context-budget.ts";
import type { ModelProfile } from "../core/types.ts";

/** 只有跨请求稳定的角色才允许进入可缓存前缀。 */
export const CACHEABLE_ROLES = ["system", "policy", "reference"] as const;
export const VOLATILE_ROLES = ["history", "tool", "question"] as const;
export type PromptRole = (typeof CACHEABLE_ROLES)[number] | (typeof VOLATILE_ROLES)[number];

export interface PromptBlock {
  id: string;
  role: PromptRole;
  text: string;
}

export interface PrefixPlan {
  blocks: PromptBlock[];
  cacheableBlocks: string[];
  volatileBlocks: string[];
  prefixTokens: number;
  suffixTokens: number;
  totalTokens: number;
  /** 稳定角色的块出现在可变块之后的数量；大于零说明前缀被截断。 */
  buriedStable: number;
  /** 每个块结尾的累计指纹，用来比较两次请求在哪一块分叉。 */
  fingerprints: string[];
  prefixFingerprint: string | null;
}

function fail(message: string): never {
  throw new AiError("invalid-request", `prefix: ${message}`);
}

function isCacheable(role: PromptRole): boolean {
  return (CACHEABLE_ROLES as readonly string[]).includes(role);
}

function validate(blocks: readonly PromptBlock[]): void {
  if (blocks.length === 0) fail("至少需要一个提示块");
  const seen = new Set<string>();
  for (const block of blocks) {
    if (block.id.trim() === "") fail("块 id 不能为空");
    if (seen.has(block.id)) fail(`块 id 重复：${block.id}`);
    seen.add(block.id);
    if (!isCacheable(block.role) && !(VOLATILE_ROLES as readonly string[]).includes(block.role)) {
      fail(`未知角色：${String(block.role)}`);
    }
    if (block.text.trim() === "") fail(`块 ${block.id} 的 text 不能为空`);
  }
}

/**
 * 前缀从第一个块开始，连续到第一个可变角色为止。
 * 后面的稳定块即使内容不变，也已经落在断点之后，无法命中缓存。
 */
export function planPrefix(blocks: readonly PromptBlock[]): PrefixPlan {
  validate(blocks);
  const cacheableBlocks: string[] = [];
  const volatileBlocks: string[] = [];
  const fingerprints: string[] = [];
  let rolling = "root";
  let prefixTokens = 0;
  let suffixTokens = 0;
  let inPrefix = true;
  let buriedStable = 0;

  for (const block of blocks) {
    rolling = createHash("sha256")
      .update(`${rolling}\u0001${block.role}\u0000${block.text}`, "utf8")
      .digest("hex")
      .slice(0, 16);
    fingerprints.push(rolling);
    const tokens = estimateTokens(block.text);
    if (isCacheable(block.role) && inPrefix) {
      cacheableBlocks.push(block.id);
      prefixTokens += tokens;
      continue;
    }
    inPrefix = false;
    volatileBlocks.push(block.id);
    suffixTokens += tokens;
    if (isCacheable(block.role)) buriedStable += 1;
  }

  return {
    blocks: [...blocks],
    cacheableBlocks,
    volatileBlocks,
    prefixTokens,
    suffixTokens,
    totalTokens: prefixTokens + suffixTokens,
    buriedStable,
    fingerprints,
    prefixFingerprint: cacheableBlocks.length === 0 ? null : fingerprints[cacheableBlocks.length - 1],
  };
}

export interface ReorderResult {
  blocks: PromptBlock[];
  moved: string[];
}

/** 稳定角色在前、可变角色在后，两组内部各自保持原相对顺序。只调位置，不改内容。 */
export function reorderForCache(blocks: readonly PromptBlock[]): ReorderResult {
  validate(blocks);
  const ranked = blocks.map((block, index) => ({ block, index }));
  ranked.sort((left, right) => {
    const group = Number(!isCacheable(left.block.role)) - Number(!isCacheable(right.block.role));
    return group || left.index - right.index;
  });
  return {
    blocks: ranked.map((entry) => entry.block),
    moved: ranked.filter((entry, position) => entry.index !== position).map((entry) => entry.block.id),
  };
}

export interface PrefixMatch {
  sharedBlocks: string[];
  sharedTokens: number;
  /** 下一个请求里第一处分叉的块 id；完全匹配时为 null。 */
  firstDivergence: string | null;
  reason: "exact" | "diverged" | "empty";
}

/** 缓存命中只看前缀：某块内容或角色不同，后面的块即使一样也不能复用。 */
export function commonPrefix(
  previous: readonly PromptBlock[],
  next: readonly PromptBlock[],
): PrefixMatch {
  validate(previous);
  validate(next);
  const sharedBlocks: string[] = [];
  let sharedTokens = 0;
  let firstDivergence: string | null = null;

  for (let index = 0; index < Math.min(previous.length, next.length); index += 1) {
    const left = previous[index];
    const right = next[index];
    if (left.role !== right.role || left.text !== right.text) {
      firstDivergence = right.id;
      break;
    }
    sharedBlocks.push(right.id);
    sharedTokens += estimateTokens(right.text);
  }
  if (firstDivergence === null && next.length > previous.length) {
    firstDivergence = next[previous.length].id;
  }
  return {
    sharedBlocks,
    sharedTokens,
    firstDivergence,
    reason: sharedBlocks.length === 0 ? "empty" : firstDivergence === null ? "exact" : "diverged",
  };
}

export interface CacheRecord {
  modelId: string;
  promptVersion: string;
  /** 缓存作用域：租户、区域与权限域的组合，不同范围不能复用。 */
  scopeKey: string;
  prefixFingerprint: string;
  prefixTokens: number;
  createdAtMs: number;
  ttlMs: number;
}

export interface CacheContext {
  modelId: string;
  promptVersion: string;
  scopeKey: string;
  nowMs: number;
  /** 提供方允许缓存的最小 token 数，低于它不会建立缓存。 */
  minCacheableTokens?: number;
}

export interface CacheEligibility {
  eligible: boolean;
  hitTokens: number;
  violations: string[];
}

/** 命中条件逐条检查，任何一条不满足都不命中；返回值同时给出原因。 */
export function checkPrefixCache(
  plan: PrefixPlan,
  record: CacheRecord | null,
  context: CacheContext,
): CacheEligibility {
  if (record === null) return { eligible: false, hitTokens: 0, violations: ["no-record"] };
  const violations: string[] = [];
  if (record.modelId !== context.modelId) violations.push("model-changed");
  if (record.promptVersion !== context.promptVersion) violations.push("prompt-version-changed");
  if (record.scopeKey !== context.scopeKey) violations.push("scope-changed");
  if (context.nowMs - record.createdAtMs >= record.ttlMs) violations.push("expired");
  if (record.prefixFingerprint !== plan.prefixFingerprint) violations.push("prefix-changed");
  if (plan.prefixTokens < (context.minCacheableTokens ?? 0)) violations.push("below-minimum");
  const eligible = violations.length === 0;
  return { eligible, hitTokens: eligible ? plan.prefixTokens : 0, violations };
}

/**
 * 缓存读取价通常是输入价的一个折扣。命中与未命中分别取整，
 * 避免一次请求因为合并取整而出现零成本。
 */
export function cachedInputCostMicros(
  model: ModelProfile,
  totalTokens: number,
  hitTokens: number,
  readFactor = 0.1,
): number {
  if (!Number.isInteger(totalTokens) || totalTokens < 0) fail("totalTokens 必须是非负整数");
  if (!Number.isInteger(hitTokens) || hitTokens < 0 || hitTokens > totalTokens) {
    fail("hitTokens 必须落在 [0, totalTokens]");
  }
  if (!(readFactor > 0 && readFactor <= 1)) fail("readFactor 必须落在 (0, 1]");
  return (
    Math.ceil((hitTokens * model.inputMicrosPerMTok * readFactor) / 1_000_000) +
    Math.ceil(((totalTokens - hitTokens) * model.inputMicrosPerMTok) / 1_000_000)
  );
}

node --test test/perf/prefix.test.ts

ℹ tests 9
ℹ pass 9
ℹ fail 0
ℹ duration_ms 208.15825