上一节把 135 条变体裁到 80 条排进产线，但「产出来」不等于「该放量」。判定谁赢这件事，卷三 8.2 显著性样本量 已从 A/B 实验角度讲过，卷三目录 《上线变现与长线运营》 给了完整方法论。本节不重做实验设计，只做素材场景的具体判定：一堆 CTR 摆在面前，怎么用整数算出「赢/平/淘汰」，再把预算分下去。
测试矩阵：把「比 CTR」变成「判显著」
最朴素的做法是「谁 CTR 高谁赢」，这在素材测试里几乎必错：小样本下 2‰ 的差距可能纯粹是噪声。正确姿势是先跑一个固定曝光量的小样本矩阵，再对每条素材和现役对照组（control）做显著性判定，只把预算分给「显著更高」的那些：胜出（增量 > 0 且显著）进放量池；持平（增量 > 0 但测不出显著）不杀也不放；淘汰（增量 ≤ 0 或低于地板线）直接停投。
判定公式：整数 z 检验，不开方
两比例 z 检验的标准形式要开方，但我们只需比大小，两边平方后通分就能全部化成整数：
d = obsCtr - controlCtr # 增量（千分比）
pooled = (clicks + controlClicks) × 1000 // (imp + controlImp)
胜出 iff d² × imp × controlImp × 100 > 384 × pooled × (1000 - pooled) × (imp + controlImp)
384 是 z²×100（z=1.96 对应双侧 95%）。左边是「观测到的差异」，右边是「噪声能解释的差异」，左边更大才叫显著。全程整数、不开方——这正是它能跨语言逐位一致的前提。
S 层仿真：从矩阵到分配
下面这段是 S 层仿真脚本 sim/o03_matrix.py。它读 config/o03_matrix.json，给每条素材生成小样本点击、判显著、给胜出者分预算，产出 JSON + CSV + SVG。
#!/usr/bin/env python3
# file: sim/o03_matrix.py
# tier: S
# verify: /opt/homebrew/bin/python3 sim/o03_matrix.py --config config/ --out out/
"""素材测试矩阵仿真：小样本测 → 整数显著性判定 → 胜出者按增量分配预算。
clicks_i = imp*trueCtr_i//1000 + (rng%7 - 3) # 小样本抖动 ±3 点击
obsCtr_i = clicks_i*1000 // imp；d_i = obsCtr_i - controlCtr # 相对增量（千分比）
pooled = (clicks_i + controlClicks)*1000 // (imp + controlImp)
胜出 iff d_i > 0 且 d_i²·imp·controlImp·100 > 384·pooled·(1000-pooled)·(imp+controlImp)
且 obsCtr_i >= minCtrPermille；预算 budget_i = scaleBudget*d_i // Σd，余数从大到小补齐。
整数双侧 z 检验，无浮点、无开方；真实 CTR 与真实显著性需真实业务环境，命令行不可复现（X）。
产出 out/o03_matrix.json + .csv + .svg。
"""
from __future__ import annotations
import argparse
import csv
import json
import os
import xml.etree.ElementTree as ET
def make_rng(seed: int, mult: int, inc: int):
state = seed & 0xFFFFFFFF
def nxt() -> int:
nonlocal state
state = (state * mult + inc) & 0xFFFFFFFF
return state
return nxt
def is_winner(d: int, imp: int, clicks: int, cimp: int, cclicks: int, z2x100: int) -> bool:
if d <= 0:
return False
pooled = (clicks + cclicks) * 1000 // (imp + cimp)
return d * d * imp * cimp * 100 > z2x100 * pooled * (1000 - pooled) * (imp + cimp)
def run(cfg: dict) -> dict:
nxt = make_rng(cfg["seed"], cfg["rngMultiplier"], cfg["rngIncrement"])
cimp = cfg["control"]["impressions"]
cclicks = cimp * cfg["control"]["ctrPermille"] // 1000
ctr_control = cclicks * 1000 // cimp
imp = cfg["testImpressionsPerCreative"]
rows = []
for c in cfg["creatives"]:
clicks = max(0, imp * c["trueCtrPermille"] // 1000 + (nxt() % 7 - 3))
obs = clicks * 1000 // imp
d = obs - ctr_control
win = is_winner(d, imp, clicks, cimp, cclicks, cfg["z2ThresholdX100"])
verdict = "淘汰" if (obs < cfg["minCtrPermille"] or d <= 0) else ("胜出" if win else "持平")
cpi = cfg["cpmFen"] * 1000 // (obs * cfg["ipmPermille"]) if obs else 10 ** 9
rows.append({"id": c["id"], "theme": c["theme"], "trueCtrPermille": c["trueCtrPermille"],
"clicks": clicks, "obsCtrPermille": obs, "liftPermille": d, "verdict": verdict,
"winner": win and obs >= cfg["minCtrPermille"], "cpiFen": cpi,
"budgetFen": 0, "installs": 0})
winners = [r for r in rows if r["winner"]]
total_lift = sum(r["liftPermille"] for r in winners)
if total_lift > 0:
rem = []
for r in winners:
exact = cfg["scaleBudgetFen"] * r["liftPermille"]
r["budgetFen"] = exact // total_lift
rem.append((exact % total_lift, r["id"], r))
left = cfg["scaleBudgetFen"] - sum(r["budgetFen"] for r in winners)
rem.sort(key=lambda t: (-t[0], t[1]))
for i in range(left):
rem[i % len(rem)][2]["budgetFen"] += 1
for r in winners:
r["installs"] = r["budgetFen"] // r["cpiFen"] if r["cpiFen"] else 0
killed = sum(1 for r in rows if r["verdict"] == "淘汰")
win_ids = ",".join(f'{r["id"]}:{r["liftPermille"]}:{r["budgetFen"]}' for r in winners)
return {"controlCtrPermille": ctr_control, "rows": rows, "winnerCount": len(winners),
"killedCount": killed, "totalBudgetFen": sum(r["budgetFen"] for r in winners),
"totalInstalls": sum(r["installs"] for r in winners),
"digest": f'control:{ctr_control}|winners:{win_ids}|killed:{killed}'}
def draw_svg(path: str, rep: dict) -> int:
"""纯 Python SVG：每条素材一根 CTR 横条，颜色标判定，虚线为现役素材基线。"""
rows, W, PAD, ROW = rep["rows"], 760, 150, 32
H = 50 + ROW * len(rows)
root = ET.Element("svg", {"xmlns": "http://www.w3.org/2000/svg", "width": str(W),
"height": str(H), "viewBox": f"0 0 {W} {H}"})
ET.SubElement(root, "rect", {"width": str(W), "height": str(H), "fill": "#ffffff"})
ET.SubElement(root, "text", {"x": "16", "y": "24", "font-size": "14",
"fill": "#111827"}).text = "素材测试矩阵：各素材 CTR（千分比）与判定"
span = W - PAD - 120
hi = max([r["obsCtrPermille"] for r in rows] + [rep["controlCtrPermille"]]) or 1
xc = PAD + span * rep["controlCtrPermille"] // hi
ET.SubElement(root, "line", {"x1": str(xc), "y1": "34", "x2": str(xc), "y2": str(H - 12),
"stroke": "#ef4444", "stroke-dasharray": "4 4"})
colors = {"胜出": "#2563eb", "持平": "#93c5fd", "淘汰": "#d1d5db"}
for i, r in enumerate(rows):
y, w = 42 + i * ROW, span * r["obsCtrPermille"] // hi
ET.SubElement(root, "text", {"x": "16", "y": str(y + 14), "font-size": "11",
"fill": "#374151"}).text = f'{r["id"]} {r["verdict"]}'
ET.SubElement(root, "rect", {"x": str(PAD), "y": str(y + 2), "width": str(w),
"height": "14", "rx": "3", "fill": colors[r["verdict"]]})
ET.SubElement(root, "text", {"x": str(PAD + w + 6), "y": str(y + 14), "font-size": "10",
"fill": "#6b7280"}).text = f'{r["obsCtrPermille"]}‰'
os.makedirs(os.path.dirname(path), exist_ok=True)
ET.ElementTree(root).write(path, encoding="utf-8", xml_declaration=True)
return len(ET.tostring(root))
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--config", default="config/")
ap.add_argument("--out", default="out/")
args = ap.parse_args()
with open(os.path.join(args.config, "o03_matrix.json"), encoding="utf-8") as f:
cfg = json.load(f)
rep = run(cfg)
os.makedirs(args.out, exist_ok=True)
svg_bytes = draw_svg(os.path.join(args.out, "o03_matrix.svg"), rep)
report = {"version": cfg["version"], "game": cfg["game"], "observedAt": cfg["observedAt"],
**rep, "svg": os.path.join(args.out, "o03_matrix.svg"), "svgBytes": svg_bytes}
with open(os.path.join(args.out, "o03_matrix.json"), "w", encoding="utf-8") as f:
json.dump(report, f, ensure_ascii=False, indent=2)
with open(os.path.join(args.out, "o03_matrix.csv"), "w", encoding="utf-8", newline="") as f:
w = csv.writer(f)
cols = ("id", "theme", "trueCtrPermille", "clicks", "obsCtrPermille", "liftPermille",
"verdict", "cpiFen", "budgetFen", "installs")
w.writerow(cols)
for r in rep["rows"]:
w.writerow([r[k] for k in cols])
print(json.dumps({"controlCtrPermille": rep["controlCtrPermille"],
"rows": [{k: r[k] for k in ("id", "obsCtrPermille", "liftPermille",
"verdict", "budgetFen", "installs")}
for r in rep["rows"]],
"winnerCount": rep["winnerCount"], "killedCount": rep["killedCount"],
"totalInstalls": rep["totalInstalls"], "digest": rep["digest"]},
ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
素材与对照组参数放在 config/o03_matrix.json。CTR 基线是「公开可观察机制」口径的建模假设，不是任何后台实测。
{
"version": 1,
"game": "merge-island",
"observedAt": "2026-10",
"seed": 20261013,
"rngMultiplier": 1103515245,
"rngIncrement": 12345,
"control": { "id": "control", "name": "现役素材", "ctrPermille": 26, "impressions": 60000 },
"testImpressionsPerCreative": 20000,
"cpmFen": 1600,
"ipmPermille": 140,
"minCtrPermille": 20,
"z2ThresholdX100": 384,
"scaleBudgetFen": 6000000,
"note": "素材测试矩阵参数，口径是「公开可观察机制」的建模假设，非任何渠道后台实测。control=现役对照组（CTR 千分比与曝光固定，无噪声）；每条测试素材给定 trueCtrPermille，点击 = imp*trueCtr//1000 + LCG 噪声（±3，模拟小样本抖动）。z2ThresholdX100=z²×100 阈值，384 对应双侧 95%。胜出判定用整数形式 d²·n1·n2·100 > 384·p̄·(1000-p̄)·(n1+n2)，无浮点无开方。真实 CTR 需真实业务环境，命令行不可复现（X）。",
"creatives": [
{ "id": "c01", "theme": "result", "trueCtrPermille": 34 }, { "id": "c02", "theme": "contrast", "trueCtrPermille": 31 }, { "id": "c03", "theme": "result", "trueCtrPermille": 29 }, { "id": "c04", "theme": "gameplay", "trueCtrPermille": 26 },
{ "id": "c05", "theme": "story", "trueCtrPermille": 23 }, { "id": "c06", "theme": "contrast", "trueCtrPermille": 21 }, { "id": "c07", "theme": "gameplay", "trueCtrPermille": 19 }, { "id": "c08", "theme": "story", "trueCtrPermille": 17 }
]
}
在本机跑 /opt/homebrew/bin/python3 sim/o03_matrix.py --config config/ --out out/，矩阵跑出来的结果（同机相对量级，绝对数值不可移植）：
| 素材 | 主题 | CTR‰ | 增量‰ | 判定 | 预算(分) | 预计安装 |
|---|---|---|---|---|---|---|
| c01 | result | 33 | 7 | 胜出 | 3500000 | 10115 |
| c02 | contrast | 31 | 5 | 胜出 | 2500000 | 6793 |
| c03 | result | 28 | 2 | 持平 | 0 | 0 |
| c04 | gameplay | 25 | -1 | 淘汰 | 0 | 0 |
| c05 | story | 23 | -3 | 淘汰 | 0 | 0 |
| c06 | contrast | 21 | -5 | 淘汰 | 0 | 0 |
（c07、c08 增量分别为 -8‰、-10‰，同为淘汰，此处从略。）对照组 CTR 是 26‰。注意 c03：它真实基线 29‰ 高于对照组，但小样本抖动把观测值压到 28‰，增量只剩 2‰——测不出显著，只能判持平。这不是算法保守，是数据量不够：20000 曝光下 1‰ 等于 20 次点击，2‰ 的差距淹没在噪声里；要把它测出来，唯一办法是加大曝光。
P 层内核：跨语言对拍，证明判定没写错
本套用一份 P 层 TypeScript 内核 core/src/o03/matrix.ts 复算同一个矩阵：同样的可播种 LCG、同样的整数 z 检验、同样的预算分配。它读同一份 config/o03_matrix.json，产出和 Python 逐字符一致的摘要。
// file: core/src/o03/matrix.ts
// tier: P
// verify: node --test 'core/test/o03_*.test.ts'
//
// 素材测试矩阵内核：与 sim/o03_matrix.py 逐位等价。整数 z 检验，不开方不用浮点；全部中间量 < 2^53，无需 BigInt；只用可擦除语法（禁 enum / namespace）。
export interface MatrixCreative { readonly id: string; readonly theme: string; readonly trueCtrPermille: number; }
export interface MatrixControl { readonly id: string; readonly ctrPermille: number; readonly impressions: number; }
export interface MatrixConfig {
readonly version: number; readonly game: string; readonly seed: number;
readonly rngMultiplier: number; readonly rngIncrement: number; readonly control: MatrixControl;
readonly testImpressionsPerCreative: number; readonly cpmFen: number; readonly ipmPermille: number;
readonly minCtrPermille: number; readonly z2ThresholdX100: number; readonly scaleBudgetFen: number;
readonly creatives: readonly MatrixCreative[];
}
export interface MatrixRow {
readonly id: string; readonly theme: string; readonly clicks: number;
readonly obsCtrPermille: number; readonly liftPermille: number; readonly verdict: string;
readonly winner: boolean; readonly cpiFen: number; budgetFen: number; installs: number;
}
export interface MatrixResult {
readonly controlCtrPermille: number; readonly rows: MatrixRow[]; readonly winnerCount: number;
readonly killedCount: number; readonly totalBudgetFen: number; readonly totalInstalls: number;
readonly digest: string;
}
/** 可播种整数 LCG（与 Python 侧同乘子/同增量，32 位回绕）。 */
export function makeRng(seed: number, mult: number, inc: number): () => number {
let state = seed >>> 0;
return () => {
state = (Math.imul(state, mult) + inc) >>> 0;
return state;
};
}
/** 整数双侧 z 检验：d²·n1·n2·100 > z²·100·p̄·(1000-p̄)·(n1+n2)。 */
export function isWinner(d: number, imp: number, clicks: number, cimp: number,
cclicks: number, z2x100: number): boolean {
if (d <= 0) return false;
const pooled = Math.floor(((clicks + cclicks) * 1000) / (imp + cimp));
const lhs = d * d * imp * cimp * 100;
const rhs = z2x100 * pooled * (1000 - pooled) * (imp + cimp);
return lhs > rhs;
}
export function run(cfg: MatrixConfig): MatrixResult {
const nxt = makeRng(cfg.seed, cfg.rngMultiplier, cfg.rngIncrement);
const cimp = cfg.control.impressions;
const cclicks = Math.floor((cimp * cfg.control.ctrPermille) / 1000);
const ctrControl = Math.floor((cclicks * 1000) / cimp);
const imp = cfg.testImpressionsPerCreative;
const rows: MatrixRow[] = cfg.creatives.map((c) => {
const clicks = Math.max(0, Math.floor((imp * c.trueCtrPermille) / 1000) + ((nxt() % 7) - 3));
const obs = Math.floor((clicks * 1000) / imp);
const d = obs - ctrControl;
const win = isWinner(d, imp, clicks, cimp, cclicks, cfg.z2ThresholdX100);
const verdict = obs < cfg.minCtrPermille || d <= 0 ? "淘汰" : win ? "胜出" : "持平";
const cpiFen = obs > 0 ? Math.floor((cfg.cpmFen * 1000) / (obs * cfg.ipmPermille)) : 1e9;
return { id: c.id, theme: c.theme, clicks, obsCtrPermille: obs, liftPermille: d,
verdict, winner: win && obs >= cfg.minCtrPermille, cpiFen, budgetFen: 0, installs: 0 };
});
const winners = rows.filter((r) => r.winner);
const totalLift = winners.reduce((s, r) => s + r.liftPermille, 0);
if (totalLift > 0) {
const rem: { r: number; id: string; row: MatrixRow }[] = [];
for (const r of winners) {
const exact = cfg.scaleBudgetFen * r.liftPermille;
r.budgetFen = Math.floor(exact / totalLift);
rem.push({ r: exact % totalLift, id: r.id, row: r });
}
const left = cfg.scaleBudgetFen - winners.reduce((s, r) => s + r.budgetFen, 0);
rem.sort((a, b) => (b.r - a.r) || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
for (let i = 0; i < left; i += 1) rem[i % rem.length].row.budgetFen += 1;
}
for (const r of winners) r.installs = r.cpiFen ? Math.floor(r.budgetFen / r.cpiFen) : 0;
const killedCount = rows.filter((r) => r.verdict === "淘汰").length;
const winIds = winners.map((r) => `${r.id}:${r.liftPermille}:${r.budgetFen}`).join(",");
return { controlCtrPermille: ctrControl, rows, winnerCount: winners.length, killedCount,
totalBudgetFen: winners.reduce((s, r) => s + r.budgetFen, 0),
totalInstalls: winners.reduce((s, r) => s + r.installs, 0),
digest: `control:${ctrControl}|winners:${winIds}|killed:${killedCount}` };
}
在本机跑 node --test 'core/test/o03_*.test.ts'，两套独立实现给出同一个摘要：
digest = control:26|winners:c01:7:3500000,c02:5:2500000|killed:5
只要有一边的 z 检验边界、LCG 参数或余数补齐写错，摘要立刻分叉。
X 层：真实显著性不可复现
上面算的是判定侧：给定点击与曝光，谁显著更高。下面这些属于 X 层，需真实业务环境，命令行不可复现：真实的素材点击率（受素材、人群、时段共同影响，平台侧不透明）；真实的归因与去重（同一用户被多条素材触达时算谁的）；真实的「持平」后续表现（继续观察的素材可能下一轮就显著）。工程上正确的做法是把可复现的判定层与不可复现的真实数据层分开——判定层做「这个样本量能不能判出胜负」的推演，真实数据层用来在投放后反标定 CTR 基线与噪声幅度。
自检表
| 检查项 | 合格标准 |
|---|---|
| 对照组明确 | 每条素材都和同一个 control 比，不比来比去 |
| 样本量够 | 曝光量写进 config，能算出可分辨的最小增量 |
| 判定可复算 | 整数 z 检验，同一输入必得同一结论 |
| 分配守恒 | Σ预算 = 放量预算，余数补齐规则明确 |
| 边界清晰 | 真实 CTR 与归因标 X，不写成「实测」 |
素材测试最危险的错觉，是把「这一轮 CTR 最高的那条」当成「应该 all in 的那条」。统计判定的价值恰恰在于拒绝在数据不够时下结论。下一节处理素材生命周期的最后一段：跑起来的素材会衰减，什么时候该换新。
阅读导航：上一节：3.1 素材生产线与批量产出 · 下一节：3.3 素材衰减与疲劳替换 。
继续阅读
探索更多技术文章
浏览归档，发现更多关于系统设计、工具链和工程实践的内容。