前两节解决了「怎么分」和「怎么判」，但一个显著的结果不等于该上线。这一节把三样东西合成一张决策规则表：护栏（不能变坏的底线指标）、显著性（差异是不是真的）、效应量（值不值得做），让「这次改动上不上」有据可依。
D 层：把决策规则落成配置
下面这份是 D 层（静态层）产物，全整数。它定义主指标、最小效应量、显著性水平、护栏清单（ARPDAU 允许下跌幅度、崩溃率允许上涨幅度），以及四个待决策实验的观测数据与护栏实测值。
{
"version": 1,
"game": "merge-island",
"seed": 20261012,
"note": "从实验到决策：把一组实验结果喂进一张决策规则表——先看护栏（guardrail）有没有破，再看显著性，再看效应是否达到最小可检测效应，最后才判 ship/iterate/keep_running/stop。护栏含 ARPDAU（分，允许下跌 maxDropPermille‰）与崩溃率（‰，允许上涨 maxRisePermille‰）。每个实验的观测数、转化数、护栏实测值都是假设数据，真实值需真实业务环境，命令行不可复现（X）。全整数，禁浮点。",
"primaryMetric": "d7RetentionPermille",
"minEffectPermille": 5,
"alphaPermille": 50,
"zCritMilli": 1960,
"guardrails": [
{ "id": "arpdauCent", "name": "ARPDAU(分)", "baseline": 24, "maxDropPermille": 50 },
{ "id": "crashPermille", "name": "崩溃率‰", "baseline": 3, "maxRisePermille": 1000 }
],
"experiments": [
{ "id": "exp1", "name": "插屏降频次", "n1": 40000, "conv1": 4800, "n2": 40000, "conv2": 5160, "guardrailMeasured": { "arpdauCent": 23, "crashPermille": 3 } },
{ "id": "exp2", "name": "首屏奖励翻倍", "n1": 40000, "conv1": 4800, "n2": 40000, "conv2": 4850, "guardrailMeasured": { "arpdauCent": 24, "crashPermille": 3 } },
{ "id": "exp3", "name": "强弹窗召回", "n1": 40000, "conv1": 4800, "n2": 40000, "conv2": 4980, "guardrailMeasured": { "arpdauCent": 19, "crashPermille": 3 } },
{ "id": "exp4", "name": "新手指引重写", "n1": 40000, "conv1": 4800, "n2": 40000, "conv2": 4620, "guardrailMeasured": { "arpdauCent": 24, "crashPermille": 4 } }
]
}
S 层：一张决策规则表
下面这段是 S 层脚本，读 config/l08_decision.json，对每个实验先查护栏、再做 z 检验、最后按「护栏 → 显著 → 方向 → 效应量」的顺序给结论，产出 JSON + CSV + SVG；在本机跑它，四个实验正好给出四种结论（仿真数值为同机相对量级，绝对数值不可移植）：
#!/usr/bin/env python3
# file: sim/l08_decision.py
# tier: S
# verify: /opt/homebrew/bin/python3 sim/l08_decision.py --config config/ --out out/
"""从实验到决策：读 config/l08_decision.json，把一组实验结果喂进一张决策规则表。
顺序：① 护栏（ARPDAU 下跌 / 崩溃率上涨）破了 → stop；② 主指标不显著 → keep_running；
③ 显著但变差 → stop；④ 显著但效应不足 minEffectPermille → iterate；⑤ 否则 ship。
产出 JSON + CSV + SVG。所有观测数据均为假设值，真实值需真实业务环境，命令行不可复现（X）。"""
from __future__ import annotations
import argparse
import csv
import json
import math
import os
import xml.etree.ElementTree as ET
from l08_significance import two_proportion_z
COLOR = {"ship": "#16a34a", "iterate": "#f59e0b", "keep_running": "#94a3b8", "stop": "#dc2626"}
def load(d: str) -> dict:
with open(os.path.join(d, "l08_decision.json"), encoding="utf-8") as f:
return json.load(f)
def guardrail_breach(cfg: dict, measured: dict) -> tuple:
"""逐个护栏判断是否越界；返回 (是否破护栏, 越界明细)。"""
hits = []
for g in cfg["guardrails"]:
val = measured.get(g["id"], g["baseline"])
if "maxDropPermille" in g and val < g["baseline"] * (1000 - g["maxDropPermille"]) // 1000:
hits.append({"id": g["id"], "value": val, "kind": "drop"})
if "maxRisePermille" in g and val > g["baseline"] * (1000 + g["maxRisePermille"]) // 1000:
hits.append({"id": g["id"], "value": val, "kind": "rise"})
return bool(hits), hits
def decide(cfg: dict, z: dict, breached: bool) -> str:
if breached:
return "stop"
if z["pValue"] >= cfg["alphaPermille"] / 1000.0:
return "keep_running"
if z["effectPermille"] <= 0:
return "stop"
if z["effectPermille"] < cfg["minEffectPermille"]:
return "iterate"
return "ship"
def draw_svg(path: str, rows: list, min_effect: int) -> int:
W, PAD, ROW = 720, 210, 46
H, span = 70 + ROW * len(rows), 720 - 210 - 60
peak = max((abs(r["effectPermille"]) for r in rows), default=1) or 1
zero = PAD + span // 2
svg = ET.Element("svg", {"xmlns": "http://www.w3.org/2000/svg", "width": str(W),
"height": str(H), "viewBox": f"0 0 {W} {H}"})
ET.SubElement(svg, "rect", {"width": str(W), "height": str(H), "fill": "#ffffff"})
ET.SubElement(svg, "text", {"x": "16", "y": "28", "font-size": "15", "fill": "#111827"}).text = \
f"各实验的效应（‰）与决策（虚线 = 最小效应 {min_effect}‰）"
for x, color, dash in ((zero, "#9ca3af", None), (zero + span * min_effect // (2 * peak), "#f59e0b", "4 3")):
attr = {"x1": str(x), "y1": "44", "x2": str(x), "y2": str(50 + ROW * len(rows)),
"stroke": color}
if dash:
attr["stroke-dasharray"] = dash
ET.SubElement(svg, "line", attr)
for i, r in enumerate(rows):
y = 54 + i * ROW
w = span * abs(r["effectPermille"]) // (2 * peak)
ET.SubElement(svg, "text", {"x": "16", "y": str(y + 20), "font-size": "12",
"fill": "#374151"}).text = r["name"]
ET.SubElement(svg, "rect", {"x": str(zero if r["effectPermille"] >= 0 else zero - w),
"y": str(y + 6), "width": str(w), "height": "22",
"rx": "4", "fill": COLOR[r["decision"]]})
ET.SubElement(svg, "text", {"x": str(zero + span // 2 + 6), "y": str(y + 22), "font-size": "12",
"fill": "#6b7280"}).text = f'{r["effectPermille"]}‰ · {r["decision"]}'
os.makedirs(os.path.dirname(path), exist_ok=True)
ET.ElementTree(svg).write(path, encoding="utf-8", xml_declaration=True)
return os.path.getsize(path)
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--config", default="config/")
ap.add_argument("--out", default="out/")
a = ap.parse_args()
cfg = load(a.config)
zc = cfg["zCritMilli"] / 1000.0
rows = []
for e in cfg["experiments"]:
z = two_proportion_z(e["n1"], e["conv1"], e["n2"], e["conv2"])
p1, p2 = e["conv1"] / e["n1"], e["conv2"] / e["n2"]
se = math.sqrt(p1 * (1 - p1) / e["n1"] + p2 * (1 - p2) / e["n2"])
eff = z["effectPermille"] / 1000.0
breached, hits = guardrail_breach(cfg, e["guardrailMeasured"])
rows.append({"id": e["id"], "name": e["name"], "effectPermille": round(z["effectPermille"], 2),
"pValue": round(z["pValue"], 6),
"ciLowPermille": round((eff - zc * se) * 1000, 2),
"ciHighPermille": round((eff + zc * se) * 1000, 2),
"guardrailBreach": breached, "guardrailHits": hits,
"decision": decide(cfg, z, breached)})
summary = {d: sum(1 for r in rows if r["decision"] == d)
for d in ("ship", "iterate", "keep_running", "stop")}
os.makedirs(a.out, exist_ok=True)
svg = draw_svg(os.path.join(a.out, "l08_decision.svg"), rows, cfg["minEffectPermille"])
with open(os.path.join(a.out, "l08_decision.csv"), "w", encoding="utf-8", newline="") as f:
w = csv.writer(f)
w.writerow(["id", "name", "effectPermille", "pValue", "ciLowPermille",
"ciHighPermille", "guardrailBreach", "decision"])
for r in rows:
w.writerow([r["id"], r["name"], r["effectPermille"], r["pValue"],
r["ciLowPermille"], r["ciHighPermille"], r["guardrailBreach"], r["decision"]])
rep = {"version": cfg["version"], "game": cfg["game"], "seed": cfg["seed"],
"primaryMetric": cfg["primaryMetric"], "minEffectPermille": cfg["minEffectPermille"],
"alphaPermille": cfg["alphaPermille"], "experiments": rows, "summary": summary,
"csv": os.path.join(a.out, "l08_decision.csv"),
"svg": os.path.join(a.out, "l08_decision.svg"), "svgBytes": svg}
with open(os.path.join(a.out, "l08_decision.json"), "w", encoding="utf-8") as f:
json.dump(rep, f, ensure_ascii=False, indent=2)
print(json.dumps({"summary": summary,
"decisions": [(r["id"], r["effectPermille"], r["pValue"],
r["guardrailBreach"], r["decision"]) for r in rows]},
ensure_ascii=False))
return 0
if __name__ == "__main__":
raise SystemExit(main())
在本机跑它，四个实验给出四种结论（仿真数值为同机相对量级，绝对数值不可移植）：
{"summary": {"ship": 1, "iterate": 0, "keep_running": 1, "stop": 2}, "decisions": [["exp1", 9.0, 0.000116, false, "ship"], ["exp2", 1.25, 0.587285, false, "keep_running"], ["exp3", 4.5, 0.052046, true, "stop"], ["exp4", -4.5, 0.048328, false, "stop"]]}
逐条读：exp1 效应 +9‰、p≈0.000116、护栏没破 → ship；exp2 效应仅 +1.25‰、p≈0.587 不显著 → keep_running（样本不够，继续跑）；exp3 效应 +4.5‰ 且护栏破了（ARPDAU 从 24 跌到 19，超过允许的 5% 跌幅）→ stop，哪怕主指标是正的也一票否决；exp4 效应 -4.5‰ 显著为负 → stop。四种结论各对应一条规则，这正是决策表的用处：把「该不该上」变成一张查表。
W 层：可交互的决策看板
把上面的内核打包成一个能在浏览器里拖动的看板，能直观看到「样本量、p 值、决策」怎么联动。下面两段是 W 层构建脚本与 Playwright 无头验收脚本。
#!/usr/bin/env node
// file: web/build_l08_experiment.mjs
// tier: W
// verify: node web/build_l08_experiment.mjs
// 把 core/src/l08/split.ts + core/src/l08/stats.ts 打成单文件 HTML（A/B 实验决策看板）。复用 pack.mjs 的擦除逻辑。
import { readFileSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { bundle, page, writePage, inlineConfig } from "./pack.mjs";
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const experiment = JSON.parse(readFileSync(join(ROOT, "config", "l08_experiment.json"), "utf8"));
const significance = JSON.parse(readFileSync(join(ROOT, "config", "l08_significance.json"), "utf8"));
const cfg = { experiment, significance };
const js = bundle(ROOT, ["core/src/l08/split.ts", "core/src/l08/stats.ts"]);
const script = `const cfg = ${inlineConfig(cfg)};
${js}
const expCfg = cfg.experiment, sigCfg = cfg.significance;
const cv = document.getElementById("stage"), g = cv.getContext("2d");
let state = { baselinePermille: sigCfg.baselineRatePermille, mdePermille: sigCfg.mdePermille,
alphaPermille: sigCfg.alphaPermille, powerPermille: sigCfg.powerPermille,
nPerArm: sigCfg.observed.control.n, controlPermille: 120, treatPermille: 129 };
let d = null;
function compute() {
const requiredPerArm = sampleSizePerArm(state.baselinePermille, state.mdePermille, state.alphaPermille, state.powerPermille);
const daysNeeded = Math.ceil(requiredPerArm / sigCfg.dailyUsersPerArm);
const c1 = Math.floor(state.nPerArm * state.controlPermille / 1000);
const c2 = Math.floor(state.nPerArm * state.treatPermille / 1000);
const zr = twoProportionZ(state.nPerArm, c1, state.nPerArm, c2, 1.96);
const decision = decide(zr.pValue, zr.effectPermille, 5, state.alphaPermille, false);
const t = tallyArms(expCfg, expCfg.populationUsers);
const srm = srmTally(expCfg, t.assigned, t.logged);
const maxDev = Math.max(...srm.map((r) => r.deviationPermille));
d = { requiredPerArm, daysNeeded, c1, c2, z: zr.z, pValue: zr.pValue, effectPermille: zr.effectPermille,
ciLow: zr.ciLowPermille, ciHigh: zr.ciHighPermille, decision, srm, maxDev,
srmPass: maxDev <= expCfg.srmTolerancePermille, assigned: t.assigned, logged: t.logged };
draw();
}
function draw() {
const W = cv.width, H = cv.height;
g.clearRect(0, 0, W, H);
g.font = "13px system-ui";
const maxN = Math.max(state.nPerArm, d.requiredPerArm, 1), bw = W - 210;
g.fillStyle = "#e2e8f0"; g.fillText("样本进度", 8, 18); g.fillStyle = "#94a3b8"; g.fillText("已有 " + state.nPerArm, 8, 46);
g.fillStyle = "#38bdf8"; g.fillRect(96, 30, Math.floor(bw * state.nPerArm / maxN), 20);
g.fillStyle = "#94a3b8"; g.fillText("所需 " + d.requiredPerArm, 8, 78); g.fillStyle = "#334155"; g.fillRect(96, 62, Math.floor(bw * d.requiredPerArm / maxN), 20);
g.fillStyle = "#e2e8f0"; g.fillText("效应 95% CI (‰)", 8, 112);
const span = Math.max(Math.abs(d.ciHigh), Math.abs(d.ciLow), 1) * 1.2, cx = W / 2, sc = (W - 80) / (2 * span);
g.strokeStyle = "#475569"; g.beginPath(); g.moveTo(cx - span * sc, 130); g.lineTo(cx + span * sc, 130); g.stroke();
g.strokeStyle = "#9ca3af"; g.beginPath(); g.moveTo(cx, 122); g.lineTo(cx, 138); g.stroke();
g.strokeStyle = d.decision === "ship" ? "#16a34a" : d.decision === "stop" ? "#dc2626" : "#f59e0b";
g.lineWidth = 3; g.beginPath(); g.moveTo(cx + d.ciLow * sc, 130); g.lineTo(cx + d.ciHigh * sc, 130); g.stroke();
g.lineWidth = 1; g.fillStyle = "#38bdf8";
g.beginPath(); g.arc(cx + d.effectPermille * sc, 130, 4, 0, 6.3); g.fill();
g.fillStyle = "#94a3b8"; g.fillText("0", cx - 4, 152);
document.getElementById("required").textContent = String(d.requiredPerArm);
document.getElementById("days").textContent = String(d.daysNeeded);
document.getElementById("pval").textContent = d.pValue.toFixed(4);
document.getElementById("decision").textContent = d.decision;
document.getElementById("srmdev").textContent = d.maxDev + "‰" + (d.srmPass ? " (通过)" : " (SRM 未通过)");
globalThis.__pebble = { requiredPerArm: d.requiredPerArm, daysNeeded: d.daysNeeded, z: d.z, pValue: d.pValue,
effectPermille: d.effectPermille, decision: d.decision, maxDev: d.maxDev, srmPass: d.srmPass,
assigned: d.assigned, logged: d.logged, canvasDrawn: true, api };
}
const api = {
stats(s) { Object.assign(state, s); compute(); return { requiredPerArm: d.requiredPerArm, daysNeeded: d.daysNeeded }; },
evaluate(s) { Object.assign(state, s); compute(); return { z: d.z, pValue: d.pValue, effectPermille: d.effectPermille, decision: d.decision }; },
split() { return { assigned: d.assigned, logged: d.logged, maxDev: d.maxDev, srmPass: d.srmPass }; },
get() { return { requiredPerArm: d.requiredPerArm, pValue: d.pValue, effectPermille: d.effectPermille, decision: d.decision, maxDev: d.maxDev }; },
};
compute();
`;
const body = ` <h1>《合成小岛》· A/B 实验决策看板</h1>
<div class="sub">分流内核 core/src/l08/split.ts + 显著性内核 core/src/l08/stats.ts：全整数哈希 + 自实现 z 检验，无 scipy、无 Math.random</div>
<div class="card row"><div><div class="k">每臂最小样本量</div><div class="big" id="required">0</div></div><div><div class="k">预计天数</div><div class="big" id="days">0</div></div><div><div class="k">p 值</div><div class="big" id="pval">0</div></div><div><div class="k">决策</div><div class="big" id="decision">-</div></div></div>
<div class="card"><canvas id="stage" width="480" height="170"></canvas></div>
<div class="sub">SRM 偏差：<span id="srmdev">-</span>（分配均衡，但 treat 臂日志丢失 30‰ 后样本比例失衡）</div>`;
const bytes = writePage(join(ROOT, "web/dist/l08.html"), page({ title: "《合成小岛》A/B 实验决策看板 · l08", body, script, css: "#stage { width: 480px; height: 170px; }" }));
console.log(JSON.stringify({ html: "web/dist/l08.html", htmlBytes: bytes, bundleLines: js.split("\n").length }));
#!/usr/bin/env node
// file: web/verify_l08_experiment.mjs
// tier: W
// verify: NODE_PATH=/opt/homebrew/lib/node_modules node web/verify_l08_experiment.mjs web/dist/l08.html
// W 层验收：Playwright 无头打开单文件 HTML，调 api 改样本量/转化率，读回断言 + 截图。
// playwright 用 createRequire 加载——NODE_PATH 对 ESM 的 import 无效。
import { createRequire } from "node:module";
import { pathToFileURL, fileURLToPath } from "node:url";
import { dirname, join, resolve } from "node:path";
import { mkdirSync } from "node:fs";
import assert from "node:assert/strict";
const { chromium } = createRequire(import.meta.url)("playwright");
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const url = pathToFileURL(resolve(ROOT, process.argv[2] ?? "web/dist/l08.html")).href;
mkdirSync(join(ROOT, "out/shots"), { recursive: true });
const browser = await chromium.launch();
const warm = await browser.newPage(); await warm.goto(url); await warm.close(); // 预热：冷启动首个 page 读 canvas 像素恒为空白
const errors = [];
const p = await browser.newPage({ viewport: { width: 800, height: 700 } });
p.on("pageerror", (e) => errors.push(String(e)));
await p.goto(url);
await p.waitForFunction(() => globalThis.__pebble?.canvasDrawn === true, null, { timeout: 10000 });
const base = await p.evaluate(() => globalThis.__pebble.api.get());
const keep = await p.evaluate(() => globalThis.__pebble.api.evaluate({ treatPermille: 121 }));
const stop = await p.evaluate(() => globalThis.__pebble.api.evaluate({ treatPermille: 108 }));
const split = await p.evaluate(() => globalThis.__pebble.api.split());
const stats = await p.evaluate(() => globalThis.__pebble.api.stats({ mdePermille: 8 }));
const domDecision = (await p.textContent("#decision")).trim();
const domSrm = (await p.textContent("#srmdev")).trim();
const painted = await p.evaluate(() => {
const d = document.getElementById("stage").getContext("2d").getImageData(0, 0, 480, 170).data;
let n = 0; for (let i = 0; i < d.length; i += 4) if (d[i + 3] > 0 && d[i] < 250) n += 1; return n;
});
await p.screenshot({ path: join(ROOT, "out/shots/l08_experiment.png"), fullPage: true });
await p.close();
const p2 = await browser.newPage();
await p2.goto(url);
await p2.waitForFunction(() => globalThis.__pebble?.canvasDrawn === true, null, { timeout: 10000 });
const rerun = await p2.evaluate(() => globalThis.__pebble.api.get());
await p2.close();
await browser.close();
assert.equal(errors.length, 0, `页面错误: ${errors.join("; ")}`);
assert.equal(base.requiredPerArm, 11512); assert.equal(base.decision, "ship");
assert.ok(Math.abs(base.pValue - 0.000116) < 1e-5, "默认 p 值应≈0.000116");
assert.equal(keep.decision, "keep_running"); assert.equal(stop.decision, "stop");
assert.equal(split.assigned.control, 100071); assert.equal(split.assigned.treat, 99929);
assert.equal(split.logged.treat, 96892); assert.equal(split.maxDev, 16); assert.equal(split.srmPass, false);
assert.equal(stats.requiredPerArm, 25902); assert.equal(domDecision, "stop");
assert.ok(domSrm.includes("SRM 未通过")); assert.ok(painted > 300, `canvas 绘制 ${painted}px`);
assert.deepEqual(rerun.requiredPerArm, base.requiredPerArm, "同一次加载应可复现");
console.log(JSON.stringify({
page: "web/dist/l08.html", base, keep, stop, split, stats, domDecision, domSrm,
canvasPaintedPx: painted, rerunMatches: true, screenshot: "out/shots/l08_experiment.png",
}));
console.log("SMOKE-W8: PASS");
本机跑 NODE_PATH=/opt/homebrew/lib/node_modules node web/verify_l08_experiment.mjs web/dist/l08.html，最后打印 SMOKE-W8: PASS：默认每臂样本量 11512、默认判 ship、p 值≈0.000116；改 treatPermille 为 121 判 keep_running、为 108 判 stop；split 读回 100071 / 99929 / 96892 与 16‰，stats({ mdePermille: 8 }) 得 25902——与 8.1、8.2 的 S 层逐位一致。warm 那一行是必需的预热：冷启动首个 page 读 canvas 像素恒为空白，必须先开一个页面丢弃。
小结与下一步
这一节把实验闭环补齐了：分流（8.1）保证可比、显著性（8.2）保证结论可信、决策表（8.3）保证结论可执行。一条规则值得记牢——护栏优先：主指标再好，只要底线指标（ARPDAU、崩溃率）破了，直接 stop；实验的意义不是「证明我对」，而是「在可控风险下把决策交给数据」。实验跑通、结论落定，下一步就是把它真正推上线——上线不是「点一下发布」，要走审核、灰度、回滚，还要有一份能照着走的清单，这是下一章的主题。
阅读导航：上一节：显著性、样本量与常见陷阱 · 下一节：上线评审与发布流程 。
继续阅读
探索更多技术文章
浏览归档，发现更多关于系统设计、工具链和工程实践的内容。