《互联网产品的服务端架构·交易、增长与平台化》8.2 指标、显著性与误判

se = sqrt(p * (1-p) * (1/n1 + 1/n2))
z  = (p2 - p1) / se

// file: internal/experiment/stats.go
// tier: L1
// verify: go test ./internal/experiment/

package experiment

import (
	"errors"
	"fmt"
	"math"
)

var (
	// ErrInvalidSample 表示样本量为非正数，或成功数超出 0..N。
	ErrInvalidSample = errors.New("experiment: invalid sample")
	// ErrZeroStandardError 表示合并标准误为 0，z 值没有定义。
	ErrZeroStandardError = errors.New("experiment: zero standard error")
)

// Sample 是一组实验数据：N 是样本量，Success 是成功数（点击、下单、转化）。
type Sample struct {
	N       int
	Success int
}

// Rate 返回样本的成功率。N 必须大于 0，Success 必须落在 0..N。
// 调用方不要自己用 Success/N 绕开校验，否则零样本会被算成 0%。
func Rate(s Sample) (float64, error) {
	if s.N <= 0 {
		return 0, fmt.Errorf("%w: n=%d", ErrInvalidSample, s.N)
	}
	if s.Success < 0 || s.Success > s.N {
		return 0, fmt.Errorf("%w: success=%d n=%d", ErrInvalidSample, s.Success, s.N)
	}
	return float64(s.Success) / float64(s.N), nil
}

// Z 用双比例 z 检验比较实验组与对照组。返回 (实验组转化率 - 对照组转化率)
// 除以合并标准误。z 为正表示实验组更高，为负表示更低。
//
// 这里使用合并比例 p=(x1+x2)/(n1+n2)，标准误为
// sqrt(p*(1-p)*(1/n1+1/n2))。两组都全成功或都全失败时标准误为 0，
// 此时差异没有统计意义，返回 ErrZeroStandardError 而不是 NaN。
func Z(control, treat Sample) (float64, error) {
	pc, err := Rate(control)
	if err != nil {
		return 0, fmt.Errorf("control: %w", err)
	}
	pt, err := Rate(treat)
	if err != nil {
		return 0, fmt.Errorf("treat: %w", err)
	}

	n1 := float64(control.N)
	n2 := float64(treat.N)
	pooled := (float64(control.Success) + float64(treat.Success)) / (n1 + n2)
	se := math.Sqrt(pooled * (1 - pooled) * (1/n1 + 1/n2))
	if se == 0 {
		return 0, ErrZeroStandardError
	}
	return (pt - pc) / se, nil
}

// Significant 判断 z 的绝对值是否达到阈值。threshold 必须为正；非正阈值
// 返回 false，表示调用方没有给出有效的显著性标准，不能据此上线。
func Significant(z, threshold float64) bool {
	if threshold <= 0 {
		return false
	}
	return math.Abs(z) >= threshold
}

// file: internal/experiment/stats_test.go
// tier: L1
// verify: go test ./internal/experiment/

package experiment

import (
	"errors"
	"math"
	"testing"
)

func TestRate(t *testing.T) {
	got, err := Rate(Sample{N: 1000, Success: 120})
	if err != nil {
		t.Fatalf("Rate = %v, want nil", err)
	}
	if math.Abs(got-0.12) > 1e-9 {
		t.Fatalf("Rate = %.6f, want 0.12", got)
	}
	if got, err := Rate(Sample{N: 10, Success: 0}); err != nil || got != 0 {
		t.Fatalf("Rate(0/10) = %.6f, %v; want 0, nil", got, err)
	}
	if got, err := Rate(Sample{N: 10, Success: 10}); err != nil || got != 1 {
		t.Fatalf("Rate(10/10) = %.6f, %v; want 1, nil", got, err)
	}
}

func TestRateRejectsBadSample(t *testing.T) {
	for _, s := range []Sample{{N: 0}, {N: -1}, {N: 10, Success: -1}, {N: 10, Success: 11}} {
		if _, err := Rate(s); !errors.Is(err, ErrInvalidSample) {
			t.Fatalf("Rate(%+v) = %v, want ErrInvalidSample", s, err)
		}
	}
}

func TestZKnownDifference(t *testing.T) {
	// p1=0.10, p2=0.12, pooled=0.11, se≈0.013993, z≈1.4293。
	z, err := Z(Sample{N: 1000, Success: 100}, Sample{N: 1000, Success: 120})
	if err != nil {
		t.Fatalf("Z = %v, want nil", err)
	}
	if math.Abs(z-1.4293) > 1e-3 {
		t.Fatalf("Z = %.4f, want 1.4293", z)
	}
}

func TestZSymmetric(t *testing.T) {
	a := Sample{N: 800, Success: 96}
	b := Sample{N: 820, Success: 110}
	ab, err := Z(a, b)
	if err != nil {
		t.Fatalf("Z(a,b) = %v, want nil", err)
	}
	ba, _ := Z(b, a)
	if math.Abs(ab+ba) > 1e-9 {
		t.Fatalf("Z(a,b)=%.6f, Z(b,a)=%.6f, want opposite signs", ab, ba)
	}
}

func TestZZeroStandardError(t *testing.T) {
	if _, err := Z(Sample{N: 100, Success: 0}, Sample{N: 100, Success: 0}); !errors.Is(err, ErrZeroStandardError) {
		t.Fatalf("both zero = %v, want ErrZeroStandardError", err)
	}
	if _, err := Z(Sample{N: 100, Success: 100}, Sample{N: 100, Success: 100}); !errors.Is(err, ErrZeroStandardError) {
		t.Fatalf("both full = %v, want ErrZeroStandardError", err)
	}
	if _, err := Z(Sample{N: 0}, Sample{N: 100, Success: 10}); !errors.Is(err, ErrInvalidSample) {
		t.Fatalf("bad control = %v, want ErrInvalidSample", err)
	}
}

func TestSignificant(t *testing.T) {
	if !Significant(1.96, 1.96) {
		t.Fatal("Significant(1.96, 1.96) = false, want true")
	}
	if Significant(1.959, 1.96) {
		t.Fatal("Significant(1.959, 1.96) = true, want false")
	}
	if !Significant(-2.5, 1.96) {
		t.Fatal("Significant(-2.5, 1.96) = false, want true")
	}
	if Significant(5, 0) || Significant(5, -1) {
		t.Fatal("non-positive threshold should never be significant")
	}
}

cd .local/checks/webapp-lab && go test ./internal/experiment/ -run 'TestRate|TestZ|TestSignificant' -count=1 -v

=== RUN   TestRate
--- PASS: TestRate (0.00s)
=== RUN   TestRateRejectsBadSample
--- PASS: TestRateRejectsBadSample (0.00s)
=== RUN   TestZKnownDifference
--- PASS: TestZKnownDifference (0.00s)
=== RUN   TestZSymmetric
--- PASS: TestZSymmetric (0.00s)
=== RUN   TestZZeroStandardError
--- PASS: TestZZeroStandardError (0.00s)
=== RUN   TestSignificant
--- PASS: TestSignificant (0.00s)
PASS
ok  	webapp/internal/experiment	0.270s