《后端架构设计·演进与治理》8.2 演练设计：稳态假设、爆炸半径与止损

// file: internal/platform/chaos/experiment.go
// tier: L1
// verify: go test ./internal/platform/chaos/ -count=1
package chaos

import (
	"fmt"
	"math"
	"time"
)

// Signal is one measured quantity. Guard signals abort; non-guard signals
// disprove the hypothesis without stopping the run.
type Signal struct {
	Name      string
	Guard     bool
	Baseline  float64
	Tolerance float64
	Breach    int
}

type Sample struct {
	At     time.Time
	Values map[string]float64
}

type Verdict struct {
	Sufficient   bool
	HypothesisOK bool
	Abort        bool
	Reason       string
}

type Hypothesis struct {
	Window     time.Duration
	MinSamples int
	Signals    []Signal
}

func (h Hypothesis) validate() error {
	if h.Window <= 0 || h.MinSamples < 1 || len(h.Signals) == 0 {
		return fmt.Errorf("chaos: hypothesis needs a window, samples and signals")
	}
	seen := map[string]bool{}
	var claim, guard bool
	for _, s := range h.Signals {
		if s.Name == "" || seen[s.Name] || !(s.Tolerance > 0) || s.Breach < 1 {
			return fmt.Errorf("chaos: invalid signal %q", s.Name)
		}
		seen[s.Name] = true
		claim, guard = claim || !s.Guard, guard || s.Guard
	}
	if !claim || !guard {
		return fmt.Errorf("chaos: hypothesis needs a claim and a guardrail")
	}
	return nil
}

// Evaluate refuses to conclude from too few samples.
func Evaluate(h Hypothesis, samples []Sample) (Verdict, error) {
	if err := h.validate(); err != nil {
		return Verdict{}, err
	}
	return evaluate(h, samples)
}

func evaluate(h Hypothesis, samples []Sample) (Verdict, error) {
	v := Verdict{Sufficient: len(samples) >= h.MinSamples, HypothesisOK: true}
	if len(samples) == 0 {
		return v, nil
	}
	start := samples[0].At
	for i, s := range samples {
		if i > 0 && !s.At.After(samples[i-1].At) {
			return Verdict{}, fmt.Errorf("chaos: samples are out of order")
		}
		if s.At.Sub(start) > h.Window {
			return Verdict{}, fmt.Errorf("chaos: sample is outside the window")
		}
		for _, sig := range h.Signals {
			if _, ok := s.Values[sig.Name]; !ok {
				return Verdict{}, fmt.Errorf("chaos: sample is missing %q", sig.Name)
			}
		}
	}
	type state struct{ run, peak int; breach bool }
	states := map[string]*state{}
	for _, sig := range h.Signals {
		states[sig.Name] = &state{}
	}
	for _, s := range samples {
		for _, sig := range h.Signals {
			st := states[sig.Name]
			if math.Abs(s.Values[sig.Name]-sig.Baseline) > sig.Tolerance {
				st.run++
			} else {
				st.run = 0
			}
			if st.run > st.peak {
				st.peak = st.run
			}
			if st.run >= sig.Breach {
				st.breach = true
			}
		}
	}
	for _, sig := range h.Signals {
		st := states[sig.Name]
		if !st.breach {
			continue
		}
		reason := fmt.Sprintf("%s breached %d consecutive samples", sig.Name, st.peak)
		if sig.Guard {
			v.Abort, v.Reason = true, "guardrail "+reason
			break
		}
		v.HypothesisOK, v.Reason = false, "hypothesis "+reason
	}
	return v, nil
}

type Action string

const (
	ContinueAction Action = "continue"
	PassAction     Action = "pass"
	FailAction     Action = "fail"
	AbortAction    Action = "abort"
)

// StopLoss is the independent stop path.
type StopLoss struct {
	Auto        bool
	OnCall      string
	MaxDuration time.Duration
	AbortWithin time.Duration
}

// Check fails closed when the stop path or its deadline is not usable.
func (s StopLoss) Check(v Verdict, elapsed time.Duration) (Action, string) {
	if !s.Auto && s.OnCall == "" {
		return AbortAction, "no out-of-band stop path"
	}
	if s.AbortWithin <= 0 || s.AbortWithin >= s.MaxDuration {
		return AbortAction, "abort deadline is outside the run"
	}
	if elapsed > s.MaxDuration {
		return AbortAction, "max duration exceeded"
	}
	if v.Abort {
		return AbortAction, v.Reason
	}
	if !v.Sufficient {
		return ContinueAction, "need more samples"
	}
	if !v.HypothesisOK {
		return FailAction, v.Reason
	}
	return PassAction, "hypothesis held"
}

// file: internal/platform/chaos/experiment_test.go
// tier: L1
// verify: go test ./internal/platform/chaos/ -count=1
package chaos

import (
	"strings"
	"testing"
	"time"
)

func at(hour, minute int) time.Time {
	return time.Date(2026, 10, 30, hour, minute, 0, 0, time.UTC)
}

func orderHypothesis() Hypothesis {
	return Hypothesis{
		Window: 10 * time.Minute, MinSamples: 4,
		Signals: []Signal{
			{Name: "order_success", Guard: false, Baseline: 0.995, Tolerance: 0.004, Breach: 2},
			{Name: "p99_ms", Guard: true, Baseline: 300, Tolerance: 100, Breach: 2},
		},
	}
}

func sampleAt(minute int, success, p99 float64) Sample {
	return Sample{
		At:     at(10, 0).Add(time.Duration(minute) * time.Minute),
		Values: map[string]float64{"order_success": success, "p99_ms": p99},
	}
}

func TestEvaluateConsecutiveBreach(t *testing.T) {
	h := orderHypothesis()
	one := []Sample{sampleAt(1, .995, 300), sampleAt(2, .995, 450), sampleAt(3, .995, 300), sampleAt(4, .995, 300)}
	v, err := Evaluate(h, one)
	if err != nil || v.Abort || !v.HypothesisOK {
		t.Fatalf("single spike acted: %+v, %v", v, err)
	}
	two := []Sample{sampleAt(1, .995, 300), sampleAt(2, .995, 450), sampleAt(3, .995, 450), sampleAt(4, .995, 300)}
	v, err = Evaluate(h, two)
	if err != nil || !v.Abort || !strings.Contains(v.Reason, "p99_ms") {
		t.Fatalf("two spikes: %+v, %v", v, err)
	}
	t.Logf("one spike: abort=false; two spikes: abort=true reason=%q", v.Reason)
}

func TestStopLossCheck(t *testing.T) {
	h := orderHypothesis()
	healthy, _ := Evaluate(h, []Sample{sampleAt(1, .995, 300), sampleAt(2, .995, 300), sampleAt(3, .995, 300), sampleAt(4, .995, 300)})
	guard, _ := Evaluate(h, []Sample{sampleAt(1, .995, 300), sampleAt(2, .995, 450), sampleAt(3, .995, 450), sampleAt(4, .995, 300)})
	claim, _ := Evaluate(h, []Sample{sampleAt(1, .995, 300), sampleAt(2, .995, 300), sampleAt(3, .980, 300), sampleAt(4, .980, 300)})
	short, _ := Evaluate(h, []Sample{sampleAt(1, .995, 300), sampleAt(2, .995, 300)})
	ok := StopLoss{Auto: true, MaxDuration: 8 * time.Minute, AbortWithin: 30 * time.Second}
	cases := []struct {
		name    string
		stop    StopLoss
		v       Verdict
		elapsed time.Duration
		want    Action
		reason  string
	}{
		{"healthy", ok, healthy, 4 * time.Minute, PassAction, "hypothesis held"},
		{"short", ok, short, time.Minute, ContinueAction, "more samples"},
		{"claim", ok, claim, 4 * time.Minute, FailAction, "order_success"},
		{"guard", ok, guard, 3 * time.Minute, AbortAction, "p99_ms"},
		{"no_path", StopLoss{}, healthy, time.Minute, AbortAction, "stop path"},
		{"bad_deadline", StopLoss{Auto: true, MaxDuration: 8 * time.Minute, AbortWithin: 9 * time.Minute}, healthy, time.Minute, AbortAction, "deadline"},
		{"overrun", ok, healthy, 9 * time.Minute, AbortAction, "max duration"},
	}
	for _, tc := range cases {
		action, reason := tc.stop.Check(tc.v, tc.elapsed)
		if action != tc.want || !strings.Contains(reason, tc.reason) {
			t.Fatalf("%s: %s %q, want %s %q", tc.name, action, reason, tc.want, tc.reason)
		}
		t.Logf("%s: %s (%s)", tc.name, action, reason)
	}
}

=== RUN   TestEvaluateConsecutiveBreach
    experiment_test.go:41: one spike: abort=false; two spikes: abort=true reason="guardrail p99_ms breached 2 consecutive samples"
--- PASS: TestEvaluateConsecutiveBreach (0.00s)
=== RUN   TestStopLossCheck
    experiment_test.go:90: healthy: pass (hypothesis held)
    experiment_test.go:90: short: continue (need more samples)
    experiment_test.go:90: claim: fail (hypothesis order_success breached 2 consecutive samples)
    experiment_test.go:90: guard: abort (guardrail p99_ms breached 2 consecutive samples)
    experiment_test.go:90: no_path: abort (no out-of-band stop path)
    experiment_test.go:90: bad_deadline: abort (abort deadline is outside the run)
    experiment_test.go:90: overrun: abort (max duration exceeded)
--- PASS: TestStopLossCheck (0.00s)