三个场景，三条判定线
容量规划给出部署数，故障场景回答这些实例真的坏掉时系统怎么退化。三个场景都选在订单写路径上：数据库变慢、缓存整体不可用、单个订单实例丢失。它们属于 L2：进程内可重复、依赖可控故障注入、不依赖真实网络分区。平台级混沌和跨节点网络故障放到卷四，本节不编造生产故障。
测试用内存适配器和逻辑时钟，不 sleep、不连真实中间件。所有延迟都是注入值，所有结果都是确定性计算；同一份代码运行多次必须得到完全相同的 Report，测试用 reflect.DeepEqual 守住这一点。
判定分四类：success 正常完成；degraded 走了降级路径但仍成功；failed 依赖返回错误；timeout 超过调用方截止时间。P99 用排序后的第 ceil(0.99n) 个样本，失败请求的延迟也进入样本，避免只统计成功请求把长尾抹掉。
故障注入器与判定规则
代码分三部分：Summarize 做计数与 P99；Dependency 同时充当数据库和缓存；Service 实现先缓存后数据库的订单写入。三个 Run* 入口分别固定输入与故障注入点，测试只断言关键计数、P99 和可重复性。
// file: internal/faultlab/faultlab.go
package faultlab
import (
"context"
"errors"
"fmt"
"math"
"sort"
"time"
)
type Status string
const (
Success Status = "success"
Degraded Status = "degraded"
Failed Status = "failed"
TimedOut Status = "timeout"
)
var ErrUnavailable = errors.New("dependency unavailable")
type Result struct {
ID string
Status Status
Latency time.Duration
}
type Summary struct {
Total, Success, Degraded, Failed, TimedOut int
P99 time.Duration
}
// Summarize 用精确排序算 P99；失败请求也留作延迟样本。
func Summarize(rs []Result) Summary {
s := Summary{Total: len(rs)}
lat := make([]time.Duration, 0, len(rs))
counts := map[Status]int{}
for _, r := range rs {
counts[r.Status]++
lat = append(lat, r.Latency)
}
s.Success, s.Degraded = counts[Success], counts[Degraded]
s.Failed, s.TimedOut = counts[Failed], counts[TimedOut]
if len(lat) > 0 {
sort.Slice(lat, func(i, j int) bool { return lat[i] < lat[j] })
s.P99 = lat[int(math.Ceil(0.99*float64(len(lat))))-1]
}
return s
}
// Dependency 同时模拟数据库和缓存；Down 表示整体不可用。
type Dependency struct {
Latency time.Duration
Down bool
Calls int
Errors int
}
func (d *Dependency) Get(ctx context.Context) (time.Duration, error) {
d.Calls++
if err := ctx.Err(); err != nil {
return 0, err
}
if d.Down {
d.Errors++
return d.Latency, ErrUnavailable
}
return d.Latency, nil
}
type Service struct {
Cache, DB *Dependency
Timeout time.Duration
}
// Place 先查缓存；缓存不可用时降级到数据库，数据库超时返回 timeout。
func (s *Service) Place(ctx context.Context, id string) Result {
r := Result{ID: id}
elapsed := time.Duration(0)
degraded := false
if s.Cache != nil {
lat, err := s.Cache.Get(ctx)
if err == nil {
r.Status, r.Latency = Success, lat
return r
}
degraded = true
elapsed += lat
}
lat, err := s.DB.Get(ctx)
if elapsed+lat > s.Timeout {
r.Status, r.Latency = TimedOut, s.Timeout
return r
}
elapsed += lat
if err != nil {
r.Status, r.Latency = Failed, elapsed
return r
}
r.Status = Success
if degraded {
r.Status = Degraded
}
r.Latency = elapsed
return r
}
type Phase struct {
Name string
Summary Summary
DBCalls int
CacheErrors int
BackendErrors int
}
type Report struct {
Name string
Phases []Phase
Recovery string
}
func (r Report) String() string {
s := "scenario=" + r.Name
for _, p := range r.Phases {
s += fmt.Sprintf(" | %s total=%d success=%d degraded=%d failed=%d timeout=%d p99=%s db_calls=%d cache_errors=%d backend_errors=%d", p.Name, p.Summary.Total, p.Summary.Success, p.Summary.Degraded, p.Summary.Failed, p.Summary.TimedOut, p.Summary.P99, p.DBCalls, p.CacheErrors, p.BackendErrors)
}
return s + " | recovery=" + r.Recovery
}
func runService(svc *Service, n int) []Result {
out := make([]Result, 0, n)
for i := 0; i < n; i++ {
out = append(out, svc.Place(context.Background(), fmt.Sprintf("o-%d", i)))
}
return out
}
// RunDBLatency：30ms 正常 -> 120ms 超时 -> 20ms 恢复。
func RunDBLatency() Report {
db := &Dependency{Latency: 30 * time.Millisecond}
svc := &Service{DB: db, Timeout: 100 * time.Millisecond}
healthy := runService(svc, 20)
healthyDB := db.Calls
db.Latency = 120 * time.Millisecond
faulty := runService(svc, 20)
db.Latency = 20 * time.Millisecond
back := runService(svc, 1)
return Report{Name: "db_latency", Phases: []Phase{
{"healthy_30ms", Summarize(healthy), healthyDB, 0, 0},
{"fault_120ms", Summarize(faulty), 20, 0, 0},
{"recovered_20ms", Summarize(back), 1, 0, 0},
}, Recovery: "first success after latency restored: requests=1 latency=20ms"}
}
// RunCacheOutage：缓存整体不可用时全部请求降级到数据库。
func RunCacheOutage() Report {
db := &Dependency{Latency: 15 * time.Millisecond}
cache := &Dependency{Latency: 2 * time.Millisecond}
svc := &Service{Cache: cache, DB: db, Timeout: 100 * time.Millisecond}
healthy := runService(svc, 100)
cache.Down = true
outage := runService(svc, 100)
errors := cache.Errors
cache.Down = false
back := runService(svc, 1)
return Report{Name: "cache_outage", Phases: []Phase{
{"healthy_cache", Summarize(healthy), 0, 0, 0},
{"cache_down", Summarize(outage), 100, errors, 0},
{"cache_back", Summarize(back), 0, 0, 0},
}, Recovery: "first cache hit after cache restored: requests=1 latency=2ms"}
}
type Endpoint struct {
Name string
Latency time.Duration
Failing, Ejected bool
}
type LoadBalancer struct {
Endpoints []*Endpoint
Next int
Retry bool
RetryCost time.Duration
Failures map[string]int
Calls int
FirstFailure int
EjectedAt int
BackendErrors int
}
func (lb *LoadBalancer) pick() *Endpoint {
for i := 0; i < len(lb.Endpoints); i++ {
ep := lb.Endpoints[lb.Next%len(lb.Endpoints)]
lb.Next++
if !ep.Ejected {
return ep
}
}
return nil
}
// Call 在失败实例上最多重试一次；重试不改变健康检测，否则故障实例会被永久留在池子里。
func (lb *LoadBalancer) Call(ctx context.Context, id string) Result {
call := lb.Calls
lb.Calls++
r := Result{ID: id}
if ctx.Err() != nil {
r.Status = Failed
return r
}
elapsed := time.Duration(0)
for attempt := 0; attempt < 2; attempt++ {
ep := lb.pick()
if ep == nil {
r.Status, r.Latency = Failed, elapsed
return r
}
elapsed += ep.Latency
if !ep.Failing {
r.Status, r.Latency = Success, elapsed
if attempt > 0 {
r.Status = Degraded
}
return r
}
lb.BackendErrors++
lb.Failures[ep.Name]++
if lb.FirstFailure < 0 {
lb.FirstFailure = call
}
if lb.Failures[ep.Name] >= 3 {
ep.Ejected = true
if lb.EjectedAt < 0 {
lb.EjectedAt = call
}
}
if !lb.Retry || attempt == 1 {
break
}
elapsed += lb.RetryCost
}
r.Status, r.Latency = Failed, elapsed
return r
}
// RunInstanceLoss：第 30 个请求让 order-2 失败，对比有/无重试。
func RunInstanceLoss(retry bool) Report {
eps := []*Endpoint{
{Name: "order-1", Latency: 10 * time.Millisecond},
{Name: "order-2", Latency: 10 * time.Millisecond},
{Name: "order-3", Latency: 10 * time.Millisecond},
}
lb := &LoadBalancer{Endpoints: eps, Retry: retry, RetryCost: 2 * time.Millisecond,
Failures: map[string]int{}, FirstFailure: -1, EjectedAt: -1}
results := make([]Result, 0, 90)
lat := make([]time.Duration, 0, 90)
for i := 0; i < 90; i++ {
if i == 30 {
eps[1].Failing = true
}
r := lb.Call(context.Background(), fmt.Sprintf("o-%d", i))
results, lat = append(results, r), append(lat, r.Latency)
}
rec := "first backend failure to ejection"
if lb.FirstFailure >= 0 && lb.EjectedAt >= 0 {
sum := time.Duration(0)
for i := lb.FirstFailure; i <= lb.EjectedAt; i++ {
sum += lat[i]
}
rec = fmt.Sprintf("%s: requests=%d latency=%s", rec, lb.EjectedAt-lb.FirstFailure+1, sum)
}
return Report{Name: fmt.Sprintf("instance_loss_retry_%v", retry), Phases: []Phase{
{"90_requests", Summarize(results), 0, 0, lb.BackendErrors},
}, Recovery: rec}
}
// file: internal/faultlab/faultlab_test.go
package faultlab
import (
"reflect"
"testing"
"time"
)
func TestFaultScenarios(t *testing.T) {
db, cache := RunDBLatency(), RunCacheOutage()
noRetry, retry := RunInstanceLoss(false), RunInstanceLoss(true)
for _, r := range []Report{db, cache, noRetry, retry} {
t.Log(r)
}
if db.Phases[0].Summary.Success != 20 || db.Phases[1].Summary.TimedOut != 20 ||
db.Phases[2].Summary.Success != 1 || db.Phases[1].Summary.P99 != 100*time.Millisecond {
t.Fatalf("db=%+v", db)
}
if cache.Phases[0].Summary.Success != 100 || cache.Phases[1].Summary.Degraded != 100 ||
cache.Phases[1].DBCalls != 100 || cache.Phases[1].Summary.P99 != 17*time.Millisecond ||
cache.Phases[2].DBCalls != 0 {
t.Fatalf("cache=%+v", cache)
}
if noRetry.Phases[0].Summary.Failed == 0 || retry.Phases[0].Summary.Failed != 0 ||
retry.Phases[0].Summary.Degraded == 0 {
t.Fatalf("loss no=%+v retry=%+v", noRetry.Phases[0].Summary, retry.Phases[0].Summary)
}
if retry.Phases[0].Summary.P99 <= noRetry.Phases[0].Summary.P99 {
t.Fatalf("retry p99=%s no=%s", retry.Phases[0].Summary.P99, noRetry.Phases[0].Summary.P99)
}
if !reflect.DeepEqual(db, RunDBLatency()) || !reflect.DeepEqual(retry, RunInstanceLoss(true)) {
t.Fatal("scenario is not repeatable")
}
}
场景一：数据库延迟
数据库延迟从 30ms 拉到 120ms，调用方超时 100ms，然后恢复到 20ms。20 个请求在健康期全部成功，P99 为 30ms；故障期 20 个请求全部超时，P99 被截止时间截断在 100ms；恢复后第一个请求成功，延迟 20ms。这里没有降级路径：订单写入在数据库超时后不能假装成功，只能把失败暴露给调用方或进入重试队列。db_calls 仍是 20，因为超时是在 Get 返回后判定的，调用已经发生。
场景二：缓存整体失效
缓存整体不可用时，100 个请求全部转到数据库：100 个 degraded、0 个 failed、P99 为 17ms、db_calls=100、cache_errors=100。17ms 由缓存失败的 2ms 加数据库的 15ms 组成。恢复后第一个请求重新命中缓存，延迟 2ms。这个场景的关键不是“降级成功”，而是数据库是否接得住 100% 的流量；如果数据库余量不足，degraded 会迅速变成 timeout。
场景三：单实例丢失
单实例丢失在第 30 个请求注入：order-2 开始返回错误，负载均衡器连续 3 次后端错误后将其摘除。不开重试时，90 个请求中 87 成功、3 失败，P99 为 10ms；从第一次后端失败到摘除经过 7 个请求，累计 70ms。开重试时，3 个请求在重试后成功并记为 degraded，失败为 0，但 P99 升到 22ms，摘除窗口缩短到 5 个请求，累计 86ms。重试改善可用性，代价是尾延迟；它不能替代健康检测，否则故障实例会一直被重试打中。
真实运行输出
命令：go test ./internal/faultlab/ -count=1 -v。环境为 macOS darwin/arm64、Go 1.26.0；测试串行执行，并发为 1，运行 1 次。完整输出保存在 .local/reports/be-arch/be-arch-perf-13-2-three-failure-scenarios-tests.txt。
=== RUN TestFaultScenarios
faultlab_test.go:13: scenario=db_latency | healthy_30ms total=20 success=20 degraded=0 failed=0 timeout=0 p99=30ms db_calls=20 cache_errors=0 backend_errors=0 | fault_120ms total=20 success=0 degraded=0 failed=0 timeout=20 p99=100ms db_calls=20 cache_errors=0 backend_errors=0 | recovered_20ms total=1 success=1 degraded=0 failed=0 timeout=0 p99=20ms db_calls=1 cache_errors=0 backend_errors=0 | recovery=first success after latency restored: requests=1 latency=20ms
faultlab_test.go:13: scenario=cache_outage | healthy_cache total=100 success=100 degraded=0 failed=0 timeout=0 p99=2ms db_calls=0 cache_errors=0 backend_errors=0 | cache_down total=100 success=0 degraded=100 failed=0 timeout=0 p99=17ms db_calls=100 cache_errors=100 backend_errors=0 | cache_back total=1 success=1 degraded=0 failed=0 timeout=0 p99=2ms db_calls=0 cache_errors=0 backend_errors=0 | recovery=first cache hit after cache restored: requests=1 latency=2ms
faultlab_test.go:13: scenario=instance_loss_retry_false | 90_requests total=90 success=87 degraded=0 failed=3 timeout=0 p99=10ms db_calls=0 cache_errors=0 backend_errors=3 | recovery=first backend failure to ejection: requests=7 latency=70ms
faultlab_test.go:13: scenario=instance_loss_retry_true | 90_requests total=90 success=87 degraded=3 failed=0 timeout=0 p99=22ms db_calls=0 cache_errors=0 backend_errors=3 | recovery=first backend failure to ejection: requests=5 latency=86ms
--- PASS: TestFaultScenarios (0.00s)
PASS
ok archlab/internal/faultlab 0.448s
这些数字证明什么、不证明什么
这些数字证明判定逻辑、降级路径和恢复定义可以在 CI 里稳定复现，不是生产环境的延迟实测。三个场景都运行在单进程内，没有真实网络、连接池、线程调度和 GC 干扰；P99 是注入延迟的确定值，不是采样统计。真实网络分区、进程被 kill、时钟漂移和跨节点一致性要放到卷四的混沌实验里验证，本节不引用任何生产故障数字。
另一个边界是重试放大：instance_loss_retry_true 的 backend_errors 仍是 3，说明重试请求也打到了故障实例。生产上要配合退避、重试预算和幂等键，否则重试会把局部故障放大成下游雪崩。
小结
三个 L2 场景给出可回归的结论：数据库超时全量失败且 P99 被截止时间截断；缓存全失效时 100% 请求降级到数据库并把 P99 推到 17ms；单实例丢失在 3 次后端错误后摘除；无重试暴露 3 个失败，有重试把失败转成 degraded 但 P99 从 10ms 升到 22ms；恢复判定分别用首个成功请求和摘除窗口固定；所有结果重复运行一致。它们补齐了容量规划之后的“坏了会怎样”，但离生产混沌还有一步。
阅读导航：上一节：13.1 一次完整的容量规划 · 下一节：13.3 进入卷三：分布式与一致性 。
继续阅读
探索更多技术文章
浏览归档，发现更多关于系统设计、工具链和工程实践的内容。