冗余不是“多开一个实例”
高可用的通用模式在 高可用架构模式 里已经讲过，本节不重复定义，只解决一个更具体的问题：arch-lab 从一台机器扩到同城双活之前，怎样用代码确认“每个组件都真的没有单点”。多开一个实例不等于消除单点：两个实例在同一台物理机、同一个可用区、同一条电源上，仍然是一个故障域。
判断依据必须是结构化的拓扑，而不是部署清单上“副本数 = 2”的字样。下面把实例、可用区、依赖关系和数据副本写成数据，再让检查器输出单点列表、可用区失效后的剩余容量，以及 N+1 后的容量。
先描述拓扑，再检查单点
拓扑只保留四个要素：实例（ID、组件、可用区、容量）、数据副本（组件、副本所在可用区、是否同步）、依赖（组件 A 需要组件 B）、故障域（用可用区表示）。网关、订单服务这类无状态组件看实例分布；数据库、缓存这类有状态组件还要看副本分布；依赖关系用来把上游的可用性上限传递出来。
检查器把单点分成三类：component 表示组件实例不足或全部挤在一个可用区；data 表示副本没有跨故障域；dependency 表示上游依赖了一个本身脆弱的组件。三类都输出 Scope + Target + Reason，方便直接生成工单，而不是只给一个布尔值。
容量口径要提前说清楚：Capacity 是示例值，单位是每秒订单请求数，只用于比较剩余比例；Coverage 是“某个可用区整体失效后剩余容量 / 全部实例容量”，不是线上真实吞吐的比例。真实容量要以压测结果为准，压测方法见 吞吐、延迟与尾延迟
。
// file: internal/topology/topology.go
// tier: L1
// verify: go test ./internal/topology/
package topology
import (
"fmt"
"sort"
"strings"
)
// Instance 是一个可独立失效的进程；同 Zone 的实例会一起失效。
type Instance struct {
ID, Component, Zone string
Capacity int
}
// ReplicaSet 描述一份数据的副本分布；Sync 表示同步复制。
type ReplicaSet struct {
Component string
Zones []string
Sync bool
}
type Topology struct {
Instances []Instance
Replicas []ReplicaSet
DependsOn map[string][]string // 组件 -> 依赖组件
}
type Finding struct {
Scope, Target, Reason string // Scope: component | data | dependency
}
// CheckSPOF 只回答结构问题：有没有把组件或数据放在单一故障域里。
func (t Topology) CheckSPOF() []Finding {
byComp := map[string][]Instance{}
for _, in := range t.Instances {
byComp[in.Component] = append(byComp[in.Component], in)
}
var names []string
for n := range byComp {
names = append(names, n)
}
sort.Strings(names)
fragile := map[string]bool{}
var out []Finding
for _, n := range names {
insts, zs := byComp[n], uniqueSortedZones(byComp[n])
switch {
case len(insts) == 1:
out = append(out, Finding{"component", n,
fmt.Sprintf("single instance %s in %s", insts[0].ID, insts[0].Zone)})
fragile[n] = true
case len(zs) < 2:
out = append(out, Finding{"component", n, "all instances in one zone " + strings.Join(zs, ",")})
fragile[n] = true
}
}
for _, rs := range t.Replicas {
if zs := uniqueSorted(rs.Zones); len(zs) < 2 {
out = append(out, Finding{"data", rs.Component, "replicas in one zone " + strings.Join(zs, ",")})
}
}
for _, n := range sortedKeys(t.DependsOn) {
refs := append([]string(nil), t.DependsOn[n]...)
sort.Strings(refs)
for _, d := range refs {
if fragile[d] {
out = append(out, Finding{"dependency", n + "->" + d, "depends on a fragile component"})
}
}
}
sort.Slice(out, func(i, j int) bool {
if out[i].Scope != out[j].Scope {
return out[i].Scope < out[j].Scope
}
return out[i].Target < out[j].Target
})
return out
}
type ZoneReport struct {
Zone string
RemainingCapacity int
Coverage float64
LostComponents []string
DataIntact bool
}
// ZoneFailures 枚举每个可用区整体失效后的容量与数据结论。
func (t Topology) ZoneFailures() []ZoneReport {
total := t.TotalCapacity()
zoneSet, compZones := map[string]bool{}, map[string]map[string]bool{}
for _, in := range t.Instances {
zoneSet[in.Zone] = true
if compZones[in.Component] == nil {
compZones[in.Component] = map[string]bool{}
}
compZones[in.Component][in.Zone] = true
}
var zones []string
for z := range zoneSet {
zones = append(zones, z)
}
sort.Strings(zones)
var out []ZoneReport
for _, z := range zones {
remaining := 0
for _, in := range t.Instances {
if in.Zone != z {
remaining += in.Capacity
}
}
var lost []string
for comp, zs := range compZones {
if len(zs) == 1 && zs[z] {
lost = append(lost, comp)
}
}
sort.Strings(lost)
intact := true
for _, rs := range t.Replicas {
ok := false
for _, rz := range rs.Zones {
if rz != z {
ok = true
break
}
}
if !ok {
intact = false
break
}
}
cov := 0.0
if total > 0 {
cov = float64(remaining) / float64(total)
}
out = append(out, ZoneReport{z, remaining, cov, lost, intact})
}
return out
}
// NPlusOne 返回任意单实例失效后的最坏剩余容量，即损失最大实例。
func (t Topology) NPlusOne() (remaining, lost int) {
total := 0
for _, in := range t.Instances {
total += in.Capacity
if in.Capacity > lost {
lost = in.Capacity
}
}
if total == 0 {
return 0, 0
}
return total - lost, lost
}
func (t Topology) TotalCapacity() int {
total := 0
for _, in := range t.Instances {
total += in.Capacity
}
return total
}
func uniqueSortedZones(insts []Instance) []string {
var zones []string
for _, in := range insts {
zones = append(zones, in.Zone)
}
return uniqueSorted(zones)
}
func uniqueSorted(in []string) []string {
seen := map[string]bool{}
var out []string
for _, s := range in {
if !seen[s] {
seen[s] = true
out = append(out, s)
}
}
sort.Strings(out)
return out
}
func sortedKeys[V any](m map[string]V) []string {
var keys []string
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
表驱动测试覆盖五种拓扑
测试不用启动真实机器，而是把五种典型拓扑喂给同一个检查器：单实例、同区双实例、跨区且数据有副本、跨区但缓存单实例、跨区但副本只在单区。第五种最容易被忽略：计算层已经跨区，数据库副本却还挤在一个区，故障时计算能力还在，数据却不可用。
第二个测试验证容量口径：两区各一半容量时，任意一区失效的 Coverage 都是 0.5，NPlusOne 损失最大实例的 300，剩 900；全部实例挤在一个区时，该区失效后剩余容量为 0，LostComponents 同时包含 app 和 db，DataIntact 为 false。
// file: internal/topology/topology_test.go
// tier: L1
// verify: go test -v ./internal/topology/
package topology
import (
"reflect"
"testing"
)
func TestSPOFTable(t *testing.T) {
cases := []struct {
name string
topo Topology
want []string
}{
{"single instance", Topology{Instances: []Instance{
{ID: "app-1", Component: "app", Zone: "az-a", Capacity: 100}}},
[]string{"component:app"}},
{"same zone two instances", Topology{Instances: []Instance{
{ID: "app-1", Component: "app", Zone: "az-a", Capacity: 100},
{ID: "app-2", Component: "app", Zone: "az-a", Capacity: 100}}},
[]string{"component:app"}},
{"cross zone with replicated data", Topology{
Instances: []Instance{
{ID: "app-a", Component: "app", Zone: "az-a", Capacity: 100},
{ID: "app-b", Component: "app", Zone: "az-b", Capacity: 100},
{ID: "db-a", Component: "db", Zone: "az-a", Capacity: 200},
{ID: "db-b", Component: "db", Zone: "az-b", Capacity: 200}},
Replicas: []ReplicaSet{{Component: "db", Zones: []string{"az-a", "az-b"}, Sync: true}}},
nil},
{"single cache is a dependency SPOF", Topology{
Instances: []Instance{
{ID: "gw-a", Component: "gateway", Zone: "az-a", Capacity: 100},
{ID: "gw-b", Component: "gateway", Zone: "az-b", Capacity: 100},
{ID: "cache-1", Component: "cache", Zone: "az-a", Capacity: 500}},
DependsOn: map[string][]string{"gateway": {"cache"}}},
[]string{"component:cache", "dependency:gateway->cache"}},
{"cross zone compute but single zone replica", Topology{
Instances: []Instance{
{ID: "db-a", Component: "db", Zone: "az-a", Capacity: 200},
{ID: "db-b", Component: "db", Zone: "az-b", Capacity: 200}},
Replicas: []ReplicaSet{{Component: "db", Zones: []string{"az-a"}}}},
[]string{"data:db"}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
var got []string
for _, f := range tc.topo.CheckSPOF() {
got = append(got, f.Scope+":"+f.Target)
}
if !reflect.DeepEqual(got, tc.want) {
t.Fatalf("findings=%v want %v", got, tc.want)
}
})
}
}
func TestZoneFailureCoverageAndNPlusOne(t *testing.T) {
topo := Topology{
Instances: []Instance{
{ID: "gw-a", Component: "gateway", Zone: "az-a", Capacity: 100},
{ID: "gw-b", Component: "gateway", Zone: "az-b", Capacity: 100},
{ID: "app-a", Component: "app", Zone: "az-a", Capacity: 300},
{ID: "app-b", Component: "app", Zone: "az-b", Capacity: 300},
{ID: "db-a", Component: "db", Zone: "az-a", Capacity: 200},
{ID: "db-b", Component: "db", Zone: "az-b", Capacity: 200}},
Replicas: []ReplicaSet{{Component: "db", Zones: []string{"az-a", "az-b"}, Sync: true}},
}
for _, rep := range topo.ZoneFailures() {
if rep.RemainingCapacity != 600 || rep.Coverage != 0.5 || !rep.DataIntact || len(rep.LostComponents) != 0 {
t.Fatalf("%+v", rep)
}
t.Logf("zone=%s remaining=%d coverage=%.2f lost=%v data_intact=%v",
rep.Zone, rep.RemainingCapacity, rep.Coverage, rep.LostComponents, rep.DataIntact)
}
if remaining, lost := topo.NPlusOne(); remaining != 900 || lost != 300 {
t.Fatalf("N+1 remaining=%d lost=%d", remaining, lost)
}
t.Logf("total=%d n_plus_one_remaining=%d n_plus_one_lost=%d", topo.TotalCapacity(), 900, 300)
single := Topology{
Instances: []Instance{
{ID: "app-1", Component: "app", Zone: "az-a", Capacity: 300},
{ID: "db-1", Component: "db", Zone: "az-a", Capacity: 200}},
Replicas: []ReplicaSet{{Component: "db", Zones: []string{"az-a"}}},
}
rep := single.ZoneFailures()[0]
if rep.RemainingCapacity != 0 || rep.Coverage != 0 || rep.DataIntact ||
!reflect.DeepEqual(rep.LostComponents, []string{"app", "db"}) {
t.Fatalf("single zone report=%+v", rep)
}
t.Logf("single_zone=%s remaining=%d coverage=%.2f lost=%v data_intact=%v",
rep.Zone, rep.RemainingCapacity, rep.Coverage, rep.LostComponents, rep.DataIntact)
}
真实运行结果
以下输出来自本机 macOS（darwin/arm64，Go 1.26.0），命令 go test -v ./... -count=1，运行 1 次；原始输出在 .local/reports/be-arch/be-arch-perf-08-1-redundancy-failure-domain-spof-tests.txt。go test -race ./... 同样通过。
=== RUN TestSPOFTable
=== RUN TestSPOFTable/single_instance
=== RUN TestSPOFTable/same_zone_two_instances
=== RUN TestSPOFTable/cross_zone_with_replicated_data
=== RUN TestSPOFTable/single_cache_is_a_dependency_SPOF
=== RUN TestSPOFTable/cross_zone_compute_but_single_zone_replica
--- PASS: TestSPOFTable (0.00s)
--- PASS: TestSPOFTable/single_instance (0.00s)
--- PASS: TestSPOFTable/same_zone_two_instances (0.00s)
--- PASS: TestSPOFTable/cross_zone_with_replicated_data (0.00s)
--- PASS: TestSPOFTable/single_cache_is_a_dependency_SPOF (0.00s)
--- PASS: TestSPOFTable/cross_zone_compute_but_single_zone_replica (0.00s)
=== RUN TestZoneFailureCoverageAndNPlusOne
topology_test.go:71: zone=az-a remaining=600 coverage=0.50 lost=[] data_intact=true
topology_test.go:71: zone=az-b remaining=600 coverage=0.50 lost=[] data_intact=true
topology_test.go:77: total=1200 n_plus_one_remaining=900 n_plus_one_lost=300
topology_test.go:90: single_zone=az-a remaining=0 coverage=0.00 lost=[app db] data_intact=false
--- PASS: TestZoneFailureCoverageAndNPlusOne (0.00s)
PASS
ok archlab/internal/topology 0.443s
从单实例到同城双活的迁移顺序
arch-lab 的迁移顺序不是“先上双活”，而是先把可回退的层做掉：
- 数据先有第二份副本：同区主备，确认备份可读、切换脚本可执行，但暂不自动切换。
- 无状态层同区双实例：前面加负载均衡，摘掉一个实例时容量仍能承接峰值。
- 第二份副本跨可用区：先同步复制，再验证复制延迟和故障时的数据完整性。
- 订单服务跨区双实例：先只把读流量切到第二区，写仍然单区，观察错误率和延迟。
- 写路径切换：引入 8.2 的租约与 fencing，才允许第二区接管写；在此之前不做自动切写。
每一步都有退出条件：复制延迟超过示例阈值 1 秒、跨区调用 P99 超过预算、演练中数据校验不一致，就退回上一步。多活写入、单元化、跨城容灾留到卷三，本节和 8.2 只把单区域加同城双活讲完整。
不做的部分
不做多活写，是因为写冲突、全局顺序和单元路由不是“再加一个副本”能解决的；不做跨城自动切换，是因为仲裁和脑裂代价远高于单区域；不做“副本数等于可用性”的推断，是因为同机架、同电源、同网络设备仍然是共享故障域。拓扑检查器只覆盖声明出来的故障域，物理机架、交换机、DNS 这些未声明的共享依赖，必须在部署评审里单独确认。
常见错误与边界
第一，只数实例不看可用区。两个实例在同一可用区，对该区故障仍然是单点；检查器必须把 Zone 作为一等字段。
第二，计算层跨区就认为高可用。数据库副本、缓存、消息队列如果还在单区，故障时上游活着也写不进去；data 类单点必须单独检查。
第三，把依赖单点当成上游的问题。网关依赖单实例缓存时，缓存的故障会直接变成网关不可用；dependency 结论要能定位到具体边。
第四，把 Coverage 当线上容量。它只是按声明容量算的比例，真实吞吐还受连接池、下游配额和热点影响，必须用压测结果校准。
小结
单点消除的第一步不是加机器，而是把实例、可用区、依赖和数据副本写成可检查的结构。本节实现的检查器能枚举 component、data、dependency 三类 SPOF，计算每个可用区失效后的剩余容量、覆盖率和数据完整性，并给出 N+1 的最坏剩余容量；表驱动测试覆盖了单实例、同区双实例、跨区副本和依赖单点。迁移顺序坚持“数据副本先行、读流量先切、写切换最后”，多活留到卷三。下一节处理单区域内真正接管写流量时的租约、fencing 与脑裂。
阅读导航：上一节：7.3 幂等键与去重窗口的实现与故障注入验证 · 下一节：8.2 主备与故障转移：单区域高可用 。
继续阅读
探索更多技术文章
浏览归档，发现更多关于系统设计、工具链和工程实践的内容。