《MMO 服务器开发》8.2 多活与故障切换

// file: internal/dr/statemachine.go
// tier: L1
// verify: go test ./internal/dr/ -run TestFailover -v
package dr

import (
	"errors"
	"fmt"
	"time"
)

// ErrBadTransition：状态机收到了当前状态不该有的事件。
var ErrBadTransition = errors.New("dr: invalid state transition")

// State 是故障切换状态机的状态。
type State string

const (
	StateSteady    State = "steady"    // 稳定态：主在任
	StateDetecting State = "detecting" // 检测到主异常
	StateElecting  State = "electing"  // 选主
	StateSwitching State = "switching" // 流量切换 + 围栏旧主 + 一致性检查
	StateDone      State = "done"      // 切换完成
	StateAborted   State = "aborted"   // 放弃（如失去多数派）
)

// Event 是驱动状态机的外部事件。
type Event string

const (
	EvPrimaryDown     Event = "primary_down"
	EvCandidateReady  Event = "candidate_ready"
	EvTrafficSwitched Event = "traffic_switched"
	EvConsistencyOK   Event = "consistency_ok"
	EvConsistencyFail Event = "consistency_fail"
	EvQuorumLost      Event = "quorum_lost"
)

// transitions 是状态转移表：state -> event -> next state。
var transitions = map[State]map[Event]State{
	StateSteady: {
		EvPrimaryDown: StateDetecting,
		EvQuorumLost:  StateAborted,
	},
	StateDetecting: {
		EvCandidateReady: StateElecting,
		EvQuorumLost:     StateAborted,
	},
	StateElecting: {
		EvTrafficSwitched: StateSwitching,
		EvQuorumLost:      StateAborted,
	},
	StateSwitching: {
		EvConsistencyOK:   StateDone,
		EvConsistencyFail: StateAborted,
	},
}

// TimelineEntry 记录一次状态迁移，供事故复盘使用。
type TimelineEntry struct {
	At    time.Time
	From  State
	Event Event
	To    State
}

// Controller 驱动一次故障切换。
type Controller struct {
	cluster     *Cluster
	state       State
	newPrimary  string
	consistency ConsistencyReport
	Timeline    []TimelineEntry
}

// NewController 在集群上创建一个切换控制器。
func NewController(c *Cluster) *Controller {
	return &Controller{cluster: c, state: StateSteady}
}

// State 返回当前状态；NewPrimary 返回选出的新主；Consistency 返回最近一次检查结果。
func (c *Controller) State() State                   { return c.state }
func (c *Controller) NewPrimary() string             { return c.newPrimary }
func (c *Controller) Consistency() ConsistencyReport { return c.consistency }

// Step 推进状态机。非法事件返回 ErrBadTransition，不会改变状态。
func (c *Controller) Step(ev Event) (State, error) {
	next, ok := transitions[c.state][ev]
	if !ok {
		return c.state, fmt.Errorf("%w: state=%s event=%s", ErrBadTransition, c.state, ev)
	}
	c.Timeline = append(c.Timeline, TimelineEntry{At: time.Now(), From: c.state, Event: ev, To: next})
	c.state = next
	if err := c.effect(next); err != nil {
		return c.state, err
	}
	return c.state, nil
}

// effect 执行进入某状态时的副作用。
func (c *Controller) effect(s State) error {
	switch s {
	case StateElecting:
		// 选主：挑数据最全的备节点。
		cand, err := c.cluster.bestStandby()
		if err != nil {
			return err
		}
		c.newPrimary = cand.ID
	case StateSwitching:
		// 切换：先把旧主围栏掉（挡住它醒来的写），再把流量指到新主。
		old := c.cluster.PrimaryID
		c.cluster.Fence(old)
		cand := c.cluster.Nodes[c.newPrimary]
		cand.Role = RolePrimary
		cand.Term = c.cluster.Term
		c.cluster.PrimaryID = c.newPrimary
		c.cluster.TrafficTo = c.newPrimary
		// 切换完成后立刻做一致性检查。
		c.consistency = c.cluster.CheckConsistency()
	}
	return nil
}

// snippet: internal/dr/cluster.go —— 围栏旧主 + 写路径拒绝
// Fence 把某个节点围栏掉，并推进集群任期。
func (c *Cluster) Fence(nodeID string) {
	c.Term++
	if n, ok := c.Nodes[nodeID]; ok {
		n.Fenced = true
		n.Role = RoleStandby
	}
}

// Write 只有未围栏、可达、且是当前主的写会被接受。
func (c *Cluster) Write(nodeID, key string, delta int64) error {
	n := c.Nodes[nodeID]
	if n.Fenced {
		return fmt.Errorf("%w: %s term=%d", ErrFenced, nodeID, n.Term)
	}
	if nodeID != c.PrimaryID {
		return fmt.Errorf("%w: %s (primary=%s)", ErrNotPrimary, nodeID, c.PrimaryID)
	}
	// ... 应用到本地并尽力复制到可达备节点
	return nil
}

// snippet: internal/dr/cluster.go —— 一致性检查：不可达副本是滞后风险，不能跳过
func (c *Cluster) CheckConsistency() ConsistencyReport {
	rep := ConsistencyReport{PrimaryID: c.PrimaryID, Versions: map[string]int64{}}
	for _, n := range c.Nodes {
		if n.Fenced {
			continue // 已围栏的旧主不参与
		}
		rep.Versions[n.ID] = sumData(n)
		if !n.Reachable {
			rep.Unreachable = append(rep.Unreachable, n.ID) // 标出来，而不是跳过
		}
	}
	// ... 以新主为基准算 MaxDiff
	rep.OK = rep.MaxDiff == 0 && len(rep.Unreachable) == 0
	return rep
}

=== RUN   TestFailoverTakesOverAndFencesOldPrimary
    failover_test.go:77: 新主=db-c 旧主已围栏 term=2 一致性 maxDiff=1000 掉队=[db-b]
--- PASS: TestFailoverTakesOverAndFencesOldPrimary (0.00s)
=== RUN   TestFailoverQuorumLostAborts
--- PASS: TestFailoverQuorumLostAborts (0.00s)
PASS
ok  	aegis/internal/dr	0.866s