38. MongoDB 压测与容量规划

数据集大小与分布（键的基数、热点比例）
读写比例（如 80:20）
写关注与读关注等级
并发连接数与线程模型
索引集合（与生产完全一致）
副本集拓扑（几节点、是否跨机房）

# 安装并加载数据（示意）
./bin/ycsb load mongodb -s -P workloads/workloada \
  -p mongodb.url="mongodb://rs-shard01:27018/shop?w=majority" \
  -p recordcount=10000000

# 运行负载：50% 读 50% 更新
./bin/ycsb run mongodb -s -P workloads/workloada \
  -p mongodb.url="mongodb://rs-shard01:27018/shop?w=majority" \
  -p operationcount=5000000 \
  -threads 64

// insert_bench.js：批量插入压测
const TOTAL = 1000000
const BATCH = 1000
const coll = db.getSiblingDB("bench").events

coll.drop()
coll.createIndex({ userId: 1, createdAt: -1 })

const start = Date.now()
for (let i = 0; i < TOTAL; i += BATCH) {
  const docs = []
  for (let j = 0; j < BATCH; j++) {
    docs.push({
      userId: "u" + ((i + j) % 100000),      // 高基数，打散
      type: ["click", "view", "buy"][(i + j) % 3],
      amount: (i + j) % 500,
      createdAt: new Date()
    })
  }
  coll.insertMany(docs, { ordered: false, writeConcern: { w: "majority" } })
}
const elapsed = (Date.now() - start) / 1000
print(`inserted ${TOTAL} in ${elapsed.toFixed(1)}s, ${(TOTAL / elapsed).toFixed(0)} ops/s`)

// read_bench.js：混合读写压测
const coll = db.getSiblingDB("bench").events
const CONCURRENCY = 32
const DURATION_MS = 60000
const deadline = Date.now() + DURATION_MS
let reads = 0, writes = 0, errors = 0

// 用 startParallel 模拟并发
const futures = []
for (let c = 0; c < CONCURRENCY; c++) {
  futures.push(() => {
    while (Date.now() < deadline) {
      try {
        if (Math.random() < 0.8) {
          coll.find({ userId: "u" + Math.floor(Math.random() * 100000) })
              .sort({ createdAt: -1 }).limit(10).toArray()
          reads++
        } else {
          coll.insertOne({ userId: "u" + Math.floor(Math.random() * 100000),
                           type: "buy", amount: 42, createdAt: new Date() })
          writes++
        }
      } catch (e) { errors++ }
    }
  })
}
startParallel(...futures)
const total = reads + writes
print(`reads=${reads} writes=${writes} errors=${errors} total=${total} in ${DURATION_MS / 1000}s`)
print(`throughput=${(total / (DURATION_MS / 1000)).toFixed(0)} ops/s`)

// 估算集合与索引体积
db.events.stats()

{
  ns: "bench.events",
  count: 10000000,
  size: 2048576000,          // 文档总字节数
  avgObjSize: 205,
  storageSize: 2345678901,   // 磁盘占用（含压缩与填充）
  totalIndexSize: 850000000,
  indexSizes: {
    "_id_": 320000000,
    "userId_1_createdAt_-1": 530000000
  }
}

所需内存 ≈ 工作集文档大小 + 工作集索引大小 + 连接开销 + 预留余量

// 从 WiredTiger 缓存看实际内存占用
db.serverStatus().wiredTiger.cache

{
  "bytes currently in the cache": 5368709120,
  "maximum bytes configured": 7516192768,
  "bytes read into cache": 12345678901,
  "tracked dirty bytes in the cache": 104857600,
  "pages read into cache": 234567,
  "pages written from cache": 12345
}

// 用 db.serverStatus 观察页读入速率
db.serverStatus().wiredTiger.cache["pages read into cache"]
// 间隔采样两次，若稳定期仍在增长，说明工作集超出内存

// 用索引与文档大小估算
const stats = db.events.stats()
const workingSet = stats.size * 0.2 + stats.totalIndexSize   // 20% 热点数据 + 全部索引
print(`estimated working set: ${(workingSet / 1024 / 1024 / 1024).toFixed(2)} GB`)

// 查看当前连接状态
db.serverStatus().connections

{
  current: 142,
  available: 837930,
  totalCreated: 10432,
  active: 12,
  threaded: 142,
  exhaustIsMaster: 0,
  exhaustHello: 0,
  awaitingTopologyChanges: 0
}

# 启动参数（mongod.conf）
net:
  maxIncomingConnections: 20000
  port: 27018

// 运行时查看与估算
const conns = db.serverStatus().connections
print(`used ${conns.current}, available ${conns.available}`)

所需连接上限 ≈ 应用实例数 × 每实例连接池大小 × 安全系数(1.5)

const client = new MongoClient(uri, {
  maxPoolSize: 100,        // 每实例最大连接数
  minPoolSize: 10,         // 保持的最小连接
  maxIdleTimeMS: 60000,    // 空闲回收
  waitQueueTimeoutMS: 5000 // 获取连接超时
})

单文档字节 ≈ 字段名长度 + 值大小 + BSON 开销（每字段约 2 到 5 字节 + 类型开销）
集合体积 ≈ 文档数 × 平均文档大小
磁盘占用 ≈ 集合体积 / 压缩比（WiredTiger 默认约 2 到 5 倍压缩）

// 用 collStats 精确获取
db.events.stats(1024 * 1024)   // 以 MB 为单位返回

// 查看各索引的实际大小
db.events.stats().indexSizes
// { "_id_": 320000000, "userId_1_createdAt_-1": 530000000 }

总磁盘 ≈ (集合体积 + 索引体积) × 副本数 × (1 + 预留比例)

// 水平扩容：新增分片
sh.addShard("shard04/rs-shard04:27018")

// 查看分片状态
sh.status()

shards:
  {  "_id": "shard01",  "host": "shard01/rs-shard01:27018",  "state": 1 }
  {  "_id": "shard02",  "host": "shard02/rs-shard02:27018",  "state": 1 }
  {  "_id": "shard03",  "host": "shard03/rs-shard03:27018",  "state": 1 }
  {  "_id": "shard04",  "host": "shard04/rs-shard04:27018",  "state": 1 }