26. MongoDB 监控与可观测性实践

// 服务端全局状态（核心入口）
db.serverStatus()
// { "host": "...", "version": "7.0.x", "uptime": 86400,
//   "connections": {...}, "opcounters": {...}, "network": {...},
//   "mem": {...}, "wiredTiger": {...}, "repl": {...}, ... }

// 会话级：查看当前正在执行的操作
db.currentOp({ active: true, $or: [{ op: "query" }, { op: "command" }] })

// 简洁的"一分钟体检"
db.serverStatus().opcounters
db.serverStatus().connections
db.serverStatus().mem
db.serverStatus().wiredTiger.cache["dirty percentage in the cache"]
db.getReplicationInfo()   // oplog 窗口

// 开启慢查询收集（只记录 > 100ms 的操作）
db.setProfilingLevel(1, { slowms: 100 })

// 查看当前 profiling 状态
db.getProfilingStatus()
// { was: 1, slowms: 100, sampleRate: 1 }

// 查询最近的慢查询
db.system.profile.find().sort({ ts: -1 }).limit(10).toArray()

// 按集合聚合慢查询：平均耗时、次数、最大耗时
db.system.profile.aggregate([
  { $match: { op: "query", millis: { $gt: 100 } } },
  { $group: {
      _id: "$ns",
      avgMillis: { $avg: "$millis" },
      maxMillis: { $max: "$millis" },
      count: { $sum: 1 }
  }},
  { $sort: { avgMillis: -1 } }
])

// 查看某条慢查询是否走了 COLLSCAN
db.system.profile.find(
  { "ns": "shop.orders", "planSummary": /COLLSCAN/ },
  { "millis": 1, "query": 1, "planSummary": 1 }
).sort({ millis: -1 }).limit(5)

// 副本集复制概况
rs.printReplicationInfo()
// configured oplog size:   1879MB
// log length start to end: 86400 (24 hours)

// 各 Secondary 的同步位置与滞后
rs.printSecondaryReplicationInfo()
// source: 172.17.0.2:27017
//   syncedTo: Thu Sep 27 2026 12:00:00
//   0 secs (0 hrs) behind the primary

// 直接读取 oplog 首尾时间戳
db.getSiblingDB("local").oplog.rs.find().sort({ $natural: 1 }).limit(1).next().ts
db.getSiblingDB("local").oplog.rs.find().sort({ $natural: -1 }).limit(1).next().ts

// 增大 oplog 窗口（需要重启生效的经典做法）
// 1) 对副本集某节点停止服务
// 2) 以 --oplogSize MB 重启
// 或使用 replSetResizeOplog（4.4+ 可在线调整）
db.adminCommand({ replSetResizeOplog: 1, size: 4096 })  // 4096 MB

# docker-compose.yml：mongodb-exporter 部署
services:
  mongodb-exporter:
    image: percona/mongodb_exporter:0.40
    command:
      - "--mongodb.uri=mongodb://monitor:pass@mongodb-primary:27017/admin?replicaSet=rs0"
      - "--discovering-mode"
      - "--compatible-mode"
      - "--collector.collection"
      - "--collector.topmetrics"
    ports:
      - "9216:9216"
    restart: unless-stopped

# 验证指标输出
curl -s http://localhost:9216/metrics | head -n 20
# mongodb_mongod_connections_current 120
# mongodb_mongod_connections_available 49999
# mongodb_mongod_global_lock_currentqueue_total{type="read"} 0
# mongodb_mongod_metrics_cursor_open_total{state="total"} 12

scrape_configs:
  - job_name: mongodb
    static_configs:
      - targets: ["mongodb-exporter:9216"]

# 一个典型的 PromQL 示例：查询 5 分钟内慢查询速率
# rate(mongodb_mongod_slow_queries_total[5m])

# prometheus-rules.yml
groups:
  - name: mongodb_critical
    rules:
      - alert: MongoDBPrimaryDown
        expr: mongodb_mongod_replset_member_state{state="PRIMARY"} == 0
        for: 1m
        labels: { severity: critical }
        annotations:
          summary: "MongoDB 主节点不可用"

      - alert: MongoDBReplicationLagHigh
        expr: mongodb_mongod_replset_member_replication_lag > 10
        for: 2m
        labels: { severity: warning }
        annotations:
          summary: "复制滞后超过 10 秒"

      - alert: MongoDBOplogWindowTooSmall
        expr: time() - mongodb_mongod_replset_oplog_tail_timestamp > 86400
        for: 10m
        labels: { severity: critical }
        annotations:
          summary: "oplog 窗口小于 24 小时，恢复能力受限"

      - alert: MongoDBConnectionsExhausted
        expr: mongodb_mongod_connections_current / mongodb_mongod_connections_available > 0.8
        for: 5m
        labels: { severity: warning }
        annotations:
          summary: "连接数超过可用连接的 80%"

      - alert: MongoDBCacheEvictionPressure
        expr: rate(mongodb_mongod_wiredtiger_cache_pages_evicted_total[5m]) > 100
        for: 10m
        labels: { severity: warning }
        annotations:
          summary: "WiredTiger 驱逐压力增大，检查 cacheSizeGB"

// 连接数实况
db.serverStatus().connections
// { current: 320, available: 65216, totalCreated: 12800, ... }

// 找出占用连接的来源（应用/监控/备份工具）
db.currentOp({ "connectionId": { $exists: true } }).inprog.forEach(op => {
  print(op.client, op.connectionId)
})

// 查看驱动上报的等待队列（若有 waitQueue 指标）
// 4.4+ 通过 mongo shell 观察 db.currentOp() 中等待锁的操作占比
db.currentOp({ waitForLock: true }).inprog.length

# mtools：解析慢日志并可视化
mplotqueries --type histogram mongod.log
mprof mongod.log | mplotqueries --type histogram --format PNG > profile.png
mlogfilter mongod.log --slow --threshold 100