Erlang/Elixir 可观测性：日志、Telemetry 指标与分布式追踪

# config/config.exs
config :logger,
  level: :info,
  backends: [:console]

config :logger, :console,
  format: "$time $metadata[$level] $message\n",
  metadata: [:request_id, :user_id]

# 写入文件
config :logger,
  backends: [{LoggerFileBackend, :error_log}]

config :logger, :error_log,
  path: "/var/log/app/error.log",
  level: :error

# 或使用 structured logging backend（如 logger_json）

非结构化："User 42 logged in from 192.168.1.1 at 10:00"
结构化：{"@timestamp":"2024-01-15T10:00:00Z","event":"login","user_id":42,"ip":"192.168.1.1"}
# JSON 可被 Elasticsearch/Loki 索引和查询

# mix.exs: {:logger_json, "~> 5.0"}
config :logger_json, :backend,
  metadata: :all,
  formatter: LoggerJSON.Formatters.BasicLogger

config :logger,
  backends: [LoggerJSON.Backend]

# 用 metadata 而非字符串拼接
Logger.info("User login", user_id: user.id, ip: conn.remote_ip)

# 统一 request_id 贯穿请求
Logger.metadata(request_id: request_id)

# 错误日志带异常和堆栈
Logger.error("Payment failed", error: inspect(e), stacktrace: __STACKTRACE__)

Telemetry 是「标准化事件发射库」
不是监控工具本身，而是数据 producer
第三方库（Ecto/Phoenix/Redix）内置 Telemetry 事件
应用代码 attach 处理器来消费事件 → 生成指标

# 定义事件（通常在库代码中）
:telemetry.execute([:web, :request], %{duration: 230}, %{path: "/users"})

# 附加处理器（应用代码中订阅）
:telemetry.attach(
  "web-request-handler",
  [:web, :request],
  &handle_event/4,
  nil
)

def handle_event([:web, :request], measurements, metadata, _config) do
  # measurements = %{duration: 230}
  # metadata = %{path: "/users"}
  Statsd histogram("web.request.duration", measurements.duration,
    tags: ["path:#{metadata.path}"])
end

[:phoenix, :endpoint, :stop]          # Phoenix 请求完成
e[:phoenix, :router_dispatch, :stop]   # 路由分派完成
e[:phoenix, :socket_connected]         # WebSocket 连接
e[:ecto, :query]                       # Ecto 查询
e[:vm_memory, :total]                  # VM 内存

# 定义指标规格
metrics = [
  counter("web.request.count"),
  sum("web.request.body_bytes", unit: :byte),
  last_value("vm.memory.total", unit: :byte),
  summary("web.request.duration",
    unit: {:native, :millisecond},
    tags: [:path, :method]
  ),
  distribution("web.request.duration",
    buckets: [10, 50, 100, 200, 500],
    unit: {:native, :millisecond}
  )
]

# mix.exs: {:prometheus_ex, "~> 3.0"}, {:prometheus_plugs, "~> 1.0"}

# 定义 metrics
defmodule MyApp.Metrics do
  use Prometheus.Metric

  defsetup do
    Counter.declare(name: :http_requests_total, labels: [:method, :path])
    Histogram.declare(name: :http_request_duration_seconds, labels: [:method])
  end
end

# 更新
def handle_request(conn) do
  Counter.inc(name: :http_requests_total, labels: [conn.method, conn.path_info])
  # ...
end

# Phoenix 路由
get "/metrics", Prometheus.PlugsExporter

# Prometheus scrape 配置
scrape_configs:
  - job_name: 'myapp'
    static_configs:
      - targets: ['localhost:4000']

# 导入 BEAM VM 看板模板
# 关键面板：
#   - 请求 QPS / 错误率
#   - P50/P95/P99 响应时间
#   - BEAM 进程数 / reductions 每秒
#   - 内存使用（atom / binary / ets / process）
#   - ETS 表大小

Trace：一次请求的完整链路
Span：链路中的一个操作（如 DB 查询）
SpanContext：Trace ID + Span ID + Flags（跨进程传递）
Baggage：跨 Span 传递的键值对

# mix.exs: {:opentelemetry, "~> 1.0"}, {:opentelemetry_exporter, "~> 1.0"}

# 初始化 Trace
require OpenTelemetry.Tracer

OpenTelemetry.Tracer.with_span "process_order" do
  OpenTelemetry.Tracer.set_attribute("order.id", order_id)

  OpenTelemetry.Tracer.with_span "db_query" do
    Repo.get(Order, order_id)
  end
end

# mix.exs: {:opentelemetry_phoenix, "~> 1.0"}
# 自动追踪 HTTP 请求、DB 查询、外部调用
# 导出到 Jaeger/Tempo/Datadog

$ mix deps.get observer_cli
$ iex -S mix
iex> :observer_cli.start()

# 实时显示：
# - 进程列表（内存/ reductions /消息队列）
# - ETS 表
# - 端口 / 调度器负载
# - Mnesia 表

# 内存分析
:recon.proc_count(:memory, 10)   # 内存占用最高的 10 个进程
:recon.proc_count(:message_queue_len, 10)  # 消息队列最长的进程

# 节点状态
:recon.node_stats(1000, 10)       # 每秒采样，共 10 次

# 定期输出 VM 统计到 StatsD
# 进程数 / reductions / atom / binary / ets / port
# 适合长期趋势监控

# 进程信息
Process.info(pid, [:message_queue_len, :memory, :reductions])

# VM 内存
:erlang.memory()      # => [total: ..., processes: ..., binary: ..., ets: ...]

# 统计
:erlang.statistics(:reductions)
:erlang.system_info(:process_count)

App (JSON 日志) → Filebeat/Fluent Bit → Logstash → Elasticsearch → Kibana
                                          ↓
                                        Loki (轻量替代)

# BEAM 应用日志通过 stdout/stderr → 容器收集器 → 聚合存储

# Prometheus Alertmanager
- alert: HighErrorRate
  expr: rate(http_requests_total{status=~"5.."}[5m]) > 0.05
  for: 5m
  annotations:
    summary: "错误率 > 5%"

- alert: MemoryLeaking
  expr: rate(vm_memory_total_bytes[10m]) > 1000000
  for: 10m
  annotations:
    summary: "内存持续泄漏"

- alert: MessageQueueBlocked
  expr: max_over_time(process_message_queue_length[5m]) > 10000
  annotations:
    summary: "进程消息队列阻塞"