模型量化与蒸馏：INT8/INT4 与部署权衡

权重显存 = 参数量 × 每参数字节数
KV Cache = 2 × n_layers × n_kv_heads × head_dim × seq_len × 精度字节
激活/中间张量 ≈ batch × seq_len × hidden × 若干倍

对称量化:   q = round(x / scale)                 , scale = max|x| / (2^(b-1) - 1)
非对称量化: q = round(x / scale) + zero_point    , scale = (max - min) / (2^b - 1)

# SmoothQuant 核心：s = max(|X|)^alpha / max(|W|)^(1-alpha)
import torch
def smooth_scale(act_max: torch.Tensor, w_max: torch.Tensor, alpha: float = 0.5):
    return (act_max.pow(alpha) / w_max.pow(1 - alpha)).clamp(min=1e-5)
# X' = X / s, W' = W * s  →  X'W' = XW 保持不变，但分布更友好

# 1. 从 HF 转换到 GGUF FP16
python convert_hf_to_gguf.py ./Qwen2.5-7B-Instruct \
  --outfile qwen2.5-7b-f16.gguf --outtype f16

# 2. 量化为 Q4_K_M
./llama-quantize qwen2.5-7b-f16.gguf qwen2.5-7b-q4km.gguf Q4_K_M

# 3. 量化后立刻跑困惑度，确认退化可接受
./llama-perplexity -m qwen2.5-7b-q4km.gguf -f wiki.test.raw

from awq import AutoAWQForCausalLM
from transformers import AutoTokenizer

model_path = "Qwen/Qwen2.5-7B-Instruct"
quant_path = "qwen2.5-7b-awq"

model = AutoAWQForCausalLM.from_pretrained(model_path, device_map="auto")
tokenizer = AutoTokenizer.from_pretrained(model_path)

quant_config = {
    "zero_point": True,
    "q_group_size": 128,
    "w_bit": 4,
    "version": "GEMM",
}

# 需要校准集，用领域内文本效果更好
model.quantize(tokenizer, quant_config=quant_config, calib_data="pileval")
model.save_quantized(quant_path)
tokenizer.save_pretrained(quant_path)

from transformers import AutoModelForCausalLM, BitsAndBytesConfig
import torch

bnb = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",           # NormalFloat4，对正态分布最优
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=True,      # 二次量化 scale，再省 0.4 bit
)

model = AutoModelForCausalLM.from_pretrained(
    "Qwen/Qwen2.5-7B-Instruct",
    quantization_config=bnb,
    device_map="auto",
)

python -m onnxruntime.quantization.preprocess --input model.onnx --output model_pre.onnx
python -m onnxruntime.quantization.quantize \
  --input model_pre.onnx --output model_int8.onnx \
  --quant_format QDQ --op_types MatMul,Attention \
  --calibrate_dataset calib_data/

import torch
import torch.nn as nn

class FakeQuant(nn.Module):
    """模拟量化-反量化的前向，梯度用 STE 直通。"""
    def __init__(self, bits: int = 4, group_size: int = 128):
        super().__init__()
        self.qmax = 2 ** (bits - 1) - 1
        self.group_size = group_size

    def forward(self, w: torch.Tensor) -> torch.Tensor:
        orig_shape = w.shape
        wg = w.reshape(-1, self.group_size)
        scale = wg.abs().amax(dim=1, keepdim=True) / self.qmax
        scale = scale.clamp(min=1e-8)
        q = torch.round(wg / scale).clamp(-self.qmax - 1, self.qmax)
        # 反量化回浮点，STE 让梯度直接穿透 round
        wq = (q * scale).reshape(orig_shape)
        return w + (wq - w).detach()

L = α · CE(student_logits, hard_label) + (1-α) · KL(student_logits/T, teacher_logits/T)

import torch.nn.functional as F

def distillation_loss(student_logits, teacher_logits, labels, T=4.0, alpha=0.7):
    soft = F.kl_div(
        F.log_softmax(student_logits / T, dim=-1),
        F.softmax(teacher_logits / T, dim=-1),
        reduction="batchmean",
    ) * (T * T)   # 乘 T^2 保持梯度尺度
    hard = F.cross_entropy(student_logits, labels)
    return alpha * hard + (1 - alpha) * soft

# 用 Teacher 生成指令数据，再用 LoRA 微调 Student
prompts = load_seed_prompts("seeds.jsonl")
dataset = []
for p in prompts:
    answer = teacher.generate(p, temperature=0.7, top_p=0.95)
    if quality_filter(answer):        # 规则 + 打分模型过滤
        dataset.append({"instruction": p, "output": answer})
save_jsonl(dataset, "distill_train.jsonl")

大模型 (70B FP16)
  └─ 蒸馏 → 小模型 (7B FP16)         [省算力]
      └─ 剪枝 → 稀疏 7B              [省参数，需硬件支持]
          └─ 量化 → 7B INT4          [省显存]
              └─ KV 量化 → INT8 KV   [省长上下文显存]

需要微调？
├─ 是 → QLoRA (NF4 + LoRA)，单卡 24GB 可训 7B
└─ 否 → 有 GPU？
        ├─ 是（A100/H100）→ AWQ 4bit + vLLM，吞吐最优
        ├─ 是（消费级 4090）→ GPTQ/AWQ 4bit，group 128
        └─ 否（CPU / Mac）→ GGUF Q4_K_M + llama.cpp

vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --quantization fp8 \
  --kv-cache-dtype fp8

import torch, json
from transformers import AutoModelForCausalLM, AutoTokenizer
from datasets import load_dataset

def perplexity(model, tokenizer, texts, max_len=2048) -> float:
    model.eval()
    nlls, count = [], 0
    for text in texts:
        ids = tokenizer(text, return_tensors="pt", truncation=True,
                        max_length=max_len).input_ids.to(model.device)
        if ids.size(1) < 8:
            continue
        with torch.no_grad():
            out = model(ids, labels=ids)
        nlls.append(out.loss.float() * (ids.size(1) - 1))
        count += ids.size(1) - 1
    return torch.exp(torch.stack(nlls).sum() / count).item()

def compare(fp16_path: str, quant_path: str, dataset: str = "wikitext-2-raw-v1"):
    ds = load_dataset("wikitext", dataset, split="test")
    texts = [t for t in ds["text"] if len(t.strip()) > 200][:200]

    results = {}
    for name, path in [("fp16", fp16_path), ("quant", quant_path)]:
        tok = AutoTokenizer.from_pretrained(path)
        mdl = AutoModelForCausalLM.from_pretrained(path, device_map="auto")
        results[name] = perplexity(mdl, tok, texts)
        del mdl
        torch.cuda.empty_cache()

    delta = (results["quant"] / results["fp16"] - 1) * 100
    results["delta_pct"] = round(delta, 3)
    print(json.dumps(results, indent=2))
    return results["delta_pct"] < 2.0   # 阈值 2%

if __name__ == "__main__":
    ok = compare("Qwen2.5-7B-Instruct", "qwen2.5-7b-awq")
    print("PASS" if ok else "FAIL")