语音与音频机器学习：MFCC、CTC/RNN-T、Whisper、TTS 与声码器

波形 (16000 点/秒)
  → 分帧：25ms 一帧、10ms 一帧移 → 每帧 400 点
  → 加窗：汉明窗
  → STFT：每帧 → 频谱
  → 梅尔滤波 → 梅尔谱（80 或 128 维）

import torch
import torchaudio
import torchaudio.transforms as T

waveform, sr = torchaudio.load("speech.wav")   # (channels, samples)
if sr != 16000:
    waveform = torchaudio.functional.resample(waveform, sr, 16000)

mel_transform = T.MelSpectrogram(
    sample_rate=16000, n_fft=400, hop_length=160, n_mels=80, f_min=0, f_max=8000,
)
mfcc_transform = T.MFCC(
    sample_rate=16000, n_mfcc=13,
    melkwargs={"n_fft": 400, "hop_length": 160, "n_mels": 80},
)

mel = mel_transform(waveform)          # (1, 80, T)
mfcc = mfcc_transform(waveform)        # (1, 13, T)
log_mel = torch.log(mel + 1e-6)        # 取对数，动态范围更友好
print(mel.shape, mfcc.shape)

帧输出： _ _ h h _ e e _ l l l _ l l _ o o _ _
合并后： h e l l o

import torch
import torch.nn as nn

# CTC 损失：log_probs 形状 (T, N, C)，targets 是文本 id
ctc_loss = nn.CTCLoss(blank=0, zero_infinity=True)
log_probs = model(mel)                      # (T, N, C)
loss = ctc_loss(log_probs, targets, input_lengths, target_lengths)

音频 → 30s 窗口 → log-Mel (80维) → Encoder(Transformer) → Decoder → 文本 token
Decoder 前缀：[<|startoftranscript|><|zh|><|transcribe|><|notimestamps|>]

import torch
from transformers import WhisperProcessor, WhisperForConditionalGeneration

processor = WhisperProcessor.from_pretrained("openai/whisper-large-v3")
model = WhisperForConditionalGeneration.from_pretrained(
    "openai/whisper-large-v3", torch_dtype=torch.float16
).to("cuda")

import librosa
audio, sr = librosa.load("speech.wav", sr=16000)
inputs = processor(audio, sampling_rate=16000, return_tensors="pt").to("cuda")

forced_ids = processor.get_decoder_prompt_ids(language="zh", task="transcribe")
ids = model.generate(inputs.input_features, forced_decoder_ids=forced_ids, max_new_tokens=256)
print(processor.batch_decode(ids, skip_special_tokens=True)[0])

# faster-whisper：CTranslate2 加速 + int8 量化
pip install faster-whisper

from faster_whisper import WhisperModel

model = WhisperModel("large-v3", device="cuda", compute_type="float16")
segments, info = model.transcribe("long_audio.wav", language="zh", vad_filter=True)
for seg in segments:
    print(f"[{seg.start:.2f}-{seg.end:.2f}] {seg.text}")

文本 "你好" → [文本前端/音素] → [声学模型 Tacotron/FastSpeech] → 梅尔谱
             → [声码器 HiFi-GAN] → 波形

from TTS.api import TTS   # Coqui TTS

tts = TTS("tts_models/zh-CN/baker/tacotron2-DDC-GST").to("cuda")
tts.tts_to_file(text="你好，欢迎使用语音合成。", file_path="out.wav")

# 零样本音色克隆
tts_clone = TTS("tts_models/multilingual/multi-dataset/xtts_v2").to("cuda")
tts_clone.tts_to_file(
    text="这是用参考音色说的话。",
    speaker_wav="reference.wav",
    language="zh",
    file_path="clone.wav",
)

from speechbrain.inference.speaker import EncoderClassifier
import torch
import torchaudio

classifier = EncoderClassifier.from_hparams(
    source="speechbrain/spkrec-ecapa-voxceleb", run_opts={"device": "cuda"}
)

def embed(path):
    signal, sr = torchaudio.load(path)
    if sr != 16000:
        signal = torchaudio.functional.resample(signal, sr, 16000)
    return classifier.encode_batch(signal).squeeze()   # (192,)

e1, e2 = embed("a.wav"), embed("b.wav")
score = torch.nn.functional.cosine_similarity(e1, e2, dim=0)
print(f"相似度: {score:.3f}")   # 阈值通常 0.25-0.35

import torch
import torchaudio.transforms as T

spec_aug = T.FrequencyMasking(freq_mask_param=27)
time_aug = T.TimeMasking(time_mask_param=100)

def augment(mel):
    # mel: (C, n_mels, T)
    for _ in range(2):
        mel = spec_aug(mel)   # 频率掩蔽
        mel = time_aug(mel)   # 时间掩蔽
    return mel

import torch
import torchaudio

def add_noise(waveform, noise, snr_db=10):
    """按指定信噪比混合噪声"""
    signal_power = waveform.norm(p=2) ** 2
    noise_power = noise.norm(p=2) ** 2
    snr = 10 ** (snr_db / 10)
    scale = (signal_power / (snr * noise_power + 1e-8)).sqrt()
    return waveform + scale * noise[:, :waveform.size(-1)]

import jiwer

reference = "今天天气很好"
hypothesis = "今天天气好"
print("CER:", jiwer.cer(reference, hypothesis))   # 0.2

1. 前端：VAD 切分 + 降噪 + 重采样统一到 16kHz
2. 模型：faster-whisper / onnx 导出 + int8 量化
3. 后处理：标点恢复 + 数字归一化 + 热词替换
4. 部署：流式用 RNN-T/流式 Whisper，离线用大模型
5. 监控：记录 WER 抽样、RTF、失败率

# Whisper 用 initial_prompt 做热词偏置
ids = model.generate(
    inputs.input_features,
    initial_prompt="以下是关于 Plumephp 框架和 Kubernetes 的技术讨论。",
    language="zh",
)

特征层：波形 → 分帧 → 梅尔谱（MFCC）
识别层：CTC / RNN-T（流式）/ 注意力编码解码（精度）
合成层：声学模型 + 声码器（HiFi-GAN）/ 端到端 TTS
表征层：说话人嵌入（ECAPA-TDNN）+ 余弦相似度
工程层：VAD + 增强（SpecAugment）+ 后处理 + 量化