NLP 入门实战：文本向量化、词嵌入与情感分析

import re
import jieba  # 中文分词

def preprocess(text: str) -> list[str]:
    text = text.lower()                     # 1. 小写（英文）
    text = re.sub(r'[^\w一-鿿\s]', '', text)  # 2. 去标点/特殊符号
    words = jieba.lcut(text) if has_chinese(text) else text.split()  # 3. 分词
    return words

from sklearn.feature_extraction.text import CountVectorizer

corpus = ["我喜欢机器学习", "机器学习很有趣"]
vec = CountVectorizer(tokenizer=jieba.lcut, ngram_range=(1, 2))
X = vec.fit_transform(corpus)   # (2, 词表大小) 稀疏矩阵

from sklearn.feature_extraction.text import TfidfVectorizer

tfidf = TfidfVectorizer(tokenizer=jieba.lcut, max_features=10000)
X = tfidf.fit_transform(corpus)

king − man + woman ≈ queen

# 训练或加载词向量（gensim）
from gensim.models import Word2Vec

sentences = [["我喜欢", "机器学习"], ["机器学习", "很有趣"]]
model = Word2Vec(sentences, vector_size=100, window=5, min_count=1)
sim = model.wv.most_similar("机器学习")   # 语义近邻

# 用预训练词向量做文本表示
import numpy as np
def embed_text(words, wv):
    vecs = [wv[w] for w in words if w in wv]
    return np.mean(vecs, axis=0) if vecs else np.zeros(100)

RNN/LSTM（第 1 代）         Transformer（第 2 代）
逐词串行、难并行            自注意力、全并行
长序列梯度难                可建模长距离依赖

Attention(Q, K, V) = softmax(QK^T / sqrt(d)) V
Q（查询）K（键）V（值）都来自输入；权重 = 词与词的相关度

import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report

# 数据：review 文本 + label 0/1
df = pd.read_csv('reviews.csv')
X_train, X_test, y_train, y_test = train_test_split(
    df['review'], df['label'], test_size=0.2, random_state=42)

# TF-IDF 只 fit 训练集（防数据泄露！）
tfidf = TfidfVectorizer(tokenizer=jieba.lcut, max_features=10000)
X_tr = tfidf.fit_transform(X_train)
X_te = tfidf.transform(X_test)

model = LogisticRegression(C=1.0, max_iter=1000)
model.fit(X_tr, y_train)

print(classification_report(y_test, model.predict(X_te)))
# 可解释：看 Top 特征词
top = sorted(zip(tfidf.get_feature_names_out(), model.coef_[0]),
             key=lambda x: x[1], reverse=True)[:10]
print('正面词:', [w for w, _ in top])

import torch, torch.nn as nn
from torch.utils.data import Dataset, DataLoader

# 把文本映射到 id 序列
from torchtext.data.utils import get_tokenizer
from torchtext.vocab import build_vocab_from_iterator

def ids_of(reviews, vocab):
    tok = get_tokenizer(lambda s: jieba.lcut(s))
    return [vocab(tok(r)) for r in reviews]

# 简单 RNN 分类器
class SentimentRNN(nn.Module):
    def __init__(self, vocab_size, embed_dim=64, hidden=64, num_class=2):
        super().__init__()
        self.embed = nn.Embedding(vocab_size, embed_dim, padding_idx=0)
        self.rnn = nn.LSTM(embed_dim, hidden, batch_first=True)
        self.fc = nn.Linear(hidden, num_class)

    def forward(self, x):
        emb = self.embed(x)                    # (B, L, E)
        _, (h, _) = self.rnn(emb)              # 取最后隐藏态
        return self.fc(h[-1])                  # (B, 2)

# 训练循环同 /ml-deep-learning-advanced/ 的标准流程
# 要点：padding 到同长、mask 掉 padding 的损失

from transformers import pipeline

# 一条 pipeline 搞定情感分析（默认微调好的 BERT）
classifier = pipeline("sentiment-analysis",
    model="uer/roberta-base-finetuned-jd-binary-chinese")  # 中文
result = classifier("这家店的东西很好用，推荐！")
# [{'label': 'positive', 'score': 0.99}]

# 其他开箱任务
summarizer = pipeline("summarization")
translator = pipeline("translation", model="Helsinki-NLP/opus-mt-zh-en")

from transformers import AutoTokenizer, AutoModelForSequenceClassification, Trainer

tokenizer = AutoTokenizer.from_pretrained("bert-base-chinese")
model = AutoModelForSequenceClassification.from_pretrained(
    "bert-base-chinese", num_labels=2)

# Trainer 封装训练/评估/保存，见 HuggingFace 文档