多模态视觉语言模型：CLIP、ViT 与 LLaVA 类 VLM 架构全解析

       文本1  文本2  文本3
图像1 [  高    低    低 ]   ← 对角线上是正样本
图像2 [  低    高    低 ]
图像3 [  低    低    高 ]

import torch
from PIL import Image
from transformers import CLIPProcessor, CLIPModel

model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")

image = Image.open("cup.jpg")
texts = ["a photo of a red cup", "a photo of a blue cup", "a photo of a cat"]

inputs = processor(text=texts, images=image, return_tensors="pt", padding=True)
with torch.no_grad():
    outputs = model(**inputs)

# logits_per_image: 图像与每条文本的相似度
probs = outputs.logits_per_image.softmax(dim=1)
for t, p in zip(texts, probs[0]):
    print(f"{t}: {p:.3f}")

图像 224×224×3
  → 切 16×16 patch → 196 个 patch
  → 每个 patch 展平(768维) → 线性投影 → 196 个 token
  → 加位置编码 + [CLS] → Transformer → 分类

import torch
import torch.nn as nn

class PatchEmbed(nn.Module):
    def __init__(self, img_size=224, patch_size=16, in_ch=3, dim=768):
        super().__init__()
        self.num_patches = (img_size // patch_size) ** 2
        # 用 stride=patch_size 的卷积等价于切 patch + 线性投影
        self.proj = nn.Conv2d(in_ch, dim, kernel_size=patch_size, stride=patch_size)

    def forward(self, x):
        x = self.proj(x)                  # (B, dim, H/P, W/P)
        x = x.flatten(2).transpose(1, 2)  # (B, num_patches, dim)
        return x

class ViT(nn.Module):
    def __init__(self, dim=768, depth=12, heads=12, num_classes=1000):
        super().__init__()
        self.patch_embed = PatchEmbed(dim=dim)
        self.cls_token = nn.Parameter(torch.zeros(1, 1, dim))
        self.pos_embed = nn.Parameter(torch.zeros(1, 197, dim))
        encoder_layer = nn.TransformerEncoderLayer(dim, heads, batch_first=True)
        self.encoder = nn.TransformerEncoder(encoder_layer, depth)
        self.head = nn.Linear(dim, num_classes)

    def forward(self, x):
        x = self.patch_embed(x)
        cls = self.cls_token.expand(x.size(0), -1, -1)
        x = torch.cat([cls, x], dim=1) + self.pos_embed
        x = self.encoder(x)
        return self.head(x[:, 0])   # 取 CLS token 做分类

图像 → [视觉编码器 ViT/CLIP] → 视觉特征
     → [投影层 Projector]    → 视觉 token（对齐到 LLM 词向量空间）
     → [拼接文本 token]      → [LLM] → 文本输出

import torch
import torch.nn as nn
from transformers import CLIPVisionModel, LlavaForConditionalGeneration

class LlavaProjector(nn.Module):
    """两层 MLP 投影：视觉特征 → LLM 词向量空间"""
    def __init__(self, vision_dim=1024, llm_dim=4096):
        super().__init__()
        self.mlp = nn.Sequential(
            nn.Linear(vision_dim, llm_dim),
            nn.GELU(),
            nn.Linear(llm_dim, llm_dim),
        )

    def forward(self, vision_features):
        return self.mlp(vision_features)

# 实际推理直接用 transformers 的整合模型
model = LlavaForConditionalGeneration.from_pretrained(
    "llava-hf/llava-1.5-7b-hf", torch_dtype=torch.float16, device_map="auto"
)

from transformers import AutoProcessor
processor = AutoProcessor.from_pretrained("llava-hf/llava-1.5-7b-hf")
conversation = [
    {"role": "user", "content": [
        {"type": "image"},
        {"type": "text", "text": "图中杯子是什么颜色？"},
    ]},
]
prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
inputs = processor(images=image, text=prompt, return_tensors="pt").to("cuda")
out = model.generate(**inputs, max_new_tokens=64)
print(processor.decode(out[0], skip_special_tokens=True))

冻结：视觉编码器 ❄️  +  LLM ❄️
训练：投影层 🔥
数据：图文对（caption 任务）

from peft import LoraConfig, get_peft_model

lora_config = LoraConfig(
    r=16, lora_alpha=32, lora_dropout=0.05,
    target_modules=["q_proj", "v_proj"],
    task_type="CAUSAL_LM",
)
model = get_peft_model(model, lora_config)
model.print_trainable_parameters()   # 通常 < 1%

1. 提高分辨率 / 切图，让模型真的看见
2. 指令微调数据里加入"图中没有X"的负样本
3. 解码时用视觉对比（VCD）：对比有图/无图时的输出差异
4. 用更强视觉编码器 + 更多视觉 token

# 图文检索的向量化（复用 CLIP）
import torch
from transformers import CLIPModel, CLIPProcessor

model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
proc = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")

def embed_image(img):
    inputs = proc(images=img, return_tensors="pt")
    with torch.no_grad():
        return model.get_image_features(**inputs)

def embed_text(text):
    inputs = proc(text=[text], return_tensors="pt", padding=True)
    with torch.no_grad():
        return model.get_text_features(**inputs)

# 用 vLLM 部署 VLM（支持多模态输入）
python -m vllm.entrypoints.openai.api_server \
  --model llava-hf/llava-1.5-7b-hf \
  --dtype float16 --max-model-len 4096 --gpu-memory-utilization 0.9

对齐层：CLIP 对比学习 → 共享语义空间
视觉层：ViT → patch 序列（可换 SigLIP、DINOv2）
桥接层：投影层 MLP / Q-Former → 视觉 token
语言层：LLM → 推理与生成
训练层：对齐 → 指令微调 → 特化