LLM 部署实践：vLLM, TGI, Ollama

# 估算模型显存需求
def estimate_vram(params_billions: float, dtype: str) -> float:
    dtype_bytes = {"float16": 2, "int8": 1, "int4": 0.5}
    bytes_per_param = dtype_bytes.get(dtype, 2)
    # 模型权重 + KV Cache 预留 (约 20-40%)
    model_vram = params_billions * 1e9 * bytes_per_param / 1e9
    kv_cache_reserve = model_vram * 0.3
    return model_vram + kv_cache_reserve

vram = estimate_vram(70, "int4")
print(f"70B INT4 预估显存: {vram:.1f} GB")  # 约 48.1 GB

# 部署需求清单模板 (infra/requirements.yaml)
serving:
  max_latency_ms: 200        # 首 token 延迟上限
  target_qps: 50             # 预期每秒查询数
  max_concurrent_users: 500  # 最大并发用户
  model_size: "70B"
  quantization: "int4"
  budget_gpu: "A100-80GB x 2"
  availability: "99.9%"

# 安装 vLLM 并启动 API 服务
pip install vllm

# 单卡部署 Qwen2.5-7B
vllm serve Qwen/Qwen2.5-7B-Instruct \
    --host 0.0.0.0 \
    --port 8000 \
    --tensor-parallel-size 1 \
    --max-num-seqs 256 \
    --gpu-memory-utilization 0.90

# 多卡张量并行（4 卡 A100 部署 70B 模型）
vllm serve Qwen/Qwen2.5-72B-Instruct \
    --tensor-parallel-size 4 \
    --quantization awq

# vLLM Python API：离线批量推理
from vllm import LLM, SamplingParams

llm = LLM(
    model="Qwen/Qwen2.5-7B-Instruct",
    gpu_memory_utilization=0.9,
    max_model_len=8192,
)

prompts = [
    "解释量子计算的基本原理",
    "用 Python 写一个快速排序",
    "分析 2024 年 AI 行业发展趋势",
]
params = SamplingParams(temperature=0.7, max_tokens=512)
outputs = llm.generate(prompts, params)

for o in outputs:
    print(f"Prompt: {o.prompt}")
    print(f"Output: {o.outputs[0].text}\n")

# TGI Docker 部署
docker run --gpus all \
    --shm-size 1g \
    -p 8080:80 \
    -v /data:/data \
    ghcr.io/huggingface/text-generation-inference:2.2.0 \
    --model-id Qwen/Qwen2.5-7B-Instruct \
    --num-shard 1 \
    --quantize awq \
    --max-input-length 4096 \
    --max-total-tokens 8192 \
    --max-batch-prefill-tokens 16384

# 测试 API 端点
curl http://localhost:8080/v1/chat/completions \
    -H "Content-Type: application/json" \
    -d '{"model":"Qwen/Qwen2.5-7B-Instruct","messages":[{"role":"user","content":"你好"}]}'

# TGI Python 客户端（openai 兼容模式）
from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:8080/v1",
    api_key="no-key-needed",
)

# 流式对话
response = client.chat.completions.create(
    model="Qwen/Qwen2.5-7B-Instruct",
    messages=[
        {"role": "system", "content": "你是一个专业的 AI 助手。"},
        {"role": "user", "content": "如何优化大模型的推理速度？"},
    ],
    stream=True,
    max_tokens=1024,
    temperature=0.7,
)

for chunk in response:
    if chunk.choices[0].delta.content:
        print(chunk.choices[0].delta.content, end="", flush=True)

# 安装 Ollama（macOS / Linux）
curl -fsSL https://ollama.com/install.sh | sh

# 拉取并运行模型
ollama pull qwen2.5:7b
ollama pull qwen2.5:72b  # Apple Silicon M2/M3 推荐

# 启动对话
ollama run qwen2.5:7b "解释 Transformer 架构"

# 查看已加载模型
ollama list
ollama ps

# 创建自定义 Modelfile
cat > Modelfile << 'EOF'
FROM qwen2.5:7b
SYSTEM "你是一个资深程序员，擅长用简洁的代码解决问题。"
PARAMETER temperature 0.3
PARAMETER num_ctx 4096
EOF
ollama create my-coder -f Modelfile

# Ollama Python SDK
from ollama import chat, ChatResponse

# 简单对话
res: ChatResponse = chat(
    model="qwen2.5:7b",
    messages=[
        {"role": "system", "content": "简洁回答，不要废话。"},
        {"role": "user", "content": "什么是 PagedAttention？"},
    ],
    options={
        "temperature": 0.2,
        "num_predict": 256,
    },
)
print(res.message.content)

# 流式生成
for chunk in chat(
    model="qwen2.5:7b",
    messages=[{"role": "user", "content": "写一首关于 AI 的诗"}],
    stream=True,
):
    print(chunk.message.content, end="", flush=True)

# LiteLLM 代理网关：统一管理多个推理后端
pip install litellm[proxy]

# config.yaml：定义多个模型端点
cat > litellm_config.yaml << 'EOF'
model_list:
  - model_name: gpt-4o-mini
    litellm_params:
      model: openai/Qwen/Qwen2.5-7B-Instruct
      api_base: http://localhost:8000/v1
      api_key: "sk-internal"
  - model_name: gpt-4o-mini
    litellm_params:
      model: openai/Qwen/Qwen2.5-7B-Instruct
      api_base: http://localhost:8080/v1
      api_key: "unused"
  - model_name: ollama-local
    litellm_params:
      model: ollama/qwen2.5:7b
      api_base: http://localhost:11434

litellm_settings:
  drop_params: true
  max_budget: 100
  budget_duration: "30d"
EOF

litellm --config litellm_config.yaml --port 4000

# FastAPI 封装推理服务（自定义鉴权 + 限流）
from fastapi import FastAPI, HTTPException, Header
from openai import AsyncOpenAI
import asyncio

app = FastAPI()
client = AsyncOpenAI(
    base_url="http://localhost:8000/v1",
    api_key="internal",
)

API_KEYS = {"sk-prod-001": "user-a", "sk-prod-002": "user-b"}
RATE_LIMITS: dict[str, list[float]] = {}

def check_rate_limit(key: str) -> None:
    import time
    now = time.time()
    window = [t for t in RATE_LIMITS.get(key, []) if now - t < 60]
    if len(window) >= 30:  # 30 requests / minute
        raise HTTPException(status_code=429, detail="Rate limit exceeded")
    RATE_LIMITS[key] = window + [now]

@app.post("/v1/chat/completions")
async def chat(
    body: dict,
    authorization: str = Header(...),
):
    key = authorization.replace("Bearer ", "")
    if key not in API_KEYS:
        raise HTTPException(status_code=401, detail="Invalid API key")
    check_rate_limit(key)
    return await client.chat.completions.create(body)

# vLLM 内置基准测试
pip install vllm

# 使用共享数据集测试
python -m vllm.entrypoints.openai.api_server \
    --model Qwen/Qwen2.5-7B-Instruct &

# 发送基准测试（模拟 100 并发，1000 请求）
python -m vllm.benchmark_serving \
    --backend openai \
    --model Qwen/Qwen2.5-7B-Instruct \
    --dataset-name sharegpt \
    --dataset-path ShareGPT_V3_unfiltered_cleaned_split.json \
    --num-prompts 1000 \
    --request-rate 100 \
    --save-result

# 输出示例：
# Throughput: 1523.45 requests/s
# Output Token Throughput: 45678.90 tokens/s
# Mean TTFT: 89.12ms
# Median TTFT: 76.45ms

# 自定义基准测试脚本
import asyncio
import time
from openai import AsyncOpenAI
from dataclasses import dataclass

@dataclass
class BenchResult:
    ttft_ms: float      # Time to first token
    total_ms: float     # End-to-end latency
    output_tokens: int  # Generated tokens
    tokens_per_sec: float

async def benchmark_one(client: AsyncOpenAI, prompt: str) -> BenchResult:
    start = time.perf_counter()
    first_token = None
    total_tokens = 0

    stream = await client.chat.completions.create(
        model="qwen2.5:7b",
        messages=[{"role": "user", "content": prompt}],
        stream=True,
        max_tokens=512,
    )
    async for chunk in stream:
        if first_token is None:
            first_token = time.perf_counter()
        if chunk.choices[0].delta.content:
            total_tokens += 1

    elapsed = time.perf_counter() - start
    return BenchResult(
        ttft_ms=(first_token - start) * 1000,
        total_ms=elapsed * 1000,
        output_tokens=total_tokens,
        tokens_per_sec=total_tokens / elapsed,
    )

# docker-compose.yml：完整生产部署
version: "3.9"
services:
  vllm:
    image: vllm/vllm-openai:latest
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 1
              capabilities: [gpu]
    command: >
      --model Qwen/Qwen2.5-7B-Instruct
      --max-model-len 8192
      --gpu-memory-utilization 0.9
      --enable-prefix-caching
    ports:
      - "8000:8000"
    volumes:
      - hf-cache:/root/.cache/huggingface

  litellm:
    image: ghcr.io/berriai/litellm:latest
    ports:
      - "4000:4000"
    volumes:
      - ./litellm_config.yaml:/app/config.yaml
    command: --config /app/config.yaml --port 4000
    depends_on:
      - vllm

  prometheus:
    image: prom/prometheus:latest
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml

volumes:
  hf-cache:

# 上线检查清单脚本
#!/bin/bash
set -euo pipefail

echo "=== LLM 部署上线检查 ==="

# 1. GPU 状态
echo "[1/5] 检查 GPU..."
nvidia-smi --query-gpu=index,name,memory.used,memory.total,utilization.gpu \\
  --format=csv,noheader

# 2. 服务健康
echo "[2/5] 检查 vLLM 服务..."
curl -sf http://localhost:8000/v1/models > /dev/null && echo "  vLLM: OK" || echo "  vLLM: FAIL"

# 3. API 网关
echo "[3/5] 检查 LiteLLM 网关..."
curl -sf http://localhost:4000/health > /dev/null && echo "  LiteLLM: OK" || echo "  LiteLLM: FAIL"

# 4. 端到端测试
echo "[4/5] 端到端推理测试..."
RESPONSE=$(curl -s http://localhost:4000/v1/chat/completions \\
  -H "Content-Type: application/json" \\
  -d '{"model":"gpt-4o-mini","messages":[{"role":"user","content":"1+1="}],"max_tokens":5}')
echo "$RESPONSE" | grep -q "2" && echo "  推理: OK" || echo "  推理: FAIL"

# 5. 延迟测试
echo "[5/5] 延迟测试..."
START=$(date +%s%N)
curl -sf http://localhost:8000/v1/chat/completions \\
  -H "Content-Type: application/json" \\
  -d '{"model":"Qwen/Qwen2.5-7B-Instruct","messages":[{"role":"user","content":"hi"}]}' \\
  > /dev/null
END=$(date +%s%N)
LATENCY=$(( (END - START) / 1000000 ))
echo "  首请求延迟: \${LATENCY}ms"

echo "=== 检查完成 ==="