commit 20ae3cfb4883d8c2ec7e1768d3d20fd92eee5403 Author: jrodriiguezg Date: Wed Sep 9 18:22:40 2026 +0200 feat: initial commit with vLLM production stack and benchmark tools diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..e09eb96 --- /dev/null +++ b/.env.example @@ -0,0 +1,10 @@ +# Target AWQ-quantized model ID +MODEL_ID=Qwen/Qwen2.5-3B-Instruct-AWQ + +# Hugging Face Token (optional, only for gated models) +HF_TOKEN="" + +# vLLM inference parameters +MAX_MODEL_LEN=4096 +GPU_MEMORY_UTILIZATION=0.90 +MAX_NUM_SEQS=64 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d206ce2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +# Model weights cache (multi-GB data) +models_cache/ +*.bin +*.safetensors + +# Local environment file +.env + +# Docker volumes +*-data/ + +# Python cache & virtualenv +__pycache__/ +*.py[cod] +.venv/ +venv/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..08691ab --- /dev/null +++ b/README.md @@ -0,0 +1,64 @@ +# vLLM Production Stack: Concurrency, Resilience & Telemetry + +Production-grade LLM inference deployment featuring **vLLM** (PagedAttention & Continuous Batching), **LiteLLM Gateway** (circuit breakers & fallbacks), and full observability via **Prometheus** and **Grafana**. + +Read the full technical breakdown on my blog: [De Ollama a Producción: Desplegando vLLM con PagedAttention y métricas en tiempo real](https://blog.jrodriiguezg.link) + +--- + +## Architecture + +- **Engine:** [vLLM](https://github.com/vllm-project/vllm) with AWQ quantization (`Qwen/Qwen2.5-3B-Instruct-AWQ`), PagedAttention, and Continuous Batching. +- **Gateway & Resilience:** [LiteLLM Proxy](https://github.com/BerriAI/litellm) providing OpenAI-compatible routing, timeouts, and silent failover. +- **Metrics Scraper:** [Prometheus](https://prometheus.io/) scraping the native `/metrics` endpoint every 2s. +- **Dashboards:** [Grafana](https://grafana.com/) for real-time visualization of TTFT, TPOT, and KV Cache utilization. + +--- + +## Quick Start + +### 1. Prerequisites +- Linux OS (Fedora / RHEL / Debian) +- NVIDIA GPU with proprietary drivers (`nvidia-smi`) +- Docker Engine & NVIDIA Container Toolkit (`nvidia-ctk`) + +### 2. Configuration +Copy the sample environment file: +```bash +cp .env.example .env +``` + +### 3. Launch the Stack +```bash +docker compose up -d +``` + +Check logs and health status: +```bash +docker compose logs -f vllm +curl http://localhost:8000/health +``` + +--- + +## Load & Concurrency Benchmark + +Stress-test the deployment with the included asynchronous Python benchmark: + +```bash +pip install -r scripts/requirements.txt + +# Run 20 concurrent requests against vLLM +python3 scripts/benchmark.py --concurrency 20 --url http://localhost:4000/v1/chat/completions --model production-model +``` + +--- + +## Service Endpoints + +| Service | Port | Description | +| :--- | :--- | :--- | +| **LiteLLM Gateway** | `http://localhost:4000` | OpenAI-compatible endpoint with circuit breaker | +| **vLLM Engine** | `http://localhost:8000` | Raw inference API & `/metrics` | +| **Prometheus** | `http://localhost:9090` | Telemetry scraper & PromQL console | +| **Grafana** | `http://localhost:3000` | Dashboards (`admin` / `admin`) | diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..898b992 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,101 @@ +version: '3.8' + +services: + # 1. MOTOR DE INFERENCIA DE PRODUCCIÓN (vLLM) + vllm: + image: vllm/vllm-openai:latest + container_name: vllm-engine + restart: unless-stopped + environment: + - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN} + - VLLM_LOGGING_LEVEL=INFO + volumes: + - ./models_cache:/root/.cache/huggingface:Z # :Z para compatibilidad SELinux en RHEL/Fedora + ports: + - "8000:8000" + command: > + --model ${MODEL_ID} + --quantization awq + --dtype half + --gpu-memory-utilization ${GPU_MEMORY_UTILIZATION} + --max-model-len ${MAX_MODEL_LEN} + --max-num-seqs ${MAX_NUM_SEQS} + --block-size 16 + --port 8000 + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + networks: + - vllm-net + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 60s + + # 2. PROXY DE RED, RESILIENCIA Y CIRCUIT BREAKER (LiteLLM) + litellm: + image: ghcr.io/berriai/litellm:main-latest + container_name: litellm-gateway + restart: unless-stopped + volumes: + - ./litellm/config.yaml:/app/config.yaml:ro,Z + ports: + - "4000:4000" + command: ["--config", "/app/config.yaml", "--port", "4000"] + depends_on: + vllm: + condition: service_healthy + networks: + - vllm-net + + # 3. OBSERVABILIDAD: RECOLECTOR DE TELEMETRÍA (Prometheus) + prometheus: + image: prom/prometheus:latest + container_name: prometheus-telemetry + restart: unless-stopped + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro,Z + - prometheus-data:/prometheus:Z + ports: + - "9090:9090" + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/usr/share/prometheus/console_libraries' + - '--web.console.templates=/usr/share/prometheus/consoles' + networks: + - vllm-net + + # 4. OBSERVABILIDAD: VISUALIZACIÓN EN TIEMPO REAL (Grafana) + grafana: + image: grafana/grafana:latest + container_name: grafana-dashboard + restart: unless-stopped + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_USERS_ALLOW_SIGN_UP=false + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro,Z + - grafana-data:/var/lib/grafana:Z + ports: + - "3000:3000" + depends_on: + - prometheus + networks: + - vllm-net + +networks: + vllm-net: + driver: bridge + +volumes: + prometheus-data: + grafana-data: + diff --git a/grafana/provisioning/dashboards/.gitkeep b/grafana/provisioning/dashboards/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/grafana/provisioning/datasources/datasource.yml b/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 0000000..bb009bb --- /dev/null +++ b/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false diff --git a/litellm/config.yaml b/litellm/config.yaml new file mode 100644 index 0000000..823c565 --- /dev/null +++ b/litellm/config.yaml @@ -0,0 +1,27 @@ +model_list: + # Modelo primario apuntando a nuestro motor vLLM local + - model_name: production-model + litellm_params: + model: openai/Qwen/Qwen2.5-3B-Instruct-AWQ + api_base: http://vllm:8000/v1 + api_key: "token-local-vllm" + request_timeout: 30 # Timeout para evitar colgar al cliente + + # Modelo de fallback (contingencia) en caso de saturación o caída del primario + # Puede ser un modelo secundario local o un proveedor externo + # - model_name: fallback-backup + # litellm_params: + # model: openai/gpt-4o-mini # O una segunda instancia vLLM con modelo ligero + # api_key: "dummy-key-o-real" + +router_settings: + routing_strategy: "latency-based-routing" + timeout: 30 + fallbacks: + - "production-model": ["fallback-backup"] + num_retries: 2 + allowed_fails: 3 + cooldown_time: 15 # Segundos antes de reintentar el modelo primario tras abrir circuito + +general_settings: + master_key: "sk-production-admin-key" diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml new file mode 100644 index 0000000..d2d239b --- /dev/null +++ b/prometheus/prometheus.yml @@ -0,0 +1,19 @@ +global: + scrape_interval: 2s # Intervalo corto para capturar datos de latencia y saturación + evaluation_interval: 2s + +scrape_configs: + - job_name: 'vllm' + metrics_path: '/metrics' + static_configs: + - targets: ['vllm:8000'] + labels: + engine: 'vllm' + model: 'qwen2.5-3b-awq' + + - job_name: 'litellm' + metrics_path: '/metrics' + static_configs: + - targets: ['litellm:4000'] + labels: + service: 'llm-gateway' diff --git a/scripts/benchmark.py b/scripts/benchmark.py new file mode 100644 index 0000000..7e7eab7 --- /dev/null +++ b/scripts/benchmark.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +""" +Benchmark de Concurrencia para Servidores de Inferencia LLM +Compara comportamiento ante ráfagas concurrentes de peticiones. +""" + +import asyncio +import time +import aiohttp +import statistics +import argparse + +PROMPT = "Explica en tres párrafos técnicos qué es la memoria virtual y cómo se gestionan las páginas de memoria en el kernel de Linux." + +async def send_request(session, url, model, headers, req_id): + # Envia una peticion individual y mide su latencia + payload = { + "model": model, + "messages": [{"role": "user", "content": PROMPT}], + "max_tokens": 150, + "temperature": 0.7 + } + + start_time = time.perf_counter() + try: + async with session.post(url, json=payload, headers=headers, timeout=aiohttp.ClientTimeout(total=60)) as resp: + data = await resp.json() + latency = time.perf_counter() - start_time + if resp.status == 200: + tokens = data["usage"]["completion_tokens"] + return {"id": req_id, "success": True, "latency": latency, "tokens": tokens} + else: + return {"id": req_id, "success": False, "latency": latency, "error": resp.status} + except Exception as e: + latency = time.perf_counter() - start_time + return {"id": req_id, "success": False, "latency": latency, "error": str(e)} + +async def run_benchmark(url, model, concurrency, auth_header): + # Ejecuta peticiones concurrentes y calcula metricas + headers = {"Content-Type": "application/json"} + if auth_header: + headers["Authorization"] = f"Bearer {auth_header}" + + print("\n=======================================================") + print(f"Iniciando Benchmark: {concurrency} peticiones CONCURRENTES") + print(f"Target: {url} | Modelo: {model}") + print("=======================================================") + + async with aiohttp.ClientSession() as session: + t0 = time.perf_counter() + tasks = [send_request(session, url, model, headers, i) for i in range(concurrency)] + results = await asyncio.gather(*tasks) + total_wall_time = time.perf_counter() - t0 + + successful = [r for r in results if r["success"]] + failed = [r for r in results if not r["success"]] + + if successful: + latencies = [r["latency"] for r in successful] + total_tokens = sum(r["tokens"] for r in successful) + avg_latency = statistics.mean(latencies) + p95_latency = statistics.quantiles(latencies, n=20)[18] if len(latencies) >= 20 else max(latencies) + throughput_tokens_sec = total_tokens / total_wall_time + + print("\nRESULTADOS:") + print(f" - Peticiones exitosas: {len(successful)}/{concurrency}") + print(f" - Fallidas / Timeout: {len(failed)}") + print(f" - Tiempo total del test: {total_wall_time:.2f} s") + print(f" - Throughput global: {throughput_tokens_sec:.2f} tokens/segundo") + print(f" - Latencia promedio: {avg_latency:.2f} s") + print(f" - Latencia P95: {p95_latency:.2f} s") + else: + print(f"\nTodas las peticiones fallaron. Errores: {[r.get('error') for r in failed]}") + +if __name__ == "__main__": + # Parser de argumentos por linea de comandos + parser = argparse.ArgumentParser() + parser.add_argument("--url", default="http://localhost:4000/v1/chat/completions", help="Endpoint OpenAI-compatible") + parser.add_argument("--model", default="production-model", help="Nombre del modelo") + parser.add_argument("--concurrency", type=int, default=20, help="Numero de peticiones concurrentes") + parser.add_argument("--key", default="sk-production-admin-key", help="API Key si aplica") + args = parser.parse_args() + + asyncio.run(run_benchmark(args.url, args.model, args.concurrency, args.key)) diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 0000000..ee4ba4f --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1 @@ +aiohttp