440 lines
15 KiB
Python
440 lines
15 KiB
Python
"""Real end-to-end integration tests for LLM inference API.
|
|
|
|
These tests hit a real running server with real LLM API calls.
|
|
No mocking - all requests go to actual endpoints.
|
|
|
|
Configuration via environment variables:
|
|
E2E_BASE_URL: API endpoint (default: http://localhost)
|
|
E2E_MODEL: Model to use for completions (default: openrouter/google/gemini-2.0-flash-001)
|
|
|
|
Usage:
|
|
# Start the service first
|
|
./deploy/deploy.sh --profile api --detach
|
|
|
|
# Run all E2E tests
|
|
uv run pytest tests/test_e2e_real.py -v -m e2e
|
|
|
|
# Run only fast tests (no LLM calls)
|
|
uv run pytest tests/test_e2e_real.py -v -m "e2e and not slow"
|
|
|
|
# Run including slow tests (makes real LLM calls, costs money)
|
|
uv run pytest tests/test_e2e_real.py -v -m e2e
|
|
"""
|
|
|
|
import os
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
# Configuration from environment
|
|
BASE_URL = os.environ.get("E2E_BASE_URL", "http://localhost")
|
|
MODEL = os.environ.get("E2E_MODEL", "openrouter/google/gemini-2.0-flash-001")
|
|
TIMEOUT = 120.0 # LLM calls can be slow
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def client() -> httpx.Client:
|
|
"""Create an HTTP client for E2E tests."""
|
|
return httpx.Client(base_url=BASE_URL, timeout=TIMEOUT)
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def async_client() -> httpx.AsyncClient:
|
|
"""Create an async HTTP client for streaming tests."""
|
|
return httpx.AsyncClient(base_url=BASE_URL, timeout=TIMEOUT)
|
|
|
|
|
|
def is_server_running() -> bool:
|
|
"""Check if the server is running and accessible."""
|
|
try:
|
|
with httpx.Client(base_url=BASE_URL, timeout=5.0) as client:
|
|
response = client.get("/health")
|
|
return response.status_code == 200
|
|
except httpx.ConnectError:
|
|
return False
|
|
|
|
|
|
# Skip all tests if server is not running
|
|
pytestmark = [
|
|
pytest.mark.e2e,
|
|
pytest.mark.skipif(
|
|
not is_server_running(),
|
|
reason=f"Server not running at {BASE_URL}. Start with: ./deploy/deploy.sh --profile api --detach",
|
|
),
|
|
]
|
|
|
|
|
|
# =============================================================================
|
|
# Health Endpoint Tests
|
|
# =============================================================================
|
|
|
|
|
|
class TestHealthEndpoints:
|
|
"""Tests for health check endpoints."""
|
|
|
|
def test_health_returns_status(self, client: httpx.Client) -> None:
|
|
"""Test /health returns overall and backend health status."""
|
|
response = client.get("/health")
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
assert "status" in data
|
|
assert "backends" in data
|
|
assert data["status"] in ["healthy", "degraded", "unhealthy"]
|
|
|
|
# Verify backends structure
|
|
assert isinstance(data["backends"], list)
|
|
for backend in data["backends"]:
|
|
assert "name" in backend
|
|
assert "healthy" in backend
|
|
|
|
def test_ready_returns_readiness(self, client: httpx.Client) -> None:
|
|
"""Test /ready returns readiness status."""
|
|
response = client.get("/ready")
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
assert "ready" in data
|
|
assert isinstance(data["ready"], bool)
|
|
|
|
|
|
# =============================================================================
|
|
# Backends & Models Endpoint Tests
|
|
# =============================================================================
|
|
|
|
|
|
class TestBackendsEndpoints:
|
|
"""Tests for backend and model management endpoints."""
|
|
|
|
def test_list_backends(self, client: httpx.Client) -> None:
|
|
"""Test GET /v1/backends returns available backends."""
|
|
response = client.get("/v1/backends")
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
assert "backends" in data
|
|
assert isinstance(data["backends"], list)
|
|
assert len(data["backends"]) > 0
|
|
# At minimum, litellm should be available
|
|
assert "litellm" in data["backends"]
|
|
|
|
def test_list_models(self, client: httpx.Client) -> None:
|
|
"""Test GET /v1/models returns available models."""
|
|
response = client.get("/v1/models")
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
assert data["object"] == "list"
|
|
assert "data" in data
|
|
assert isinstance(data["data"], list)
|
|
|
|
def test_list_models_with_backend_filter(self, client: httpx.Client) -> None:
|
|
"""Test GET /v1/models?backend=litellm filters correctly."""
|
|
response = client.get("/v1/models?backend=litellm")
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
# All returned models should be from litellm
|
|
for model in data["data"]:
|
|
assert model["backend"] == "litellm"
|
|
|
|
def test_list_models_invalid_backend_returns_400(
|
|
self, client: httpx.Client
|
|
) -> None:
|
|
"""Test that invalid backend filter returns 400."""
|
|
response = client.get("/v1/models?backend=invalid_backend")
|
|
|
|
assert response.status_code == 400
|
|
|
|
|
|
# =============================================================================
|
|
# Completions Endpoint Tests (Real LLM Calls)
|
|
# =============================================================================
|
|
|
|
|
|
class TestCompletionsEndpoints:
|
|
"""Tests for chat completions endpoint with real LLM calls."""
|
|
|
|
@pytest.mark.slow
|
|
def test_chat_completion_basic(self, client: httpx.Client) -> None:
|
|
"""Test basic non-streaming chat completion."""
|
|
response = client.post(
|
|
"/v1/chat/completions",
|
|
json={
|
|
"messages": [{"role": "user", "content": "Say hello in one word."}],
|
|
"model": MODEL,
|
|
"max_tokens": 10,
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
# Verify OpenAI-compatible response structure
|
|
assert "id" in data
|
|
assert "choices" in data
|
|
assert "model" in data
|
|
assert data["object"] == "chat.completion"
|
|
|
|
# Verify we got a completion
|
|
assert len(data["choices"]) > 0
|
|
choice = data["choices"][0]
|
|
assert "message" in choice
|
|
assert choice["message"]["role"] == "assistant"
|
|
assert len(choice["message"]["content"]) > 0
|
|
assert choice["finish_reason"] in ["stop", "length"]
|
|
|
|
@pytest.mark.slow
|
|
def test_chat_completion_with_parameters(self, client: httpx.Client) -> None:
|
|
"""Test completion with optional parameters."""
|
|
response = client.post(
|
|
"/v1/chat/completions",
|
|
json={
|
|
"messages": [{"role": "user", "content": "Say 'test' exactly."}],
|
|
"model": MODEL,
|
|
"temperature": 0.0,
|
|
"max_tokens": 5,
|
|
"top_p": 1.0,
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
assert len(data["choices"]) > 0
|
|
assert len(data["choices"][0]["message"]["content"]) > 0
|
|
|
|
@pytest.mark.slow
|
|
@pytest.mark.asyncio
|
|
async def test_streaming_completion(self, async_client: httpx.AsyncClient) -> None:
|
|
"""Test streaming chat completion via SSE."""
|
|
async with async_client.stream(
|
|
"POST",
|
|
"/v1/chat/completions",
|
|
json={
|
|
"messages": [{"role": "user", "content": "Count from 1 to 3."}],
|
|
"model": MODEL,
|
|
"max_tokens": 20,
|
|
"stream": True,
|
|
},
|
|
) as response:
|
|
assert response.status_code == 200
|
|
assert "text/event-stream" in response.headers.get("content-type", "")
|
|
|
|
chunks = []
|
|
done_received = False
|
|
|
|
async for line in response.aiter_lines():
|
|
if line.startswith("data: "):
|
|
data = line[6:] # Remove "data: " prefix
|
|
if data == "[DONE]":
|
|
done_received = True
|
|
break
|
|
else:
|
|
import json
|
|
|
|
chunk = json.loads(data)
|
|
chunks.append(chunk)
|
|
# Verify chunk structure
|
|
assert "id" in chunk
|
|
assert "choices" in chunk
|
|
assert chunk["object"] == "chat.completion.chunk"
|
|
|
|
# Verify we got chunks and the [DONE] marker
|
|
assert len(chunks) > 0
|
|
assert done_received, "Stream should end with [DONE] marker"
|
|
|
|
@pytest.mark.slow
|
|
def test_multi_turn_conversation(self, client: httpx.Client) -> None:
|
|
"""Test completion with multi-turn conversation."""
|
|
response = client.post(
|
|
"/v1/chat/completions",
|
|
json={
|
|
"messages": [
|
|
{"role": "system", "content": "You are a helpful assistant."},
|
|
{"role": "user", "content": "My name is Alice."},
|
|
{"role": "assistant", "content": "Hello Alice!"},
|
|
{"role": "user", "content": "What is my name?"},
|
|
],
|
|
"model": MODEL,
|
|
"max_tokens": 20,
|
|
"temperature": 0.0,
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
# The response should reference "Alice"
|
|
content = data["choices"][0]["message"]["content"].lower()
|
|
assert "alice" in content
|
|
|
|
|
|
# =============================================================================
|
|
# Validation Error Tests
|
|
# =============================================================================
|
|
|
|
|
|
class TestValidationErrors:
|
|
"""Tests for request validation errors."""
|
|
|
|
def test_missing_messages_returns_422(self, client: httpx.Client) -> None:
|
|
"""Test that missing messages returns validation error."""
|
|
response = client.post(
|
|
"/v1/chat/completions",
|
|
json={"model": MODEL},
|
|
)
|
|
|
|
assert response.status_code == 422
|
|
|
|
def test_missing_model_returns_422(self, client: httpx.Client) -> None:
|
|
"""Test that missing model returns validation error."""
|
|
response = client.post(
|
|
"/v1/chat/completions",
|
|
json={"messages": [{"role": "user", "content": "Hello"}]},
|
|
)
|
|
|
|
assert response.status_code == 422
|
|
|
|
def test_empty_messages_returns_422(self, client: httpx.Client) -> None:
|
|
"""Test that empty messages list returns validation error."""
|
|
response = client.post(
|
|
"/v1/chat/completions",
|
|
json={"messages": [], "model": MODEL},
|
|
)
|
|
|
|
assert response.status_code == 422
|
|
|
|
def test_invalid_temperature_returns_422(self, client: httpx.Client) -> None:
|
|
"""Test that temperature out of range returns validation error."""
|
|
response = client.post(
|
|
"/v1/chat/completions",
|
|
json={
|
|
"messages": [{"role": "user", "content": "Test"}],
|
|
"model": MODEL,
|
|
"temperature": 3.0, # Max is 2.0
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 422
|
|
|
|
def test_invalid_max_tokens_returns_422(self, client: httpx.Client) -> None:
|
|
"""Test that max_tokens below 1 returns validation error."""
|
|
response = client.post(
|
|
"/v1/chat/completions",
|
|
json={
|
|
"messages": [{"role": "user", "content": "Test"}],
|
|
"model": MODEL,
|
|
"max_tokens": 0, # Min is 1
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 422
|
|
|
|
def test_empty_message_content_returns_422(self, client: httpx.Client) -> None:
|
|
"""Test that empty message content returns validation error."""
|
|
response = client.post(
|
|
"/v1/chat/completions",
|
|
json={
|
|
"messages": [{"role": "user", "content": ""}],
|
|
"model": MODEL,
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 422
|
|
|
|
def test_extra_fields_rejected(self, client: httpx.Client) -> None:
|
|
"""Test that extra fields in request are rejected."""
|
|
response = client.post(
|
|
"/v1/chat/completions",
|
|
json={
|
|
"messages": [{"role": "user", "content": "Test"}],
|
|
"model": MODEL,
|
|
"invalid_field": "should_fail",
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 422
|
|
|
|
|
|
# =============================================================================
|
|
# Model Management Tests
|
|
# =============================================================================
|
|
|
|
|
|
class TestModelManagement:
|
|
"""Tests for model load/unload endpoints."""
|
|
|
|
def test_load_model_unsupported_backend_returns_400(
|
|
self, client: httpx.Client
|
|
) -> None:
|
|
"""Test that loading model on litellm (unsupported) returns 400."""
|
|
response = client.post(
|
|
"/v1/models/load",
|
|
json={"model": "test-model", "backend": "litellm"},
|
|
)
|
|
|
|
# LiteLLM doesn't support model loading
|
|
assert response.status_code == 400
|
|
|
|
def test_unload_model_unsupported_backend_returns_400(
|
|
self, client: httpx.Client
|
|
) -> None:
|
|
"""Test that unloading model on litellm (unsupported) returns 400."""
|
|
response = client.post(
|
|
"/v1/models/unload",
|
|
json={"model": "test-model", "backend": "litellm"},
|
|
)
|
|
|
|
# LiteLLM doesn't support model unloading
|
|
assert response.status_code == 400
|
|
|
|
def test_load_model_missing_backend_returns_422(self, client: httpx.Client) -> None:
|
|
"""Test that load request without backend returns 422."""
|
|
response = client.post(
|
|
"/v1/models/load",
|
|
json={"model": "test-model"},
|
|
)
|
|
|
|
assert response.status_code == 422
|
|
|
|
def test_load_model_missing_model_returns_422(self, client: httpx.Client) -> None:
|
|
"""Test that load request without model returns 422."""
|
|
response = client.post(
|
|
"/v1/models/load",
|
|
json={"backend": "litellm"},
|
|
)
|
|
|
|
assert response.status_code == 422
|
|
|
|
|
|
# =============================================================================
|
|
# Middleware Tests
|
|
# =============================================================================
|
|
|
|
|
|
class TestMiddleware:
|
|
"""Tests for API middleware behavior."""
|
|
|
|
def test_request_id_header_added(self, client: httpx.Client) -> None:
|
|
"""Test that X-Request-ID header is added to responses."""
|
|
response = client.get("/health")
|
|
|
|
assert response.status_code == 200
|
|
assert "x-request-id" in response.headers
|
|
|
|
def test_request_id_preserved_when_provided(self, client: httpx.Client) -> None:
|
|
"""Test that provided X-Request-ID is preserved."""
|
|
custom_request_id = "e2e-test-request-id-12345"
|
|
response = client.get(
|
|
"/health",
|
|
headers={"X-Request-ID": custom_request_id},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
assert response.headers["x-request-id"] == custom_request_id
|