"""Real end-to-end integration tests for LLM inference API. These tests hit a real running server with real LLM API calls. No mocking - all requests go to actual endpoints. Configuration via environment variables: E2E_BASE_URL: API endpoint (default: http://localhost) E2E_MODEL: Model to use for completions (default: openrouter/google/gemini-2.0-flash-001) Usage: # Start the service first ./deploy/deploy.sh --profile api --detach # Run all E2E tests uv run pytest tests/test_e2e_real.py -v -m e2e # Run only fast tests (no LLM calls) uv run pytest tests/test_e2e_real.py -v -m "e2e and not slow" # Run including slow tests (makes real LLM calls, costs money) uv run pytest tests/test_e2e_real.py -v -m e2e """ import os import httpx import pytest # Configuration from environment BASE_URL = os.environ.get("E2E_BASE_URL", "http://localhost") MODEL = os.environ.get("E2E_MODEL", "openrouter/google/gemini-2.0-flash-001") TIMEOUT = 120.0 # LLM calls can be slow @pytest.fixture(scope="module") def client() -> httpx.Client: """Create an HTTP client for E2E tests.""" return httpx.Client(base_url=BASE_URL, timeout=TIMEOUT) @pytest.fixture(scope="module") def async_client() -> httpx.AsyncClient: """Create an async HTTP client for streaming tests.""" return httpx.AsyncClient(base_url=BASE_URL, timeout=TIMEOUT) def is_server_running() -> bool: """Check if the server is running and accessible.""" try: with httpx.Client(base_url=BASE_URL, timeout=5.0) as client: response = client.get("/health") return response.status_code == 200 except httpx.ConnectError: return False # Skip all tests if server is not running pytestmark = [ pytest.mark.e2e, pytest.mark.skipif( not is_server_running(), reason=f"Server not running at {BASE_URL}. Start with: ./deploy/deploy.sh --profile api --detach", ), ] # ============================================================================= # Health Endpoint Tests # ============================================================================= class TestHealthEndpoints: """Tests for health check endpoints.""" def test_health_returns_status(self, client: httpx.Client) -> None: """Test /health returns overall and backend health status.""" response = client.get("/health") assert response.status_code == 200 data = response.json() assert "status" in data assert "backends" in data assert data["status"] in ["healthy", "degraded", "unhealthy"] # Verify backends structure assert isinstance(data["backends"], list) for backend in data["backends"]: assert "name" in backend assert "healthy" in backend def test_ready_returns_readiness(self, client: httpx.Client) -> None: """Test /ready returns readiness status.""" response = client.get("/ready") assert response.status_code == 200 data = response.json() assert "ready" in data assert isinstance(data["ready"], bool) # ============================================================================= # Backends & Models Endpoint Tests # ============================================================================= class TestBackendsEndpoints: """Tests for backend and model management endpoints.""" def test_list_backends(self, client: httpx.Client) -> None: """Test GET /v1/backends returns available backends.""" response = client.get("/v1/backends") assert response.status_code == 200 data = response.json() assert "backends" in data assert isinstance(data["backends"], list) assert len(data["backends"]) > 0 # At minimum, litellm should be available assert "litellm" in data["backends"] def test_list_models(self, client: httpx.Client) -> None: """Test GET /v1/models returns available models.""" response = client.get("/v1/models") assert response.status_code == 200 data = response.json() assert data["object"] == "list" assert "data" in data assert isinstance(data["data"], list) def test_list_models_with_backend_filter(self, client: httpx.Client) -> None: """Test GET /v1/models?backend=litellm filters correctly.""" response = client.get("/v1/models?backend=litellm") assert response.status_code == 200 data = response.json() # All returned models should be from litellm for model in data["data"]: assert model["backend"] == "litellm" def test_list_models_invalid_backend_returns_400( self, client: httpx.Client ) -> None: """Test that invalid backend filter returns 400.""" response = client.get("/v1/models?backend=invalid_backend") assert response.status_code == 400 # ============================================================================= # Completions Endpoint Tests (Real LLM Calls) # ============================================================================= class TestCompletionsEndpoints: """Tests for chat completions endpoint with real LLM calls.""" @pytest.mark.slow def test_chat_completion_basic(self, client: httpx.Client) -> None: """Test basic non-streaming chat completion.""" response = client.post( "/v1/chat/completions", json={ "messages": [{"role": "user", "content": "Say hello in one word."}], "model": MODEL, "max_tokens": 10, }, ) assert response.status_code == 200 data = response.json() # Verify OpenAI-compatible response structure assert "id" in data assert "choices" in data assert "model" in data assert data["object"] == "chat.completion" # Verify we got a completion assert len(data["choices"]) > 0 choice = data["choices"][0] assert "message" in choice assert choice["message"]["role"] == "assistant" assert len(choice["message"]["content"]) > 0 assert choice["finish_reason"] in ["stop", "length"] @pytest.mark.slow def test_chat_completion_with_parameters(self, client: httpx.Client) -> None: """Test completion with optional parameters.""" response = client.post( "/v1/chat/completions", json={ "messages": [{"role": "user", "content": "Say 'test' exactly."}], "model": MODEL, "temperature": 0.0, "max_tokens": 5, "top_p": 1.0, }, ) assert response.status_code == 200 data = response.json() assert len(data["choices"]) > 0 assert len(data["choices"][0]["message"]["content"]) > 0 @pytest.mark.slow @pytest.mark.asyncio async def test_streaming_completion(self, async_client: httpx.AsyncClient) -> None: """Test streaming chat completion via SSE.""" async with async_client.stream( "POST", "/v1/chat/completions", json={ "messages": [{"role": "user", "content": "Count from 1 to 3."}], "model": MODEL, "max_tokens": 20, "stream": True, }, ) as response: assert response.status_code == 200 assert "text/event-stream" in response.headers.get("content-type", "") chunks = [] done_received = False async for line in response.aiter_lines(): if line.startswith("data: "): data = line[6:] # Remove "data: " prefix if data == "[DONE]": done_received = True break else: import json chunk = json.loads(data) chunks.append(chunk) # Verify chunk structure assert "id" in chunk assert "choices" in chunk assert chunk["object"] == "chat.completion.chunk" # Verify we got chunks and the [DONE] marker assert len(chunks) > 0 assert done_received, "Stream should end with [DONE] marker" @pytest.mark.slow def test_multi_turn_conversation(self, client: httpx.Client) -> None: """Test completion with multi-turn conversation.""" response = client.post( "/v1/chat/completions", json={ "messages": [ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "My name is Alice."}, {"role": "assistant", "content": "Hello Alice!"}, {"role": "user", "content": "What is my name?"}, ], "model": MODEL, "max_tokens": 20, "temperature": 0.0, }, ) assert response.status_code == 200 data = response.json() # The response should reference "Alice" content = data["choices"][0]["message"]["content"].lower() assert "alice" in content # ============================================================================= # Validation Error Tests # ============================================================================= class TestValidationErrors: """Tests for request validation errors.""" def test_missing_messages_returns_422(self, client: httpx.Client) -> None: """Test that missing messages returns validation error.""" response = client.post( "/v1/chat/completions", json={"model": MODEL}, ) assert response.status_code == 422 def test_missing_model_returns_422(self, client: httpx.Client) -> None: """Test that missing model returns validation error.""" response = client.post( "/v1/chat/completions", json={"messages": [{"role": "user", "content": "Hello"}]}, ) assert response.status_code == 422 def test_empty_messages_returns_422(self, client: httpx.Client) -> None: """Test that empty messages list returns validation error.""" response = client.post( "/v1/chat/completions", json={"messages": [], "model": MODEL}, ) assert response.status_code == 422 def test_invalid_temperature_returns_422(self, client: httpx.Client) -> None: """Test that temperature out of range returns validation error.""" response = client.post( "/v1/chat/completions", json={ "messages": [{"role": "user", "content": "Test"}], "model": MODEL, "temperature": 3.0, # Max is 2.0 }, ) assert response.status_code == 422 def test_invalid_max_tokens_returns_422(self, client: httpx.Client) -> None: """Test that max_tokens below 1 returns validation error.""" response = client.post( "/v1/chat/completions", json={ "messages": [{"role": "user", "content": "Test"}], "model": MODEL, "max_tokens": 0, # Min is 1 }, ) assert response.status_code == 422 def test_empty_message_content_returns_422(self, client: httpx.Client) -> None: """Test that empty message content returns validation error.""" response = client.post( "/v1/chat/completions", json={ "messages": [{"role": "user", "content": ""}], "model": MODEL, }, ) assert response.status_code == 422 def test_extra_fields_rejected(self, client: httpx.Client) -> None: """Test that extra fields in request are rejected.""" response = client.post( "/v1/chat/completions", json={ "messages": [{"role": "user", "content": "Test"}], "model": MODEL, "invalid_field": "should_fail", }, ) assert response.status_code == 422 # ============================================================================= # Model Management Tests # ============================================================================= class TestModelManagement: """Tests for model load/unload endpoints.""" def test_load_model_unsupported_backend_returns_400( self, client: httpx.Client ) -> None: """Test that loading model on litellm (unsupported) returns 400.""" response = client.post( "/v1/models/load", json={"model": "test-model", "backend": "litellm"}, ) # LiteLLM doesn't support model loading assert response.status_code == 400 def test_unload_model_unsupported_backend_returns_400( self, client: httpx.Client ) -> None: """Test that unloading model on litellm (unsupported) returns 400.""" response = client.post( "/v1/models/unload", json={"model": "test-model", "backend": "litellm"}, ) # LiteLLM doesn't support model unloading assert response.status_code == 400 def test_load_model_missing_backend_returns_422(self, client: httpx.Client) -> None: """Test that load request without backend returns 422.""" response = client.post( "/v1/models/load", json={"model": "test-model"}, ) assert response.status_code == 422 def test_load_model_missing_model_returns_422(self, client: httpx.Client) -> None: """Test that load request without model returns 422.""" response = client.post( "/v1/models/load", json={"backend": "litellm"}, ) assert response.status_code == 422 # ============================================================================= # Middleware Tests # ============================================================================= class TestMiddleware: """Tests for API middleware behavior.""" def test_request_id_header_added(self, client: httpx.Client) -> None: """Test that X-Request-ID header is added to responses.""" response = client.get("/health") assert response.status_code == 200 assert "x-request-id" in response.headers def test_request_id_preserved_when_provided(self, client: httpx.Client) -> None: """Test that provided X-Request-ID is preserved.""" custom_request_id = "e2e-test-request-id-12345" response = client.get( "/health", headers={"X-Request-ID": custom_request_id}, ) assert response.status_code == 200 assert response.headers["x-request-id"] == custom_request_id