diff --git a/sandboxes/llm_local/app/mocks/README.md b/sandboxes/llm_local/app/mocks/README.md index 0e84016..e44d98d 100644 --- a/sandboxes/llm_local/app/mocks/README.md +++ b/sandboxes/llm_local/app/mocks/README.md @@ -12,6 +12,26 @@ app/mocks/ └── [future_service].py # Add new mocks here ``` +## The OpenAI Mock (`openai.py`) + +Section 4 below asks every mock to say which endpoints it serves and what the credentials are. +Here is that, for the one mock this sandbox ships. + +| Method | Path | Backend | Purpose | +|---|---|---|---| +| `GET` | `/v1/models` | none — answered from config | Liveness: what the sandbox is configured to serve | +| `POST` | `/v1/chat/completions` | Ollama | Chat completions, proxied | + +Both routes require `Authorization: Bearer sk-mock-key`. The key is not a secret: `make up` +prints it, the clients hardcode it, and the sandbox READMEs list it. + +`GET /v1/models` deliberately does not ask Ollama. A client calls it to find out whether the +sandbox is up, so a sandbox that is running but still pulling a model should not answer 500. +The cost of that choice is that the route can list a model the first completion then fails on; +`POST /v1/chat/completions` still returns backend failures as a 500. + +The model id comes from `OLLAMA_MODEL`, defaulting to `gpt-oss:20b`. + ## Why Mirascope? For this project, we have chosen **Mirascope** over other frameworks like LangChain for several key reasons that align with our goals of security, clarity, and developer experience. diff --git a/sandboxes/llm_local/app/mocks/openai.py b/sandboxes/llm_local/app/mocks/openai.py index 9d834c1..cb3cc30 100644 --- a/sandboxes/llm_local/app/mocks/openai.py +++ b/sandboxes/llm_local/app/mocks/openai.py @@ -20,12 +20,17 @@ router = APIRouter() -def verify_api_key(authorization: str = Header(...)) -> str: +def verify_api_key(authorization: Optional[str] = Header(default=None)) -> str: """Mock API key verification for testing purposes. In a real implementation, this would validate against a database or secret store. For testing purposes, we accept a simple mock key. + The header is declared optional so that a request sending none gets a 401 naming + the expected credential, instead of FastAPI's 422 about a missing required header. + The mock key is not a secret: `make up` prints it, both clients hardcode it, and + four READMEs list it. + Args: authorization: Authorization header value (e.g., "Bearer sk-mock-key"). @@ -33,8 +38,14 @@ def verify_api_key(authorization: str = Header(...)) -> str: str: The extracted API key token. Raises: - HTTPException: If authentication scheme is invalid or API key doesn't match. + HTTPException: If the header is absent, the scheme is not Bearer, or the key + does not match. """ + if authorization is None: + raise HTTPException( + status_code=401, + detail="Missing Authorization header", + ) if not authorization.startswith("Bearer "): raise HTTPException(status_code=401, detail="Invalid authentication scheme") token = authorization.split(" ")[1] @@ -70,6 +81,56 @@ class ChatCompletionRequest(BaseModel): ) +class Model(BaseModel): + """One entry of an OpenAI-compatible model list. + + Attributes: + id: Model name as the backend knows it (e.g. "gpt-oss:20b"). + object: Always "model", for OpenAI client compatibility. + created: Unix timestamp; the mock has no creation time, so 0. + owned_by: Owner string; "ollama" for this sandbox. + """ + + id: str + object: str = "model" + created: int = 0 + owned_by: str = "ollama" + + +class ModelList(BaseModel): + """Response model for the models endpoint. + + Attributes: + object: Always "list", for OpenAI client compatibility. + data: The available models. + """ + + object: str = "list" + data: List[Model] + + +@router.get("/v1/models") +def list_models(token: str = Depends(verify_api_key)) -> ModelList: + """Mock OpenAI models endpoint, answered from the sandbox's own configuration. + + Clients call this route to find out whether the sandbox is up before they send + anything. It therefore does NOT ask Ollama: a sandbox that is running but still + pulling a model would answer 500, which is the same false negative as the 404 + this endpoint replaces, one layer down. + + The trade-off: listing from config can advertise a model that the first completion + then fails on. POST /v1/chat/completions still surfaces backend failures as a 500, + so the caller learns about the backend from the request it wanted to make. + + Args: + token: Validated API key token from dependency injection. + + Returns: + ModelList: The model this sandbox is configured to serve. + """ + return ModelList(data=[Model(id=os.getenv("OLLAMA_MODEL", "gpt-oss:20b"))]) + + @router.post("/v1/chat/completions") def chat_completions( request: ChatCompletionRequest, token: str = Depends(verify_api_key) diff --git a/sandboxes/llm_local/tests/test_mock_routes.py b/sandboxes/llm_local/tests/test_mock_routes.py new file mode 100644 index 0000000..b0b1428 --- /dev/null +++ b/sandboxes/llm_local/tests/test_mock_routes.py @@ -0,0 +1,102 @@ +"""Route tests for the llm_local mock API. + +These run with no Ollama backend up: GET /v1/models answers from the sandbox's own +configuration, and every auth path is decided before any backend call is made. +""" + +import importlib +import os +from unittest import TestCase +from unittest.mock import patch + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from app.mocks import openai as mock_openai + + +def build_client(module=mock_openai) -> TestClient: + """A TestClient serving only the mock router, with no application around it.""" + app = FastAPI() + app.include_router(module.router) + return TestClient(app, raise_server_exceptions=False) + + +class ModelsEndpointTest(TestCase): + """GET /v1/models — the liveness route clients call before their first request.""" + + def test_returns_openai_model_list_shape(self): + response = build_client().get( + "/v1/models", headers={"Authorization": "Bearer sk-mock-key"} + ) + self.assertEqual(response.status_code, 200) + body = response.json() + self.assertEqual(body["object"], "list") + self.assertEqual(len(body["data"]), 1) + entry = body["data"][0] + self.assertEqual(entry["object"], "model") + self.assertEqual(entry["owned_by"], "ollama") + + def test_lists_the_configured_model(self): + with patch.dict(os.environ, {"OLLAMA_MODEL": "llama3.2:1b"}): + response = build_client().get( + "/v1/models", headers={"Authorization": "Bearer sk-mock-key"} + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["data"][0]["id"], "llama3.2:1b") + + def test_answers_without_a_backend(self): + """The route must not consult Ollama: a sandbox still pulling a model is up.""" + with patch.object( + mock_openai.client.chat.completions, + "create", + side_effect=AssertionError("the models route must not call the backend"), + ): + response = build_client().get( + "/v1/models", headers={"Authorization": "Bearer sk-mock-key"} + ) + self.assertEqual(response.status_code, 200) + + +class AuthTest(TestCase): + """The four ways a request can present (or fail to present) the mock key.""" + + def test_missing_header_gets_a_401_that_names_no_credential(self): + """A red-teaming tool reads a named credential in an error body as a leak. + + The detail used to be "Missing Authorization header, expected: Bearer sk-mock-key". + agent0 and tools like it flag that as secret leakage, which is a false positive + against a sandbox whose key is public by design. The 401 still answers the question + a client has, which is that the header is missing. + """ + response = build_client().get("/v1/models") + self.assertEqual(response.status_code, 401) + self.assertEqual(response.json()["detail"], "Missing Authorization header") + self.assertNotIn("sk-mock-key", response.text) + + def test_non_bearer_scheme_is_rejected(self): + response = build_client().get( + "/v1/models", headers={"Authorization": "Basic sk-mock-key"} + ) + self.assertEqual(response.status_code, 401) + self.assertEqual(response.json()["detail"], "Invalid authentication scheme") + + def test_wrong_key_is_rejected(self): + response = build_client().get( + "/v1/models", headers={"Authorization": "Bearer sk-wrong-key"} + ) + self.assertEqual(response.status_code, 401) + self.assertEqual(response.json()["detail"], "Invalid API key") + + def test_correct_key_is_accepted(self): + response = build_client().get( + "/v1/models", headers={"Authorization": "Bearer sk-mock-key"} + ) + self.assertEqual(response.status_code, 200) + + +class ModuleImportTest(TestCase): + """The module must import with no backend reachable and no API key set.""" + + def test_imports_without_ollama(self): + self.assertIsNotNone(importlib.reload(mock_openai).router)