Skip to content

09 HTTP router

Mount the queue's health, status and registry routes on a FastAPI app. Use it for the admin API, Kubernetes probes and the preStop hook.

flowchart LR
  k8s["Kubernetes probes"] -- "no auth" --> h["/llm-queue/health/live<br/>/llm-queue/health/ready"]
  admin["Admin user"] -- "app's admin dependency" --> r["/llm-queue/status<br/>/llm-queue/requests/{id}<br/>/llm-queue/registry"]
  pre["preStop hook"] --> d["/drain → pause_receiving()"]
  d -. "ready → 503" .-> h

Key points

  • llmq.http_router() returns an APIRouter (install the http extra).
  • /health/live and /health/ready take no auth, so the Kubernetes probes can reach them. Every other route runs the app's admin dependencies.
  • Writes (live registry edits, region and account blocks) stay off unless allow_writes=True. Each write is audited under the actor you return.
  • A preStop hook calls pause_receiving(). Readiness then fails and no new queued work starts, while running calls finish. /drain is the app's own route (the router has none) and answers GET, which is what Kubernetes' preStop: httpGet sends; a POST-only route answers it 405 and nothing drains.
  • The sample calls the app in process with httpx's ASGI transport, so no port is opened. In an app, uvicorn serves the same router.

Code

samples/09_http_router.py
"""Mount the queue's status, registry and health routes on a FastAPI app (admin pages, probes).

`llmq.http_router()` returns an `APIRouter` (extra `http`). `/health/live` and `/health/ready`
take no auth (Kubernetes probes); every other route runs the app's admin dependencies. Writes
(live registry edits, region / account blocks) stay off unless `allow_writes=True`; they take a
JSON body and need the `X-LLMQ-Admin` header (a browser cannot send it cross-site). A preStop
hook calls `llmq.pause_receiving()`: readiness fails and no new queued work starts.

Here the app is called in process (httpx's ASGI transport), so no port is opened; in the app
the same router is served by uvicorn.

    python scripts/samples.py 09_http_router [--live]
"""

from __future__ import annotations

import asyncio
import contextlib
from collections.abc import AsyncIterator

import httpx
from fastapi import FastAPI, Header, HTTPException

from llm_queue import LLMQueue

llmq = LLMQueue.from_env()


def admin_only(x_admin_token: str = Header(default="")) -> None:
    """The app's own admin check (here a fixed header; use the app's auth)."""
    if x_admin_token != "sample-admin":
        raise HTTPException(status_code=401, detail="admin only")


@contextlib.asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
    await llmq.start()
    yield
    await llmq.stop()


app = FastAPI(lifespan=lifespan)
app.include_router(
    llmq.http_router(
        dependencies=[admin_only],
        allow_writes=True,
        actor=lambda request: "sample-admin",  # the authenticated user's name in an app
    ),
    prefix="/llm-queue",
)


# GET for Kubernetes' `preStop: httpGet` (a POST-only route answered it 405 and nothing
# drained); POST for scripts. The route is the app's own: the router has no `/drain`.
@app.api_route("/drain", methods=["GET", "POST"])
async def drain() -> dict[str, bool]:
    """PreStop hook: fail readiness and take no new queued calls; running ones finish."""
    llmq.pause_receiving()
    return {"draining": True}


async def main() -> None:
    admin = {"x-admin-token": "sample-admin"}
    writer = {**admin, "X-LLMQ-Admin": "1"}  # write routes need this header too
    transport = httpx.ASGITransport(app=app)
    async with (
        app.router.lifespan_context(app),
        httpx.AsyncClient(transport=transport, base_url="http://app") as client,
    ):
        ready = await client.get("/llm-queue/health/ready")
        print(f"GET /health/ready -> {ready.status_code} {ready.json()}")
        assert ready.status_code == 200

        denied = await client.get("/llm-queue/status")
        print(f"GET /status without the admin header -> {denied.status_code}")
        assert denied.status_code == 401

        reply = await llmq.converse(
            "gpt20b",
            {"messages": [{"role": "user", "content": [{"text": "Say hi."}]}],
             "inferenceConfig": {"maxTokens": 200}},
        )  # fmt: skip
        found = await client.get(f"/llm-queue/requests/{reply.request_id}", headers=admin)
        print(f"GET /requests/{{id}} -> {found.json()['state']}")
        assert found.json()["state"] == "done"

        status = (await client.get("/llm-queue/status?model=gpt20b", headers=admin)).json()
        for dep in status["deployments"]:
            print(f"  {dep['name']}: rpm {dep['rpm']['used']}/{dep['rpm']['limit']}")

        no_header = await client.post(
            "/llm-queue/registry", params={"model": "gpt20b"}, json={"rpm": 50}, headers=admin
        )
        print(f"POST /registry without X-LLMQ-Admin -> {no_header.status_code}")
        assert no_header.status_code == 403

        edit = await client.post(
            "/llm-queue/registry",
            params={"model": "gpt20b"},
            json={"rpm": 50},
            headers=writer,
        )
        print(f"POST /registry rpm=50 -> {edit.json()['diff']}")
        assert edit.status_code == 200
        await client.post(
            "/llm-queue/registry",
            params={"model": "gpt20b"},
            json={"rpm": 60},
            headers=writer,
        )  # put the sample value back

        region = status["deployments"][0]["name"].split("/")[-1]
        for on in (True, False):  # an emergency block of a region, then lifted
            block = await client.post(
                "/llm-queue/registry/block", json={"region": region, "on": on}, headers=writer
            )
            print(f"POST /registry/block {region} on={on} -> {block.status_code}")
            assert block.status_code == 200

        await client.get("/drain")  # what the preStop httpGet sends
        after = await client.get("/llm-queue/health/ready")
        print(f"after /drain: GET /health/ready -> {after.status_code}")
        assert after.status_code == 503


if __name__ == "__main__":
    asyncio.run(main())

Output (live, 2026-11-14)

GET /health/ready -> 200 {'live': True, 'ready': True, 'stores': {'state': 'ok', 'queue': 'ok'}}
GET /status without the admin header -> 401
GET /requests/{id} -> done
  aws-main/us-east-1: rpm 2/60
POST /registry rpm=50 -> [{'path': 'models.gpt20b.rpm', 'before': 60, 'after': 50}]
after /drain: GET /health/ready -> 503

Since this run the health body also carries eviction_policy (the state store's maxmemory-policy, unknown when unreadable) and, with a separate queue store, queue_eviction_policy; /drain is called with GET.

See also: CLI and HTTP, Operate.