09 HTTP router¶
Mount the queue's health, status and registry routes on a FastAPI app. Use it for the admin API, Kubernetes probes and the preStop hook.
flowchart LR
k8s["Kubernetes probes"] -- "no auth" --> h["/llm-queue/health/live<br/>/llm-queue/health/ready"]
admin["Admin user"] -- "app's admin dependency" --> r["/llm-queue/status<br/>/llm-queue/requests/{id}<br/>/llm-queue/registry"]
pre["preStop hook"] --> d["/drain → pause_receiving()"]
d -. "ready → 503" .-> h
Key points¶
llmq.http_router()returns anAPIRouter(install thehttpextra)./health/liveand/health/readytake no auth, so the Kubernetes probes can reach them. Every other route runs the app's admin dependencies.- Writes (live registry edits, region and account blocks) stay off unless
allow_writes=True. Each write is audited under theactoryou return. - A preStop hook calls
pause_receiving(). Readiness then fails and no new queued work starts, while running calls finish./drainis the app's own route (the router has none) and answers GET, which is what Kubernetes'preStop: httpGetsends; a POST-only route answers it 405 and nothing drains. - The sample calls the app in process with httpx's ASGI transport, so no port is opened. In an app, uvicorn serves the same router.
Code¶
samples/09_http_router.py
"""Mount the queue's status, registry and health routes on a FastAPI app (admin pages, probes).
`llmq.http_router()` returns an `APIRouter` (extra `http`). `/health/live` and `/health/ready`
take no auth (Kubernetes probes); every other route runs the app's admin dependencies. Writes
(live registry edits, region / account blocks) stay off unless `allow_writes=True`; they take a
JSON body and need the `X-LLMQ-Admin` header (a browser cannot send it cross-site). A preStop
hook calls `llmq.pause_receiving()`: readiness fails and no new queued work starts.
Here the app is called in process (httpx's ASGI transport), so no port is opened; in the app
the same router is served by uvicorn.
python scripts/samples.py 09_http_router [--live]
"""
from __future__ import annotations
import asyncio
import contextlib
from collections.abc import AsyncIterator
import httpx
from fastapi import FastAPI, Header, HTTPException
from llm_queue import LLMQueue
llmq = LLMQueue.from_env()
def admin_only(x_admin_token: str = Header(default="")) -> None:
"""The app's own admin check (here a fixed header; use the app's auth)."""
if x_admin_token != "sample-admin":
raise HTTPException(status_code=401, detail="admin only")
@contextlib.asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
await llmq.start()
yield
await llmq.stop()
app = FastAPI(lifespan=lifespan)
app.include_router(
llmq.http_router(
dependencies=[admin_only],
allow_writes=True,
actor=lambda request: "sample-admin", # the authenticated user's name in an app
),
prefix="/llm-queue",
)
# GET for Kubernetes' `preStop: httpGet` (a POST-only route answered it 405 and nothing
# drained); POST for scripts. The route is the app's own: the router has no `/drain`.
@app.api_route("/drain", methods=["GET", "POST"])
async def drain() -> dict[str, bool]:
"""PreStop hook: fail readiness and take no new queued calls; running ones finish."""
llmq.pause_receiving()
return {"draining": True}
async def main() -> None:
admin = {"x-admin-token": "sample-admin"}
writer = {**admin, "X-LLMQ-Admin": "1"} # write routes need this header too
transport = httpx.ASGITransport(app=app)
async with (
app.router.lifespan_context(app),
httpx.AsyncClient(transport=transport, base_url="http://app") as client,
):
ready = await client.get("/llm-queue/health/ready")
print(f"GET /health/ready -> {ready.status_code} {ready.json()}")
assert ready.status_code == 200
denied = await client.get("/llm-queue/status")
print(f"GET /status without the admin header -> {denied.status_code}")
assert denied.status_code == 401
reply = await llmq.converse(
"gpt20b",
{"messages": [{"role": "user", "content": [{"text": "Say hi."}]}],
"inferenceConfig": {"maxTokens": 200}},
) # fmt: skip
found = await client.get(f"/llm-queue/requests/{reply.request_id}", headers=admin)
print(f"GET /requests/{{id}} -> {found.json()['state']}")
assert found.json()["state"] == "done"
status = (await client.get("/llm-queue/status?model=gpt20b", headers=admin)).json()
for dep in status["deployments"]:
print(f" {dep['name']}: rpm {dep['rpm']['used']}/{dep['rpm']['limit']}")
no_header = await client.post(
"/llm-queue/registry", params={"model": "gpt20b"}, json={"rpm": 50}, headers=admin
)
print(f"POST /registry without X-LLMQ-Admin -> {no_header.status_code}")
assert no_header.status_code == 403
edit = await client.post(
"/llm-queue/registry",
params={"model": "gpt20b"},
json={"rpm": 50},
headers=writer,
)
print(f"POST /registry rpm=50 -> {edit.json()['diff']}")
assert edit.status_code == 200
await client.post(
"/llm-queue/registry",
params={"model": "gpt20b"},
json={"rpm": 60},
headers=writer,
) # put the sample value back
region = status["deployments"][0]["name"].split("/")[-1]
for on in (True, False): # an emergency block of a region, then lifted
block = await client.post(
"/llm-queue/registry/block", json={"region": region, "on": on}, headers=writer
)
print(f"POST /registry/block {region} on={on} -> {block.status_code}")
assert block.status_code == 200
await client.get("/drain") # what the preStop httpGet sends
after = await client.get("/llm-queue/health/ready")
print(f"after /drain: GET /health/ready -> {after.status_code}")
assert after.status_code == 503
if __name__ == "__main__":
asyncio.run(main())
Output (live, 2026-11-14)¶
GET /health/ready -> 200 {'live': True, 'ready': True, 'stores': {'state': 'ok', 'queue': 'ok'}}
GET /status without the admin header -> 401
GET /requests/{id} -> done
aws-main/us-east-1: rpm 2/60
POST /registry rpm=50 -> [{'path': 'models.gpt20b.rpm', 'before': 60, 'after': 50}]
after /drain: GET /health/ready -> 503
Since this run the health body also carries eviction_policy (the state store's
maxmemory-policy, unknown when unreadable) and, with a separate queue store,
queue_eviction_policy; /drain is called with GET.
See also: CLI and HTTP, Operate.