litellm - 💡(How to fix) Fix [Bug]: Deployment-level TPM enforcement is per-pod, not cross-pod — effective limit becomes `tpm_limit × N_replica` [1 pull requests]

await self.router_cache.async_increment_cache(
    key=tpm_key, value=total_tokens, ttl=..., parent_otel_span=...,
)

---

Configured: tpm_limit = 1,500,000
Observed (40-minute sustained burst):
  - per-minute tokens started:  ~5.5M (~3.7x over limit)
  - per-minute tokens completed: ~3.1M  (counter visible to a single replica never reaches limit)
  - HTTP 429 count:               0  ← root cause
  - backend latency:              1.4s → 65s polynomial growth (queue saturation)
  - backend 504 timeouts:        31K+ (the only thing actually protecting the backend)

Per-replica math:  5.5M / 5 replicas = 1.1M per replica < 1.5M limit → all pass

---

# mock_backend.py
import asyncio, os, json
from fastapi import FastAPI, Request
import uvicorn

LATENCY_SEC = float(os.getenv("LATENCY_SEC", "0.5"))
app = FastAPI()

@app.post("/v1/embeddings")
async def embeddings(req: Request):
    body = await req.json()
    n_tokens = max(1, len(str(body.get("input", ""))) // 4)
    await asyncio.sleep(LATENCY_SEC)
    return {
        "object": "list",
        "data": [{"object": "embedding", "embedding": [0.1, 0.2, 0.3], "index": 0}],
        "model": body.get("model"),
        "usage": {"prompt_tokens": n_tokens, "total_tokens": n_tokens, "completion_tokens": 0},
    }

if __name__ == "__main__":
    uvicorn.run(app, host="0.0.0.0", port=28000, log_level="warning")

---

model_list:
  - model_name: test-model
    litellm_params:
      model: hosted_vllm/mock-embedding
      api_base: http://localhost:28000/v1
      api_key: dummy
      tpm: 500
      rpm: 600000

router_settings:
  routing_strategy: usage-based-routing-v2
  redis_host: localhost
  redis_port: 16379

general_settings:
  master_key: sk-1234

---

for PORT in 14000 14001 14002 14003 14004; do
  litellm --config config.yaml --port $PORT &
done

---

import asyncio, json, urllib.request, urllib.error, time
from collections import Counter

PORTS = [14000, 14001, 14002, 14003, 14004]
KEY = "sk-1234"
PAYLOAD = json.dumps({"model": "test-model", "input": "hello world " * 10}).encode()

def call(idx):
    port = PORTS[idx % 5]
    req = urllib.request.Request(f"http://localhost:{port}/v1/embeddings", method="POST", data=PAYLOAD,
        headers={"Authorization": f"Bearer {KEY}", "Content-Type": "application/json"})
    try:
        with urllib.request.urlopen(req, timeout=30) as r:
            return "200"
    except urllib.error.HTTPError as e:
        b = e.read().decode("utf-8", errors="replace")
        return "429_LayerB" if "No deployments available" in b else f"err_{e.code}"

async def main():
    loop = asyncio.get_event_loop()
    futs = []
    t0 = time.monotonic()
    idx = 0
    while time.monotonic() - t0 < 50:
        idx += 1
        futs.append(loop.run_in_executor(None, call, idx))
        await asyncio.sleep(0.67)
    results = [await f for f in futs]
    print(Counter(results))

asyncio.run(main())

---

T+5s:   redis = 120
T+25s:  redis = 540   ← exceeds limit of 500, but still passing
T+45s:  redis = 975   ← nearly 2x limit, still passing

---