ollama - 💡(How to fix) Fix `/api/generate` returns `response: ""` for one concurrent request when model is loading (`raw: true`) [1 pull requests]

#!/usr/bin/env python3
"""
Reproducer: concurrent /api/generate with raw=True against cold model.
Run multiple times — failure rate is ~1-in-8 on Apple Silicon.
"""
import asyncio
import ollama

MODEL = "granite4:micro"  # replace with any available model


async def evict() -> None:
    client = ollama.AsyncClient()
    await client.generate(model=MODEL, prompt="", keep_alive=0)
    print(f"Evicted {MODEL!r}.")


async def generate_one(client: ollama.AsyncClient, idx: int) -> tuple[int, str]:
    resp = await client.generate(
        model=MODEL,
        prompt=f"What is {idx}+{idx}? Answer briefly.",
        stream=False,
        raw=True,
        options={"num_predict": 100, "num_ctx": 2048},
    )
    return idx, resp.response


async def main() -> None:
    await evict()
    print("Firing 4 concurrent requests against cold model...")
    client = ollama.AsyncClient()
    results = await asyncio.gather(*[generate_one(client, i) for i in range(1, 5)])
    any_empty = False
    for idx, r in results:
        if not r:
            print(f"  Request {idx}: EMPTY  ← bug")
            any_empty = True
        else:
            print(f"  Request {idx}: OK — {r[:60]!r}")
    print("\nBug triggered." if any_empty else "\nAll OK (try again — intermittent).")


asyncio.run(main())

---

time=...  source=runner.go:895  msg=load  request="...Parallel:1...KvSize:2048..."
time=...  source=server.go:1385 msg="waiting for llama runner to start responding"
time=...  source=sched.go:561   msg="loaded runners" count=1
time=...  source=server.go:1385 msg="waiting for llama runner to start responding"   ← second wait, after runners loaded
[GIN] 200 |  680ms  | POST /api/generate   ← returned BEFORE model was generating
[GIN] 200 |  701ms  | POST /api/generate   ← also fast — one of these is the empty response
[GIN] 200 | 1519ms  | POST /api/generate
[GIN] 200 | 2344ms  | POST /api/generate

---

time=...  source=server.go:433  msg="starting runner"
time=...  source=sched.go:484   msg="system memory"  total="64.0 GiB"
time=...  source=sched.go:491   msg="gpu memory"  available="51.3 GiB"
time=...  source=server.go:532  msg="loading model"  "model layers"=41
time=...  source=runner.go:895  msg=load  request="...Parallel:1 BatchSize:512 FlashAttention:Enabled KvSize:2048..."
time=...  source=server.go:1385 msg="waiting for llama runner to start responding"
time=...  source=server.go:1428 msg="waiting for server to become available"  status="llm server loading model"
time=...  source=sched.go:561   msg="loaded runners" count=1
time=...  source=server.go:1385 msg="waiting for llama runner to start responding"
[GIN] 200 |    680ms | POST /api/generate
[GIN] 200 |    701ms | POST /api/generate
[GIN] 200 |  1,519ms | POST /api/generate
[GIN] 200 |  2,344ms | POST /api/generate