litellm_settings:
  custom_provider_map:
    - provider: my-custom-provider
      custom_handler: my_handler.my_custom_llm

model_list:
  - model_name: my-model
    litellm_params:
      model: my-custom-provider/deployment-some-model
      api_base: https://example.com/api
      api_version: 2024-08-01-preview
      api_key: os.environ/MY_API_KEY

---

from typing import Iterator
from litellm import CustomLLM, CustomStreamWrapper
from litellm.types.utils import ModelResponse

def _sse_stream(url, headers, payload, timeout, model) -> Iterator[ModelResponse]:
    with httpx.Client(timeout=timeout) as client:
        with client.stream("POST", url, json=payload, headers=headers) as response:
            for line in response.iter_lines():
                # parse SSE chunks, yield ModelResponse
                yield parsed_chunk

class MyCustomLLM(CustomLLM):
    async def acompletion(self, ..., stream: bool = False, **extra):
        if stream:
            generator = _sse_stream(...)
            return CustomStreamWrapper(
                completion_stream=generator,
                model=ctx.deployment,
                custom_llm_provider="my-custom-provider",
                logging_obj=None,
            )
        # non-stream path works fine
        ...

---

from openai import OpenAI

client = OpenAI(api_key="sk-1234", base_url="http://localhost:4000/v1")

response = client.chat.completions.create(
    model="my-model",
    messages=[{"role": "user", "content": "hello"}],
    stream=True,
)

for chunk in response:
    print(chunk)

---

Error: 500 litellm.MidStreamFallbackError: litellm.APIConnectionError: 'coroutine' object is not an iterator
Traceback (most recent call last):
  File "/app/.venv/lib/python3.13/site-packages/litellm/litellm_core_utils/streaming_handler.py", line 2129, in __anext__
    chunk = await asyncio.to_thread(_next_sync_or_exhausted, self.completion_stream)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.13/asyncio/threads.py", line 25, in to_thread
    return await loop.run_in_executor(None, func_call)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.13/concurrent/futures/thread.py", line 59, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/app/.venv/lib/python3.13/site-packages/litellm/litellm_core_utils/streaming_handler.py", line 72, in _next_sync_or_exhausted
    return next(it)
TypeError: 'coroutine' object is not an iterator
. Received Model Group=my-model
Available Model Group Fallbacks=None

---