litellm - 💡(How to fix) Fix [Bug]: Anthropic→Responses streaming adapter never extracts cache_read_input_tokens (always 0) [1 pull requests]

# Prefer direct cache fields if present
cache_creation_tokens = int(
    getattr(usage, "cache_creation_input_tokens", 0) or 0
)
cache_read_tokens = int(
    getattr(usage, "cache_read_input_tokens", 0) or 0
)

---

cached_tokens = (
    getattr(usage.prompt_tokens_details, "cached_tokens", 0) or 0
)

---

import asyncio, os
import openai

# Pre-warm OpenAI's cache with a known body so subsequent calls have something to hit.
WARMUP_INSTRUCTIONS = (
    "You are a careful, methodical engineer. Respond with short factual statements. "
) * 50  # ~1.2K tokens — clears the 1024-token minimum for implicit caching.

async def main():
    direct = openai.AsyncOpenAI()
    # Warm cache, then verify the openai SDK can read it back.
    for i in range(3):
        raw = await direct.responses.with_raw_response.create(
            model="gpt-4o-mini",
            instructions=WARMUP_INSTRUCTIONS,
            input="hi",
            max_output_tokens=20,
        )
        resp = raw.parse()
        cached = (resp.usage.input_tokens_details.cached_tokens
                  if resp.usage and resp.usage.input_tokens_details else 0)
        print(f"direct call {i+1}: input={resp.usage.input_tokens} cached={cached}")

    # Now do the same body through LiteLLM's /v1/messages → /v1/responses bridge.
    # (Spin up `litellm --config <yaml>` with an openai-* model declared
    # ``mode: "responses"`` and ANTHROPIC_BASE_URL pointed at it.)
    from anthropic import AsyncAnthropic
    client = AsyncAnthropic(
        base_url=os.environ["LITELLM_PROXY_URL"],
        api_key=os.environ["LITELLM_MASTER_KEY"],
    )
    for i in range(3):
        msg = await client.messages.create(
            model="openai-gpt-4o-mini",
            system=WARMUP_INSTRUCTIONS,
            messages=[{"role": "user", "content": "hi"}],
            max_tokens=20,
        )
        # cache_read_input_tokens is always 0 here — even when the underlying
        # OpenAI call reports cached_tokens > 0 (verifiable via wire capture).
        print(f"litellm call {i+1}: input={msg.usage.input_tokens} cache_read={msg.usage.cache_read_input_tokens}")

asyncio.run(main())

---

direct call 1: input=1234 cached=0       # cold
direct call 2: input=1234 cached=1024    # cache hit
direct call 3: input=1234 cached=1024
litellm call 1: input=1234 cache_read=0
litellm call 2: input=1234 cache_read=1024
litellm call 3: input=1234 cache_read=1024

---

direct call 1: input=1234 cached=0
direct call 2: input=1234 cached=1024
direct call 3: input=1234 cached=1024
litellm call 1: input=1234 cache_read=0
litellm call 2: input=1234 cache_read=0   # ← bug
litellm call 3: input=1234 cache_read=0   # ← bug

---

if usage is not None:
    input_tokens = getattr(usage, "input_tokens", 0) or 0
    output_tokens = getattr(usage, "output_tokens", 0) or 0
    # OpenAI Responses API exposes the cache count at:
    #   usage.input_tokens_details.cached_tokens
    # NOT at usage.cache_read_input_tokens (Anthropic-only name).
    details = getattr(usage, "input_tokens_details", None)
    cache_read_tokens = 0
    if details is not None:
        cache_read_tokens = int(
            getattr(details, "cached_tokens", None)
            or (details.get("cached_tokens") if isinstance(details, dict) else 0)
            or 0
        )
    # Cache-creation isn't a Responses-API concept; keep it 0 for now
    # (Anthropic-side `cache_creation_input_tokens` has no OpenAI equivalent).
    cache_creation_tokens = 0