vllm - 💡(How to fix) Fix [Bug]: DeepSeek V3.2 & V4 incorrect structured output when thinking enabled [1 comments, 1 participants]

vllm serve deepseek-ai/DeepSeek-V4-Pro\
  --served-model-name "deepseek-ai/DeepSeek-V4-Pro" \
  --trust-remote-code \
  --kv-cache-dtype fp8 \
  --block-size 256 \
  --enable-expert-parallel \
  --tensor-parallel-size 8 \
  --reasoning-parser deepseek_v4 \
  --tokenizer-mode deepseek_v4 \
  --tool-call-parser deepseek_v4 \
  --enable-auto-tool-choice \
  --model-loader-extra-config '{"enable_multithread_load": true, "num_threads": 12}'

---

from openai import OpenAI

client = OpenAI(base_url="http://localhost:8000/v1/", api_key="v1",)
so_messages = [
{
  "role": "user",
  "content": "Return ONLY a minified JSON object with EXACTLY these keys and constraints: {\"location\":string, \"temperature\":integer [-100..100], \"conditions\":\"sunny|cloudy|rainy|snowy\", \"unit\":\"celsius\", \"readings\":[{t:integer, ts:ISO-8601 UTC Z} x 3]}. No code fences, no extra text, no newlines. Example shape only; fill realistic values for Boston."
}
]
so_fail = {
  "max_tokens": 65536,
  "response_format": {
   "type": "json_object"
  },
  "chat_template_kwargs": {
    "enable_thinking": True
  },
}

response = client.chat.completions.create(
    model='deepseek-ai/DeepSeek-V4.0',
    messages=so_messages,
    extra_body=so_fail,
)
print(response)

---

ChatCompletion(
  id='chatcmpl-b890b17a7bb5881d',
  choices=[
    Choice(
      finish_reason='stop',
      index=0,
      logprobs=None,
      message=ChatCompletionMessage(
        content=None,
        refusal=None,
        role='assistant',
        annotations=None,
        audio=None, 
        function_call=None, tool_calls=[],
        reasoning='We{\n  "location": "Boston",\n  "temperature": 18,\n  "conditions": "cloudy",\n  "unit": "celsius",\n  "readings": [\n    {"t": 18, "ts": "2025-03-14T19:00:00Z"},\n    {"t": 17, "ts": "2025-03-14T20:00:00Z"},\n    {"t": 16, "ts": "2025-03-14T21:00:00Z"}\n  ]\n}'
        # ^^^^^^^^^^^^^^^^^^^^^^^^ THE PROBLEM ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        ),
      stop_reason=None,
      token_ids=None
    )
  ],
  created=1777373352,
  model='deepseek-ai/DeepSeek-V4-Pro',
  object='chat.completion',
  service_tier=None,
  system_fingerprint=None, 
  usage=CompletionUsage(completion_tokens=117, prompt_tokens=101, total_tokens=218, completion_tokens_details=None, prompt_tokens_details=None),
  prompt_logprobs=None, prompt_token_ids=None, kv_transfer_params=None
)