vllm - 💡(How to fix) Fix [Bug]: Qwen3-VL-2B-Instruct Geo3K accuracy score lower than SGLang with deterministic sampling [1 pull requests]

OS: Ubuntu 22.04.5 LTS
GPU: 8 x NVIDIA H20-3e
NVIDIA driver: 580.105.08
CUDA runtime version: 12.9.86
PyTorch: 2.11.0+cu129
CUDA used to build PyTorch: 12.9
Python: 3.12.13
vLLM Version: 0.21.0
flashinfer-python: 0.6.8.post1
transformers: 5.8.1
triton: 3.6.0
flash-attn: 2.7.4.post1
transformer_engine_torch: 2.10.0

---

#!/usr/bin/env bash
set -euo pipefail

MODEL_PATH=/root/models/Qwen3-VL-2B-Instruct
HOST=127.0.0.1
GPU_IDS=(0 1 2 3 4 5 6 7)
WORKER_PORT_BASE=18100
ROUTER_PORT=18080
SEED=42

export CUDA_HOME=/usr/local/cuda-12.9
export CUDA_PATH="${CUDA_HOME}"
export PATH="/path/to/venv/bin:${CUDA_HOME}/bin:${PATH}"
export NO_PROXY="127.0.0.1,localhost,${HOST}"
export no_proxy="${NO_PROXY}"

WORKER_URLS=()
for i in "${!GPU_IDS[@]}"; do
  gpu="${GPU_IDS[$i]}"
  port=$((WORKER_PORT_BASE + i))
  WORKER_URLS+=("http://${HOST}:${port}")

  CUDA_VISIBLE_DEVICES="${gpu}" vllm serve "${MODEL_PATH}" \
    --host "${HOST}" \
    --port "${port}" \
    --trust-remote-code \
    --seed "${SEED}" \
    --gpu-memory-utilization 0.9 \
    --max-model-len 262144 \
    --generation-config vllm \
    > "vllm_worker_gpu${gpu}_port${port}.log" 2>&1 &
done

# Wait until all worker /health endpoints are ready here.

python -m vllm_router.launch_router \
  --host "${HOST}" \
  --port "${ROUTER_PORT}" \
  --worker-urls "${WORKER_URLS[@]}" \
  --policy round_robin \
  --request-timeout-secs 14400 \
  --log-level info \
  > vllm_router.log 2>&1 &

---

import base64
import io
import json
import asyncio
from pathlib import Path

import aiohttp
import pandas as pd
from PIL import Image

# I use the same math verifier as my RL eval pipeline.
from slime.rollout.rm_hub.math_utils import grade_answer_verl


MODEL = "/root/models/Qwen3-VL-2B-Instruct"
DATASET = "/root/datasets/geo3k_imgurl/test.parquet"
BASE_URL = "http://127.0.0.1:18080"
OUTPUT = "vllm_chat_geo3k_eval_full_8gpu_router.json"

MAX_TOKENS = 4096
TEMPERATURE = 0.0
TOP_P = 1.0
TOP_K = -1
SEED = 42
CONCURRENCY = 64


def decode_data_url(data_url: str) -> Image.Image:
    if data_url.startswith("data:"):
        _, encoded = data_url.split(",", 1)
    else:
        encoded = data_url
    return Image.open(io.BytesIO(base64.b64decode(encoded))).convert("RGB")


def image_to_png_data_url(image: Image.Image) -> str:
    buf = io.BytesIO()
    image.convert("RGB").save(buf, format="PNG")
    return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode("utf-8")


def as_list(value):
    if value is None:
        return []
    if hasattr(value, "tolist"):
        value = value.tolist()
    if isinstance(value, list):
        return value
    return [value]


def build_openai_messages(problem: str, image_urls: list[str]):
    parts = []
    chunks = problem.split("<image>")
    for i, chunk in enumerate(chunks):
        if chunk:
            parts.append({"type": "text", "text": chunk})
        if i < len(chunks) - 1:
            parts.append({
                "type": "image_url",
                "image_url": {"url": image_urls[i]},
            })
    return [{"role": "user", "content": parts}]


async def post_json(session, url, payload):
    async with session.post(url, json=payload, timeout=aiohttp.ClientTimeout(total=14400)) as resp:
        text = await resp.text()
        if resp.status >= 400:
            raise RuntimeError(f"{resp.status}: {text[:1000]}")
        return json.loads(text)


async def main():
    df = pd.read_parquet(DATASET)

    samples = []
    for i, row in df.iterrows():
        pil_images = [decode_data_url(x) for x in as_list(row["images"])]
        image_urls = [image_to_png_data_url(img) for img in pil_images]
        messages = build_openai_messages(str(row["problem"]), image_urls)
        samples.append({
            "index": int(i),
            "messages": messages,
            "label": str(row["answer"]),
        })

    sem = asyncio.Semaphore(CONCURRENCY)
    rows = []

    async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(limit=CONCURRENCY)) as session:
        async def one(sample):
            async with sem:
                payload = {
                    "model": MODEL,
                    "messages": sample["messages"],
                    "max_tokens": MAX_TOKENS,
                    "temperature": TEMPERATURE,
                    "top_p": TOP_P,
                    "top_k": TOP_K,
                    "seed": SEED,
                    "skip_special_tokens": False,
                    "spaces_between_special_tokens": False,
                }
                data = await post_json(session, f"{BASE_URL}/v1/chat/completions", payload)
                text = data["choices"][0]["message"]["content"] or ""
                reward = 1 if grade_answer_verl(text, sample["label"]) else 0
                return {
                    "index": sample["index"],
                    "reward": reward,
                    "label": sample["label"],
                    "response_preview": text[:1000],
                }

        tasks = [asyncio.create_task(one(s)) for s in samples]
        for t in asyncio.as_completed(tasks):
            rows.append(await t)

    rows.sort(key=lambda x: x["index"])
    score = sum(r["reward"] for r in rows) / len(rows)

    result = {
        "backend": "vllm_chat",
        "num_samples": len(rows),
        "score": score,
        "correct": sum(r["reward"] for r in rows),
        "sampling": {
            "temperature": TEMPERATURE,
            "top_p": TOP_P,
            "top_k": TOP_K,
            "max_tokens": MAX_TOKENS,
            "seed": SEED,
        },
        "rows": rows,
    }
    Path(OUTPUT).write_text(json.dumps(result, ensure_ascii=False, indent=2))
    print(json.dumps({k: result[k] for k in ["backend", "num_samples", "correct", "score", "sampling"]}, indent=2))


asyncio.run(main())

---

{
  "backend": "vllm_chat",
  "num_samples": 601,
  "correct": 116,
  "score": 0.1930116472545757,
  "sampling": {
    "temperature": 0.0,
    "top_p": 1.0,
    "top_k": -1,
    "max_tokens": 4096,
    "seed": 42
  }
}

---

CUDA_VISIBLE_DEVICES="${gpu}" python -m sglang.launch_server \
  --model-path /root/models/Qwen3-VL-2B-Instruct \
  --host 127.0.0.1 \
  --port "${port}" \
  --trust-remote-code \
  --random-seed 42 \
  --mem-fraction-static 0.6 \
  --context-length 262144 \
  --sampling-defaults openai \
  --cuda-graph-bs 1 2 4 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128 136 144 152 160 168 176 184 192 200 208 216 224 232 240 248 256

---

{
  "text": "<chat-templated prompt>",
  "image_data": ["data:image/png;base64,..."],
  "sampling_params": {
    "max_new_tokens": 4096,
    "temperature": 0.0,
    "top_p": 1.0,
    "top_k": -1,
    "sampling_seed": 42,
    "skip_special_tokens": false,
    "spaces_between_special_tokens": false
  },
  "return_logprob": false
}

---

{
  "backend": "sglang",
  "num_samples": 601,
  "correct": 173,
  "score": 0.2878535773710483
}

---

{
  "backend": "vllm_render_generate",
  "num_samples": 601,
  "correct": 109,
  "score": 0.18136439267886856
}