vllm - 💡(How to fix) Fix [Bug] vLLM 0.17.1: `zai-org/GLM-OCR` has `mtp_graph < no_mtp_graph` despite high acceptance [1 participants]

Collecting environment information...
==============================
        System Info
==============================
OS                           : Ubuntu 22.04.5 LTS (x86_64)
GCC version                  : (Ubuntu 11.4.0-1ubuntu1~22.04.3) 11.4.0
Clang version                : Could not collect
CMake version                : Could not collect
Libc version                 : glibc-2.35

==============================
       PyTorch Info
==============================
PyTorch version              : 2.10.0+cu128
Is debug build               : False
CUDA used to build PyTorch   : 12.8
ROCM used to build PyTorch   : N/A

==============================
      Python Environment
==============================
Python version               : 3.11.14 (main, Dec 17 2025, 21:07:37) [Clang 21.1.4 ] (64-bit runtime)
Python platform              : Linux-6.8.0-1052-gcp-x86_64-with-glibc2.35

==============================
       CUDA / GPU Info
==============================
Is CUDA available            : True
CUDA runtime version         : 12.8.61
CUDA_MODULE_LOADING set to   :
GPU models and configuration :
GPU 0: NVIDIA A100-SXM4-40GB
GPU 1: NVIDIA A100-SXM4-40GB

Nvidia driver version        : 570.211.01

==============================
Versions of relevant libraries
==============================
[pip3] flashinfer-python==0.6.4
[pip3] torch==2.10.0
[pip3] transformers==5.3.0
[pip3] triton==3.6.0

==============================
         vLLM Info
==============================
vLLM Version                 : 0.17.1

---

import json
import os
import statistics
import time

from vllm import LLM, SamplingParams

os.environ.setdefault("VLLM_LOGGING_LEVEL", "DEBUG")
os.environ.setdefault("VLLM_LOG_STATS_INTERVAL", "5")

MODEL = "zai-org/GLM-OCR"

IMAGES = {
    "receipt": "https://ofasys-multimodal-wlcb-3-toshanghai.oss-accelerate.aliyuncs.com/wpf272043/keepme/image/receipt.png",
    "ocr_demo": "https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/paddleocr_vl_demo.png",
}

MODES = {
    "no_mtp_eager": {
        "enforce_eager": True,
    },
    "no_mtp_graph": {
        "enforce_eager": False,
    },
    "mtp_eager": {
        "enforce_eager": True,
        "speculative_config": {
            "method": "mtp",
            "num_speculative_tokens": 1,
            "enforce_eager": True,
        },
    },
    "mtp_graph": {
        "enforce_eager": False,
        "speculative_config": {
            "method": "mtp",
            "num_speculative_tokens": 1,
        },
    },
}

PROMPT = "Text Recognition:"
SP = SamplingParams(max_tokens=500, temperature=0.0, ignore_eos=True)
RUNS = 5
WARMUPS = 1


def run_mode(extra):
    llm = LLM(
        model=MODEL,
        tensor_parallel_size=1,
        dtype="bfloat16",
        max_model_len=8192,
        seed=0,
        disable_log_stats=False,
        **extra,
    )
    out = {}
    for image_name, image_url in IMAGES.items():
        msgs = [[
            {
                "role": "user",
                "content": [
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": PROMPT},
                ],
            }
        ]]

        for _ in range(WARMUPS):
            llm.chat(msgs, sampling_params=SP)

        tps = []
        for _ in range(RUNS):
            t0 = time.perf_counter()
            outputs = llm.chat(msgs, sampling_params=SP)
            dt = time.perf_counter() - t0
            ntok = sum(len(o.outputs[0].token_ids) for o in outputs)
            tps.append(ntok / dt)

        out[image_name] = {
            "avg": statistics.mean(tps),
            "std": statistics.pstdev(tps) if len(tps) > 1 else 0.0,
            "runs": tps,
        }
    return out


all_res = {}
for mode, cfg in MODES.items():
    print(f"Running mode={mode}")
    all_res[mode] = run_mode(cfg)

print("\n=== RESULTS (tok/s) ===")
for image in IMAGES:
    ne = all_res["no_mtp_eager"][image]["avg"]
    ng = all_res["no_mtp_graph"][image]["avg"]
    me = all_res["mtp_eager"][image]["avg"]
    mg = all_res["mtp_graph"][image]["avg"]
    print(
        f"{image}: no_mtp_eager={ne:.2f}, no_mtp_graph={ng:.2f}, "
        f"mtp_eager={me:.2f}, mtp_graph={mg:.2f}, "
        f"mtp_eager/no_mtp_eager={me/ne:.3f}x, "
        f"mtp_graph/no_mtp_graph={mg/ng:.3f}x"
    )

print("\nRAW_JSON")
print(json.dumps(all_res, indent=2))