vllm - 💡(How to fix) Fix [Bug]: Qwen3 + DeepGEMM + dummy-load Cannot access data pointer of Tensor that doesn't have storage [2 comments, 3 participants]

cat > /tmp/qwen3_async_tp_dump.py <<'PY'
from vllm import LLM, SamplingParams
from vllm.config import CompilationConfig, CompilationMode, CUDAGraphMode, PassConfig

def main():
    compilation_config = CompilationConfig(
        cudagraph_mode=CUDAGraphMode.NONE,
        mode=CompilationMode.VLLM_COMPILE,
        use_inductor_graph_partition=False,
        inductor_compile_config={"force_disable_caches": True},
        custom_ops=["+quant_fp8", "+rms_norm"],
        pass_config=PassConfig(
            fuse_norm_quant=True,
            fuse_act_quant=True,
            fuse_attn_quant=True,
            enable_qk_norm_rope_fusion=True,
            enable_sp=True,
            fuse_gemm_comms=True,
            fuse_allreduce_rms=False,
            sp_min_token_num=512,
        ),
    )

    llm = LLM(
        model="Qwen/Qwen3-30B-A3B-FP8",
        tensor_parallel_size=2,
        kv_cache_dtype="fp8",
        attention_config={"backend": "FLASHINFER"},
        compilation_config=compilation_config,
        kernel_config={"enable_flashinfer_autotune": False},
        disable_custom_all_reduce=True,
        load_format="dummy",
        hf_overrides={"num_hidden_layers": 4},
        max_model_len=1024,
    )

    outputs = llm.generate(
        [
            "Hello, my name is",
            "The capital of France is",
            "The future of AI is",
            "One short test prompt",
        ],
        SamplingParams(temperature=0),
    )

    for out in outputs:
        print(out.outputs[0].text)

if __name__ == "__main__":
    main()
PY

---

rm -rf /tmp/vllm_dump_async_tp_qwen3
rm -rf /tmp/torchinductor_"$USER"

CUDA_VISIBLE_DEVICES=4,5 \
VLLM_DEBUG_DUMP_PATH=/tmp/vllm_dump_async_tp_qwen3 \
VLLM_DISABLE_COMPILE_CACHE=1 \
VLLM_PATTERN_MATCH_DEBUG=1 \
VLLM_LOGGING_LEVEL=DEBUG \
VLLM_WORKER_MULTIPROC_METHOD=spawn \
python /tmp/qwen3_async_tp_dump.py 2>&1 | tee /tmp/qwen3_async_tp.log