[multiproc_executor.py:932] File "/vllm/vllm/compilation/piecewise_backend.py", line 197, in compiled_graph_wrapper [multiproc_executor.py:932] graph_output = compiled_graph(*args) [multiproc_executor.py:932] ^^^^^^^^^^^^^^^^^^^^^ [multiproc_executor.py:932] File "/vllm/.venv/lib/python3.12/site-packages/torch/_inductor/standalone_compile.py", line 312, in call [multiproc_executor.py:932] return self.inner_fn(*args) [multiproc_executor.py:932] ^^^^^^^^^^^^^^^^^^^^ [multiproc_executor.py:932] File "/vllm/.venv/lib/python3.12/site-packages/torch/_dynamo/aot_compile_types.py", line 211, in call [multiproc_executor.py:932] return self.compiled_fn(*args, **kwargs) [multiproc_executor.py:932] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [multiproc_executor.py:932] File "/vllm/.venv/lib/python3.12/site-packages/torch/_functorch/_aot_autograd/aot_autograd_result.py", line 679, in forward [multiproc_executor.py:932] return compiled_fn(list(runtime_args)) [multiproc_executor.py:932] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [multiproc_executor.py:932] File "/vllm/.venv/lib/python3.12/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2311, in call [multiproc_executor.py:932] return self.compiled_fn(*args, **kwargs) [multiproc_executor.py:932] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [multiproc_executor.py:932] File "/vllm/.venv/lib/python3.12/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 582, in runtime_wrapper [multiproc_executor.py:932] all_outs = call_func_at_runtime_with_args( [multiproc_executor.py:932] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [multiproc_executor.py:932] File "/vllm/.venv/lib/python3.12/site-packages/torch/_functorch/_aot_autograd/utils.py", line 138, in call_func_at_runtime_with_args [multiproc_executor.py:932] out = normalize_as_list(f(args)) [multiproc_executor.py:932] ^^^^^^^ [multiproc_executor.py:932] File "/vllm/.venv/lib/python3.12/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 785, in wrapper [multiproc_executor.py:932] return compiled_fn(runtime_args) [multiproc_executor.py:932] ^^^^^^^^^^^^^^^^^^^^^^^^^ [multiproc_executor.py:932] File "/vllm/.venv/lib/python3.12/site-packages/torch/_inductor/output_code.py", line 682, in call [multiproc_executor.py:932] return self.current_callable(inputs) [multiproc_executor.py:932] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [multiproc_executor.py:932] File "/vllm/.venv/lib/python3.12/site-packages/torch/_inductor/utils.py", line 3444, in run [multiproc_executor.py:932] out = model(new_inputs) [multiproc_executor.py:932] ^^^^^^^^^^^^^^^^^ [multiproc_executor.py:932] File "/tmp/torchinductor_weizha/2s/c2sm7zcxi4ch574czlqwjjrbozol5vx2qx6wexgv4zxnlp6ysdsb.py", line 1085, in call [multiproc_executor.py:932] torch.ops.vllm.flashinfer_trtllm_fused_allreduce_norm.default(allreduce_in=buf0, residual=buf2, norm_out=buf1, quant_out=None, scale_out=None, rms_gamma=arg3_1, rms_eps=1e-06, pattern_code=1, world_size=2, launch_with_pdl=True, fp32_acc=True, max_token_num=8192) [multiproc_executor.py:932] File "/vllm/.venv/lib/python3.12/site-packages/torch/_ops.py", line 871, in call [multiproc_executor.py:932] return self._op(*args, **kwargs) [multiproc_executor.py:932] ^^^^^^^^^^^^^^^^^^^^^^^^^ [multiproc_executor.py:932] File "/vllm/.venv/lib/python3.12/site-packages/torch/_compile.py", line 54, in inner [multiproc_executor.py:932] return disable_fn(*args, **kwargs) [multiproc_executor.py:932] ^^^^^^^^^^^^^^^^^^^^^^^^^^^ [multiproc_executor.py:932] File "/vllm/.venv/lib/python3.12/site-packages/torch/_dynamo/eval_frame.py", line 1272, in _fn [multiproc_executor.py:932] return fn(*args, **kwargs) [multiproc_executor.py:932] ^^^^^^^^^^^^^^^^^^^ [multiproc_executor.py:932] File "/vllm/.venv/lib/python3.12/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 411, in torch_dispatch [multiproc_executor.py:932] res = func(*args, **kwargs) [multiproc_executor.py:932] ^^^^^^^^^^^^^^^^^^^^^ [multiproc_executor.py:932] File "/vllm/.venv/lib/python3.12/site-packages/torch/_ops.py", line 871, in call [multiproc_executor.py:932] return self._op(*args, **kwargs) [multiproc_executor.py:932] ^^^^^^^^^^^^^^^^^^^^^^^^^ [multiproc_executor.py:932] File "/vllm/vllm/compilation/passes/fusion/allreduce_rms_fusion.py", line 143, in call_trtllm_fused_allreduce_norm [multiproc_executor.py:932] assert workspace is not None, ( [multiproc_executor.py:932] ^^^^^^^^^^^^^^^^^^^^^ [multiproc_executor.py:932] AssertionError: Flashinfer workspace must be initialized when using flashinfer

Fix Plan

The fix involves initializing the Flashinfer workspace before using it. Here are the steps:

• Initialize the Flashinfer workspace by calling the init_flashinfer_workspace function before calling call_trtllm_fused_allreduce_norm.
• Ensure that the workspace variable is not None when calling call_trtllm_fused_allreduce_norm.

Example code:

import torch

# Initialize Flashinfer workspace
def init_flashinfer_workspace():
    # Initialize the workspace
    workspace = torch.ops.vllm.init_flashinfer_workspace()
    return workspace

# Call the function to initialize the workspace
workspace = init_flashinfer_workspace()

# Now you can call call_trtllm_fused_allreduce_norm
def call_trtllm_fused_allreduce_norm(workspace, allreduce_in, residual, norm_out, quant_out, scale_out, rms_gamma, rms_eps, pattern_code, world_size, launch_with_pdl, fp32_acc, max_token_num):
    torch.ops.vllm.flashinfer_trtllm_fused_allreduce_norm(
        allreduce_in=allreduce_in,
        residual=residual,
        norm_out=norm_out,
        quant_out=quant_out,
        scale_out=scale_out,
        rms_gamma=rms_gamma,
        rms_eps=rms_eps,
        pattern_code=pattern_code,
        world_size=world_size,
        launch_with_pdl=launch_with_pdl,
        fp32_acc=fp32_acc,
        max_token_num=max_token_num,
    )

# Example usage:
allreduce_in = torch.randn(1, 3, 224, 224)
residual = torch.randn(1, 3, 224, 224)
norm_out = torch.randn(1, 3, 224, 224)
quant_out = None
scale_out = None
rms_gamma = 0.1
rms_eps = 1e-6
pattern_code = 1
world_size = 2
launch_with_pdl = True
fp32_acc = True
max_token_num = 8192

call_trtllm_fused_allreduce_norm(
    workspace,
    allreduce_in,
    residual,
    norm_out,
    quant_out,
    scale_out,
    rms_gamma,
    rms_eps,
    pattern_code,
    world_size,
    launch_with_pdl,
    fp32_acc,
    max_token_num,
)

Verification

To verify that the fix worked, run the code and check that the AssertionError is no longer raised. You can also add print statements or use a debugger to verify that the workspace variable is not None when calling call_trtllm_fused_allreduce_norm.

Extra Tips

• Make sure to initialize the Flashinfer workspace only once, as initializing it multiple times can cause issues.
• If you are using a