vllm - 💡(How to fix) Fix [Bug]: v0.20 latency and throughput regression on MoE models [1 comments, 2 participants]

# Start server
docker run -d --name vllm_test --gpus all --network host --shm-size=16g \
  vllm/vllm-openai:v0.19.0 \
  --model mistralai/Mixtral-8x7B-Instruct-v0.1 \
  --tensor-parallel-size 8 \
  --max-model-len 16384 \
  --max-num-seqs 16 \
  --dtype bfloat16 \
  --compilation-config '{"compile_sizes":[1,2,4,8],"cudagraph_mode":"FULL_AND_PIECEWISE"}'

# Run benchmark
vllm bench serve \
  --model mistralai/Mixtral-8x7B-Instruct-v0.1 \
  --backend vllm \
  --dataset-name sonnet \
  --dataset-path sonnet.txt \
  --sonnet-input-len 4096 \
  --sonnet-output-len 512 \
  --num-prompts 300 \
  --max-concurrency 1

# Stop and repeat with vllm/vllm-openai:v0.20.0

---

docker run -d --name vllm_test --gpus all --network host --shm-size=16g \
  vllm/vllm-openai:v0.19.0 \
  --model deepseek-ai/DeepSeek-V2-Chat \
  --tensor-parallel-size 8 \
  --max-model-len 16384 \
  --max-num-seqs 16 \
  --dtype bfloat16 \
  --gpu-memory-utilization 0.85 \
  --compilation-config '{"compile_sizes":[1,2,4,8],"cudagraph_mode":"FULL_AND_PIECEWISE"}'

# Same benchmark command with --model deepseek-ai/DeepSeek-V2-Chat

---

docker run -d --name vllm_test --gpus all --network host --shm-size=16g \
  vllm/vllm-openai:v0.19.0 \
  --model meta-llama/Llama-3.1-70B-Instruct \
  --tensor-parallel-size 8 \
  --max-model-len 16384 \
  --max-num-seqs 16 \
  --dtype bfloat16 \
  --compilation-config '{"compile_sizes":[1,2,4,8],"cudagraph_mode":"FULL_AND_PIECEWISE"}'

# Same benchmark command with --model meta-llama/Llama-3.1-70B-Instruct