docker pull vllm/vllm-openai:v0.26.0

docker run --runtime nvidia --gpus all \ -v /data/feature/0lbg/models/hsmin92_internvl35-4b-awq-w4a16-g128:/models/internvl-awq \ -p 8080:8000 \ --ipc=host \ vllm/vllm-openai:v0.26.0 \ /models/internvl-awq \ --served-model-name internvl35-awq \ --dtype bfloat16 \ --max-model-len 8192 \ --max-num-seqs 8 \ --max-num-batched-tokens 4096 \ --limit-mm-per-prompt '{"image":2,"video":0}' \ --gpu-memory-utilization 0.52 \ --trust-remote-code \ --enable-auto-tool-choice \ --tool-call-parser hermes