{ "hf_id": "Google/gemma-4-31B-it", "meta": { "title": "Gemma 4 31B IT", "slug": "gemma-4-31b-it", "provider": "Google", "description": "Google's unified multimodal Gemma 4 dense model (31B) with native text, image, and audio, plus thinking mode and tool-use protocol.", "date_added": "2026-04-02", "date_updated": "2026-09-09", "tasks": [ "multimodal", "text" ], "performance_headline": "Unified multimodal model with structured thinking, function calling, dynamic vision resolution", "related_recipes": [], "hardware": { "h100": "verified", "mi300x": "verified", "mi325x": "verified", "mi355x": "verified", "trillium": "verified", "ironwood": "verified", "arc_pro_b60": "verified", "arc_pro_b70": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": {}, "docker_image": "vllm/vllm-openai:latest", "command": "vllm serve google/gemma-4-31B-it \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser gemma4 \\\n --chat-template examples/tool_chat_template_gemma4.jinja \\\n --reasoning-parser gemma4", "argv": [ "vllm", "serve", "google/gemma-4-31B-it", "--tensor-parallel-size", "1", "--enable-auto-tool-choice", "--tool-call-parser", "gemma4", "--chat-template", "examples/tool_chat_template_gemma4.jinja", "--reasoning-parser", "gemma4" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:latest google/gemma-4-31B-it \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser gemma4 \\\n --chat-template examples/tool_chat_template_gemma4.jinja \\\n --reasoning-parser gemma4", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "vllm/vllm-openai:latest", "google/gemma-4-31B-it", "--tensor-parallel-size", "1", "--enable-auto-tool-choice", "--tool-call-parser", "gemma4", "--chat-template", "examples/tool_chat_template_gemma4.jinja", "--reasoning-parser", "gemma4" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "multi_node_tp": "/Google/gemma-4-31B-it/strategies/multi_node_tp.json", "kv_store_centralized_mooncake": "/Google/gemma-4-31B-it/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/Google/gemma-4-31B-it/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/Google/gemma-4-31B-it/hw/h200.json", "h100": "/Google/gemma-4-31B-it/hw/h100.json", "b200": "/Google/gemma-4-31B-it/hw/b200.json", "gb200": "/Google/gemma-4-31B-it/hw/gb200.json", "b300": "/Google/gemma-4-31B-it/hw/b300.json", "gb300": "/Google/gemma-4-31B-it/hw/gb300.json", "mi300x": "/Google/gemma-4-31B-it/hw/mi300x.json", "mi325x": "/Google/gemma-4-31B-it/hw/mi325x.json", "mi350x": "/Google/gemma-4-31B-it/hw/mi350x.json", "mi355x": "/Google/gemma-4-31B-it/hw/mi355x.json", "trillium": "/Google/gemma-4-31B-it/hw/trillium.json", "ironwood": "/Google/gemma-4-31B-it/hw/ironwood.json", "arc_pro_b60": "/Google/gemma-4-31B-it/hw/arc_pro_b60.json", "arc_pro_b70": "/Google/gemma-4-31B-it/hw/arc_pro_b70.json" } }, "model": { "model_id": "google/gemma-4-31B-it", "min_vllm_version": "0.19.1", "architecture": "dense", "parameter_count": "31B", "active_parameters": "31B", "context_length": 262144, "base_args": [], "base_env": {}, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:latest" }, "extras": [ { "note": "Audio extras — only needed when serving the audio modality", "command": "uv pip install \"vllm[audio]\"", "optional": true } ] } }, "features": { "tool_calling": { "description": "Enable automatic tool choice with Gemma 4 parser and chat template", "args": [ "--enable-auto-tool-choice", "--tool-call-parser", "gemma4", "--chat-template", "examples/tool_chat_template_gemma4.jinja" ] }, "reasoning": { "description": "Enable structured thinking/reasoning output", "args": [ "--reasoning-parser", "gemma4" ] }, "text_only": { "description": "Skip loading the vision encoder for text-only workloads — frees VRAM for KV cache. Mutually exclusive with encoder_parallel.", "args": [ "--language-model-only" ] }, "spec_decoding": { "description": "MTP speculative decoding for accelerated inference", "default_mode": "mtp", "modes": { "mtp": { "label": "MTP", "description": "Built-in Multi-Token Prediction, 4 draft tokens.", "args": [ "--speculative-config", "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" ] }, "dflash": { "label": "DFlash", "description": "DFlash speculative decoding with an externally trained draft model; the served checkpoint is unchanged.", "variants": [ "default" ], "args": [ "--speculative-config", "{\"model\": \"RedHatAI/gemma-4-31B-it-speculator.dflash\", \"num_speculative_tokens\": 7, \"method\": \"dflash\"}" ] }, "dspark": { "label": "DSpark", "description": "DSpark drafting with an externally trained draft model; the served checkpoint is unchanged.", "variants": [ "default" ], "args": [ "--speculative-config", "{\"model\": \"RedHatAI/gemma-4-31B-it-speculator.dspark\", \"num_speculative_tokens\": 7, \"method\": \"dspark\"}" ] }, "eagle3": { "label": "Eagle3", "description": "Eagle3 speculative decoding with externally trained draft model; the served checkpoint is unchanged.", "variants": [ "default" ], "args": [ "--speculative-config", "{\"model\": \"RedHatAI/gemma-4-31B-it-speculator.eagle3\", \"num_speculative_tokens\": 3, \"method\": \"eagle3\"}" ] } } } }, "opt_in_features": [ "text_only", "spec_decoding" ], "variants": { "default": { "precision": "bf16", "vram_minimum_gb": 75, "description": "Full BF16 — single 80 GB NVIDIA GPU or 1x MI300X/MI325X/MI350X/MI355X" }, "fp8": { "model_id": "RedHatAI/gemma-4-31B-it-FP8-dynamic", "precision": "fp8", "vram_minimum_gb": 38, "description": "FP8 (E4M3) linear weights with dynamic per-token activation quantization (vision tower stays BF16) — Hopper or Blackwell", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/RedHatAI/gemma-4-31B-it-FP8-dynamic.json" }, "nvfp4": { "model_id": "nvidia/gemma-4-31B-it-NVFP4", "precision": "nvfp4", "vram_minimum_gb": 19, "description": "NVIDIA NVFP4 quantized weights for Blackwell GPUs", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/nvidia/gemma-4-31B-it-NVFP4.json" }, "redhat_nvfp4": { "model_id": "RedHatAI/gemma-4-31B-it-NVFP4", "precision": "nvfp4", "label": "NVFP4 (RedHat)", "vram_minimum_gb": 19, "description": "RedHatAI compressed-tensors NVFP4 quantization for Blackwell GPUs", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/RedHatAI/gemma-4-31B-it-NVFP4.json" }, "w4a16": { "model_id": "google/gemma-4-31B-it-qat-w4a16-ct", "precision": "int4", "vram_minimum_gb": 20, "description": "W4A16 QAT (4-bit weights, 16-bit activations, group_size=32) via compressed-tensors — runs on any GPU", "json": "/google/gemma-4-31B-it-qat-w4a16-ct.json" }, "redhat_fp8": { "model_id": "RedHatAI/gemma-4-31B-it-FP8-block", "precision": "fp8", "label": "FP8 Block (RedHat)", "vram_minimum_gb": 38, "description": "RedHatAI FP8 block quantization", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/RedHatAI/gemma-4-31B-it-FP8-block.json" } }, "hardware_overrides": { "xpu": { "extra_args": [ "--enforce-eager", "--attention-backend", "flash_attn", "--trust-remote-code", "--gpu-memory-utilization", "0.9", "--no-enable-prefix-caching", "--max-num-batched-tokens", "8192", "--max-model-len", "5120", "--tensor-parallel-size", "4" ] } }, "guide": "## Overview\n\n[Gemma 4](https://ai.google.dev/gemma/docs) is Google's most capable open model family, featuring a unified multimodal architecture that natively processes text, images, and audio. Gemma 4 models support structured thinking/reasoning, function calling with a custom tool-use protocol, and dynamic vision resolution — all available through vLLM's OpenAI-compatible API.\n\n### Key Features\n- **Multimodal**: Text + images natively (video via custom frame-extraction pipeline). The smaller E2B and E4B models also support audio.\n- **MoE variant**: 128 fine-grained experts with top-8 routing and custom GELU-activated FFN (Gemma 4 26B-A4B).\n- **Dual Attention**: Alternating sliding-window (local) and global attention with different head dimensions.\n- **Thinking Mode**: Structured reasoning via `<|channel>thought\\n...` delimiters.\n- **Function Calling**: Custom tool-call protocol with dedicated special tokens.\n- **Dynamic Vision Resolution**: Per-request configurable vision token budget (70, 140, 280, 560, 1120 tokens).\n\n### Supported Variants\n\nDense:\n- `google/gemma-4-E2B-it` (effective 2B)\n- `google/gemma-4-E4B-it` (effective 4B)\n- `google/gemma-4-31B-it` (31B)\n\nMoE:\n- `google/gemma-4-26B-A4B-it` (26B total / 4B active)\n\nTPU support is provided through [vLLM TPU](https://github.com/vllm-project/tpu-inference) with recipes for [Trillium](https://github.com/AI-Hypercomputer/tpu-recipes/tree/main/inference/trillium/vLLM/Gemma4) and [Ironwood](https://github.com/AI-Hypercomputer/tpu-recipes/blob/main/inference/ironwood/vLLM/Gemma4/).\n\n## Prerequisites\n\n### pip (NVIDIA CUDA)\n```bash\nuv venv\nsource .venv/bin/activate\nuv pip install -U vllm --pre \\\n --extra-index-url https://wheels.vllm.ai/nightly/cu129 \\\n --extra-index-url https://download.pytorch.org/whl/cu129 \\\n --index-strategy unsafe-best-match\n```\n\n### pip (AMD ROCm: MI300X, MI325X, MI350X, MI355X)\nRequires Python 3.12, ROCm 7.2.1, glibc >= 2.35 (Ubuntu 22.04+).\n```bash\nuv venv --python 3.12\nsource .venv/bin/activate\nuv pip install vllm --pre \\\n --extra-index-url https://wheels.vllm.ai/rocm/nightly/rocm721 --upgrade\n```\n\n### Docker\n```bash\ndocker pull vllm/vllm-openai:gemma4-0505-cu129 # NVIDIA Hopper (H100/H200, CUDA 12.9)\ndocker pull vllm/vllm-openai:gemma4-0505-cu130 # NVIDIA Blackwell (B200/B300, CUDA 13.0)\ndocker pull vllm/vllm-openai-rocm:latest # AMD\ndocker pull vllm/vllm-openai-xpu:latest # Intel XPU (B60 / B70)\n```\nTPU images are published separately by [vllm-project/tpu-inference](https://github.com/vllm-project/tpu-inference); see the Trillium / Ironwood tpu-recipes below for the pinned tag.\n\n## Deployment Configurations\n\n### Quick Start (Single GPU)\n```bash\nvllm serve google/gemma-4-E4B-it \\\n --max-model-len # up to 131072\n```\n\n### 31B Dense on 2xA100/H100 (TP=2, BF16)\n```bash\nvllm serve google/gemma-4-31B-it \\\n --tensor-parallel-size 2 \\\n --max-model-len 32768 \\\n --gpu-memory-utilization 0.90\n```\n\n### 26B MoE on 1xA100/H100 (BF16)\n```bash\nvllm serve google/gemma-4-26B-A4B-it \\\n --max-model-len 32768 \\\n --gpu-memory-utilization 0.90\n```\n\n### Full-Featured Server Launch\nEnables text, image, audio, thinking, and tool calling:\n```bash\nvllm serve google/gemma-4-31B-it \\\n --tensor-parallel-size 2 \\\n --max-model-len 16384 \\\n --gpu-memory-utilization 0.90 \\\n --enable-auto-tool-choice \\\n --reasoning-parser gemma4 \\\n --tool-call-parser gemma4 \\\n --chat-template examples/tool_chat_template_gemma4.jinja \\\n --limit-mm-per-prompt '{\"image\": 4, \"audio\": 1}' \\\n --async-scheduling \\\n --host 0.0.0.0 \\\n --port 8000\n```\n\n### Docker (NVIDIA)\n```bash\ndocker run -itd --name gemma4 \\\n --ipc=host --network host --shm-size 16G --gpus all \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:gemma4-0505-cu129 \\\n --model google/gemma-4-31B-it \\\n --tensor-parallel-size 2 \\\n --max-model-len 32768 \\\n --gpu-memory-utilization 0.90 \\\n --host 0.0.0.0 --port 8000\n```\nSwap `vllm/vllm-openai:gemma4-0505-cu129` for `vllm/vllm-openai:gemma4-0505-cu130` on Blackwell (B200/B300).\n\n### Docker (AMD MI300X/MI325X/MI350X/MI355X)\n```bash\ndocker run -itd --name gemma4-rocm \\\n --ipc=host --network=host --privileged \\\n --cap-add=CAP_SYS_ADMIN --device=/dev/kfd --device=/dev/dri \\\n --group-add=video --cap-add=SYS_PTRACE \\\n --security-opt=seccomp=unconfined --shm-size 16G \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai-rocm:latest \\\n --model google/gemma-4-31B-it \\\n --host 0.0.0.0 --port 8000\n```\n\n### Docker (Intel XPU B60 / B70, TP=4, Full-Featured)\n\nValidated on 4× Intel Arc Pro B60 / B70 with the official vLLM XPU image.\n\n```bash\ndocker run --device /dev/dri \\\n -v /dev/dri/by-path:/dev/dri/by-path --shm-size=16g \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai-xpu:latest \\\n google/gemma-4-31B-it \\\n --enforce-eager \\\n --attention-backend flash_attn \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --enable-auto-tool-choice \\\n --reasoning-parser gemma4 \\\n --tool-call-parser gemma4 \\\n --chat-template examples/tool_chat_template_gemma4.jinja \\\n --limit-mm-per-prompt '{\"image\": 4, \"audio\": 1}' \\\n --async-scheduling \\\n --gpu-memory-utilization 0.9 \\\n --no-enable-prefix-caching \\\n --max-num-batched-tokens 8192 \\\n --max-model-len 5120 \\\n --tensor-parallel-size 4\n```\n\n### Docker (Cloud TPU — Trillium / Ironwood)\nTPU uses the separate `vllm/vllm-tpu` image (no pip wheel). Pull the tag specified by the upstream [Trillium](https://github.com/AI-Hypercomputer/tpu-recipes/tree/main/inference/trillium/vLLM/Gemma4) or [Ironwood](https://github.com/AI-Hypercomputer/tpu-recipes/blob/main/inference/ironwood/vLLM/Gemma4/) recipe, then run:\n```bash\ndocker run -itd --name gemma4-tpu \\\n --privileged --network host --shm-size 16G \\\n -v /dev/shm:/dev/shm -e HF_TOKEN=$HF_TOKEN \\\n vllm/vllm-tpu:latest \\\n --model google/gemma-4-31B-it \\\n --tensor-parallel-size 8 \\\n --max-model-len 16384 \\\n --disable_chunked_mm_input \\\n --host 0.0.0.0 --port 8000\n```\n\n## Client Usage\n\n### Text Generation\n```python\nfrom openai import OpenAI\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresponse = client.chat.completions.create(\n model=\"google/gemma-4-31B-it\",\n messages=[{\"role\": \"user\", \"content\": \"Write a poem about the ocean.\"}],\n max_tokens=512, temperature=0.7,\n)\nprint(response.choices[0].message.content)\n```\n\n### Image Understanding\n```python\nresponse = client.chat.completions.create(\n model=\"google/gemma-4-31B-it\",\n messages=[{\"role\": \"user\", \"content\": [\n {\"type\": \"image_url\", \"image_url\": {\"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg\"}},\n {\"type\": \"text\", \"text\": \"Describe this image in detail.\"},\n ]}],\n max_tokens=1024,\n)\n```\n\n### Dynamic Vision Resolution\nSupported values: 70, 140, 280 (default), 560, 1120 tokens/image.\n```bash\nvllm serve google/gemma-4-31B-it \\\n --mm-processor-kwargs '{\"max_soft_tokens\": 560}'\n```\n\n### Audio (E2B / E4B)\nRequires `uv pip install \"vllm[audio]\"`.\n```bash\nvllm serve google/gemma-4-E2B-it \\\n --max-model-len 8192 \\\n --limit-mm-per-prompt '{\"image\": 4, \"audio\": 1}'\n```\n\n### Thinking Mode\n```bash\nvllm serve google/gemma-4-31B-it \\\n --max-model-len 16384 \\\n --enable-auto-tool-choice \\\n --reasoning-parser gemma4 \\\n --tool-call-parser gemma4 \\\n --chat-template examples/tool_chat_template_gemma4.jinja\n```\nEnable thinking per-request via `extra_body={\"chat_template_kwargs\": {\"enable_thinking\": True}}`, or default-on with `--default-chat-template-kwargs '{\"enable_thinking\": true}'`.\n\n### Structured Outputs\nvLLM guided decoding constrains output to a JSON schema. Include semantic instructions in the system prompt — the model does not see schema descriptions.\n\n## Configuration Tips\n\n- Set `--max-model-len` to match your workload.\n- `--gpu-memory-utilization 0.90–0.95` maximizes KV cache.\n- Image-only workloads: pass `--limit-mm-per-prompt.audio 0`.\n- Text-only workloads: pass `--limit-mm-per-prompt '{\"image\": 0, \"audio\": 0}'` to skip MM profiling.\n- `--async-scheduling` improves throughput.\n- FP8 KV cache (`--kv-cache-dtype fp8`) saves ~50% KV memory.\n\n## Quantized Variants\n\nTwo pre-quantized checkpoints are available:\n\n- [`RedHatAI/gemma-4-31B-it-FP8-dynamic`](https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-dynamic) — FP8 (E4M3) linear weights with dynamic per-token activation quantization (vision tower stays BF16); runs on Hopper and Blackwell.\n- [`nvidia/gemma-4-31B-it-NVFP4`](https://huggingface.co/nvidia/gemma-4-31B-it-NVFP4) — NVFP4 (4-bit) weights; requires Blackwell (B200/B300).\n\nPick them from the **Variant** dropdown above, or pass the repo id directly to `vllm serve`.\n\n## Throughput vs Latency\n\n| Goal | TP | `--max-num-seqs` | Notes |\n|------|----|------------------|-------|\n| Max throughput | 1-2 | 256-512 | Best tok/s per GPU |\n| Min latency | 4-8 | 8-16 | Best TTFT/TPOT |\n| Balanced | 2 | 128 | Mixed workloads |\n\n## Speculative Decoding (MTP)\n\nEnable the **Spec Decoding** feature toggle (above) or add `--speculative-config` manually to use MTP drafting with the [assistant model](https://huggingface.co/google/gemma-4-31B-it-assistant). Recommended `num_speculative_tokens`: 4–8 for this model. See the [Gemma 4 usage guide](../../Google/Gemma4) for details and benchmarks.\n\n> **Note:** MTP speculative decoding for Gemma 4 is only available on the vLLM nightly build — it has not yet landed in a stable release. Install via the nightly wheel (`uv pip install -U vllm --pre --extra-index-url https://wheels.vllm.ai/nightly/cu129 …`) or use the `vllm/vllm-openai:gemma4-0505-cu129` / `vllm/vllm-openai:gemma4-0505-cu130` images above; the standard `:latest` stable tag does not include this feature.\n\n## References\n\n- [Model card](https://huggingface.co/google/gemma-4-31B-it)\n- [FP8 variant](https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-dynamic)\n- [NVFP4 variant](https://huggingface.co/nvidia/gemma-4-31B-it-NVFP4)\n- [Gemma docs](https://ai.google.dev/gemma/docs)\n- [vLLM Gemma 4 tool-call template](https://github.com/vllm-project/vllm/blob/main/examples/tool_chat_template_gemma4.jinja)\n- [TPU recipes: Trillium](https://github.com/AI-Hypercomputer/tpu-recipes/tree/main/inference/trillium/vLLM/Gemma4)\n- [TPU recipes: Ironwood](https://github.com/AI-Hypercomputer/tpu-recipes/blob/main/inference/ironwood/vLLM/Gemma4/)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "389ffebe555705c85337025d5df108d0384d365e", "path": "recipes/models/Google/gemma-4-31B-it.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/Google/gemma-4-31B-it.yaml", "overlays": [] } }