{ "hf_id": "Google/gemma-4-26B-A4B-it", "meta": { "title": "Gemma 4 26B-A4B IT", "slug": "gemma-4-26b-a4b-it", "provider": "Google", "description": "Google's Gemma 4 MoE multimodal model (26B total / 4B active) with 128 fine-grained experts, top-8 routing, thinking mode, and tool-use protocol.", "date_added": "2026-04-02", "date_updated": "2026-09-27", "tasks": [ "multimodal", "text" ], "performance_headline": "MoE multimodal model — 26B total / 4B active, 128 experts with top-8 routing", "related_recipes": [ "google/gemma-4-E2B-it", "google/gemma-4-E4B-it", "google/gemma-4-31B-it" ], "platforms": [ { "id": "modal", "blurb": "Serverless deploy via the Gemma 4 Modal script bundled with this recipe.", "script": "https://github.com/vllm-project/recipes/blob/main/Google/gemma4-modal.py", "install": "curl -O https://raw.githubusercontent.com/vllm-project/recipes/main/Google/gemma4-modal.py\npip install modal\nmodal setup\nmodal deploy gemma4-modal.py\nmodal run gemma4-modal.py\n" } ], "hardware": { "h100": "verified", "h200": "verified", "mi300x": "verified", "mi325x": "verified", "mi355x": "verified", "trillium": "verified", "ironwood": "verified", "xeon6": "verified", "dgx_spark_gb10": "verified", "dgx_station_gb300": "verified", "rtx_pro_6000": "verified", "rtx_5090": "verified", "arc_pro_b60": "verified", "arc_pro_b70": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": {}, "docker_image": "vllm/vllm-openai:latest", "command": "vllm serve google/gemma-4-26B-A4B-it \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser gemma4 \\\n --chat-template examples/tool_chat_template_gemma4.jinja \\\n --reasoning-parser gemma4", "argv": [ "vllm", "serve", "google/gemma-4-26B-A4B-it", "--tensor-parallel-size", "1", "--enable-auto-tool-choice", "--tool-call-parser", "gemma4", "--chat-template", "examples/tool_chat_template_gemma4.jinja", "--reasoning-parser", "gemma4" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:latest google/gemma-4-26B-A4B-it \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser gemma4 \\\n --chat-template examples/tool_chat_template_gemma4.jinja \\\n --reasoning-parser gemma4", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "vllm/vllm-openai:latest", "google/gemma-4-26B-A4B-it", "--tensor-parallel-size", "1", "--enable-auto-tool-choice", "--tool-call-parser", "gemma4", "--chat-template", "examples/tool_chat_template_gemma4.jinja", "--reasoning-parser", "gemma4" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "single_node_tep": "/Google/gemma-4-26B-A4B-it/strategies/single_node_tep.json", "single_node_dep": "/Google/gemma-4-26B-A4B-it/strategies/single_node_dep.json", "multi_node_tp": "/Google/gemma-4-26B-A4B-it/strategies/multi_node_tp.json", "multi_node_tep": "/Google/gemma-4-26B-A4B-it/strategies/multi_node_tep.json", "multi_node_dep": "/Google/gemma-4-26B-A4B-it/strategies/multi_node_dep.json", "kv_store_centralized_mooncake": "/Google/gemma-4-26B-A4B-it/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/Google/gemma-4-26B-A4B-it/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/Google/gemma-4-26B-A4B-it/hw/h200.json", "h100": "/Google/gemma-4-26B-A4B-it/hw/h100.json", "dgx_station_gb300": "/Google/gemma-4-26B-A4B-it/hw/dgx_station_gb300.json", "dgx_spark_gb10": "/Google/gemma-4-26B-A4B-it/hw/dgx_spark_gb10.json", "rtx_pro_6000": "/Google/gemma-4-26B-A4B-it/hw/rtx_pro_6000.json", "rtx_5090": "/Google/gemma-4-26B-A4B-it/hw/rtx_5090.json", "mi300x": "/Google/gemma-4-26B-A4B-it/hw/mi300x.json", "mi325x": "/Google/gemma-4-26B-A4B-it/hw/mi325x.json", "mi355x": "/Google/gemma-4-26B-A4B-it/hw/mi355x.json", "trillium": "/Google/gemma-4-26B-A4B-it/hw/trillium.json", "ironwood": "/Google/gemma-4-26B-A4B-it/hw/ironwood.json", "xeon6": "/Google/gemma-4-26B-A4B-it/hw/xeon6.json" } }, "model": { "model_id": "google/gemma-4-26B-A4B-it", "min_vllm_version": "0.25.0", "architecture": "moe", "parameter_count": "26B", "active_parameters": "4B", "context_length": 131072, "base_args": [], "base_env": {}, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:latest" } } }, "features": { "tool_calling": { "description": "Enable automatic tool choice with Gemma 4 parser and chat template", "args": [ "--enable-auto-tool-choice", "--tool-call-parser", "gemma4", "--chat-template", "examples/tool_chat_template_gemma4.jinja" ], "hardware_overrides": { "dgx_spark_gb10": { "args": [ "--enable-auto-tool-choice", "--tool-call-parser", "gemma4" ] } } }, "reasoning": { "description": "Enable structured thinking/reasoning output", "args": [ "--reasoning-parser", "gemma4" ] }, "text_only": { "description": "Skip loading the vision encoder for text-only workloads — frees VRAM for KV cache. Mutually exclusive with encoder_parallel.", "args": [ "--language-model-only" ] }, "spec_decoding": { "description": "MTP speculative decoding for accelerated inference", "default_mode": "mtp", "modes": { "mtp": { "label": "MTP", "description": "Built-in Multi-Token Prediction, 4 draft tokens.", "args": [ "--speculative-config", "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" ], "hardware_overrides": { "dgx_spark_gb10": { "args": [ "--speculative-config", "{\"method\":\"mtp\",\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4,\"moe_backend\":\"triton\"}" ] }, "dgx_station_gb300": { "args": [ "--speculative-config", "{\"method\":\"mtp\",\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4,\"moe_backend\":\"triton\"}" ] }, "rtx_pro_6000": { "args": [ "--speculative-config", "{\"method\":\"mtp\",\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4,\"moe_backend\":\"triton\"}" ] } } }, "eagle3": { "label": "Eagle3", "description": "Eagle3 speculative decoding with externally trained draft model; the served checkpoint is unchanged.", "variants": [ "default" ], "args": [ "--speculative-config", "{\"model\": \"RedHatAI/gemma-4-26B-A4B-it-speculator.eagle3\", \"num_speculative_tokens\": 3, \"method\": \"eagle3\"}" ] } } } }, "opt_in_features": [ "text_only", "spec_decoding" ], "variants": { "default": { "precision": "bf16", "vram_minimum_gb": 64, "description": "Full BF16 — single 80 GB NVIDIA GPU or 1x MI300X/MI325X/MI350X/MI355X or 2x Xeon6/Xeon5 NUMA nodes", "supported_hardware": [ "h100", "h200", "mi300x", "mi325x", "mi355x", "trillium", "ironwood", "xeon6", "dgx_spark_gb10", "dgx_station_gb300", "rtx_pro_6000", "rtx_5090" ], "hardware_overrides": { "dgx_spark_gb10": { "extra_env": { "VLLM_USE_V2_MODEL_RUNNER": "1" } }, "dgx_station_gb300": { "extra_env": { "VLLM_USE_V2_MODEL_RUNNER": "1" } }, "rtx_pro_6000": { "extra_env": { "VLLM_USE_V2_MODEL_RUNNER": "1" }, "extra_args": [ "--gpu-memory-utilization", "0.9" ] } } }, "fp8": { "model_id": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic", "precision": "fp8", "vram_minimum_gb": 32, "description": "FP8 (E4M3) weights with dynamic per-token activation quantization — Hopper or Blackwell", "supported_hardware": [ "h100", "h200", "dgx_spark_gb10", "dgx_station_gb300", "rtx_pro_6000", "rtx_5090" ], "extra_args": [ "--kv-cache-dtype", "fp8" ], "hardware_overrides": { "dgx_spark_gb10": { "extra_env": { "VLLM_USE_V2_MODEL_RUNNER": "1" } }, "dgx_station_gb300": { "extra_env": { "VLLM_USE_V2_MODEL_RUNNER": "1" } }, "rtx_pro_6000": { "extra_env": { "VLLM_USE_V2_MODEL_RUNNER": "1" }, "extra_args": [ "--gpu-memory-utilization", "0.9" ] }, "rtx_5090": { "extra_env": { "VLLM_USE_V2_MODEL_RUNNER": "1" }, "extra_args": [ "--gpu-memory-utilization", "0.92", "--max-model-len", "131072", "--max-num-seqs", "1" ] } }, "json": "/RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic.json" }, "nvfp4": { "model_id": "RedHatAI/gemma-4-26B-A4B-it-NVFP4", "precision": "nvfp4", "label": "NVFP4 - RedHatAI", "vram_minimum_gb": 16, "description": "NVFP4 (4-bit) quantized weights — requires Blackwell (B200/B300)", "supported_hardware": [ "dgx_spark_gb10", "dgx_station_gb300", "rtx_pro_6000", "rtx_5090" ], "extra_args": [ "--kv-cache-dtype", "fp8" ], "hardware_overrides": { "dgx_spark_gb10": { "extra_env": { "VLLM_USE_V2_MODEL_RUNNER": "1" } }, "dgx_station_gb300": { "extra_env": { "VLLM_USE_V2_MODEL_RUNNER": "1" } }, "rtx_pro_6000": { "extra_env": { "VLLM_USE_V2_MODEL_RUNNER": "1" }, "extra_args": [ "--gpu-memory-utilization", "0.9" ] }, "rtx_5090": { "extra_env": { "VLLM_USE_V2_MODEL_RUNNER": "1" }, "extra_args": [ "--gpu-memory-utilization", "0.9", "--max-num-batched-tokens", "4096", "--max-num-seqs", "4" ] } }, "json": "/RedHatAI/gemma-4-26B-A4B-it-NVFP4.json" }, "nvidia_nvfp4": { "model_id": "nvidia/Gemma-4-26B-A4B-NVFP4", "precision": "nvfp4", "label": "NVFP4 - NVIDIA", "vram_minimum_gb": 16, "description": "NVIDIA ModelOpt NVFP4 (4-bit) quantized weights — validated on the single-GPU Blackwell desktops (DGX Spark GB10, DGX Station GB300, RTX PRO 6000)", "supported_hardware": [ "dgx_spark_gb10", "dgx_station_gb300", "rtx_pro_6000" ], "hardware_overrides": { "dgx_spark_gb10": { "docker_image": "eugr/spark-vllm:nightly-20260704", "docker_serve_command": "vllm serve", "install": { "pip": false }, "extra_env": { "VLLM_USE_V2_MODEL_RUNNER": "1" }, "extra_args": [ "--gpu-memory-utilization", "0.8", "--max-model-len", "262144", "--max-num-seqs", "8", "--max-num-batched-tokens", "8192", "--enable-prefix-caching", "--load-format", "fastsafetensors" ] }, "dgx_station_gb300": { "extra_env": { "VLLM_USE_V2_MODEL_RUNNER": "1" }, "extra_args": [ "--gpu-memory-utilization", "0.92", "--max-model-len", "262144", "--max-num-seqs", "64", "--max-num-batched-tokens", "8192", "--enable-chunked-prefill", "--enable-prefix-caching", "--load-format", "fastsafetensors" ] }, "rtx_pro_6000": { "extra_env": { "VLLM_USE_V2_MODEL_RUNNER": "1" }, "extra_args": [ "--gpu-memory-utilization", "0.92", "--max-model-len", "262144", "--max-num-seqs", "8", "--max-num-batched-tokens", "8192", "--enable-prefix-caching", "--load-format", "fastsafetensors" ] }, "rtx_5090": { "extra_env": { "VLLM_USE_V2_MODEL_RUNNER": "1" }, "extra_args": [ "--kv-cache-dtype", "fp8", "--gpu-memory-utilization", "0.9", "--max-num-batched-tokens", "4096", "--enable-prefix-caching", "--enable-chunked-prefill", "--async-scheduling", "--max-num-seqs", "4" ] } }, "json": "/nvidia/Gemma-4-26B-A4B-NVFP4.json" } }, "hardware_overrides": { "xpu": { "extra_args": [ "--enforce-eager", "--attention-backend", "flash_attn", "--trust-remote-code", "--gpu-memory-utilization", "0.9", "--no-enable-prefix-caching", "--max-num-batched-tokens", "8192", "--max-model-len", "5120", "--tensor-parallel-size", "4" ] } }, "guide": "## Overview\n\n[Gemma 4 26B-A4B](https://huggingface.co/google/gemma-4-26B-A4B-it) is the Mixture-of-Experts member of Google's Gemma 4 family — 26B total parameters with only 4B active per token via 128 fine-grained experts and top-8 routing. It supports text + images natively, structured thinking, function calling, and dynamic vision resolution.\n\nIntel Xeon 6 CPU support remains verified. Benchmark-specific TP/DP, CPU binding,\nsequence limits, and repro-only timeouts remain deployment choices.\n\n### Key Features\n- **MoE**: 128 fine-grained experts with top-8 routing and custom GELU-activated FFN.\n- **Multimodal**: Text + images natively (video via custom frame-extraction pipeline). Audio is only supported on the smaller E2B/E4B variants.\n- **Dual Attention**: Alternating sliding-window (local) and global attention with different head dimensions.\n- **Thinking Mode**: Structured reasoning via `<|channel>thought\\n...` delimiters.\n- **Function Calling**: Custom tool-call protocol with dedicated special tokens.\n- **Dynamic Vision Resolution**: Per-request configurable vision token budget (70, 140, 280, 560, 1120 tokens).\n\nTPU support is provided through [vLLM TPU](https://github.com/vllm-project/tpu-inference) with recipes for [Trillium](https://github.com/AI-Hypercomputer/tpu-recipes/tree/main/inference/trillium/vLLM/Gemma4) and [Ironwood](https://github.com/AI-Hypercomputer/tpu-recipes/blob/main/inference/ironwood/vLLM/Gemma4/).\n\n## Prerequisites\n\n- Hardware: Intel Xeon 6/Xeon 5 CPUs\n- vLLM >= 0.25.0\n\n### pip (NVIDIA CUDA)\n```bash\nuv venv\nsource .venv/bin/activate\nuv pip install -U vllm --pre \\\n --extra-index-url https://wheels.vllm.ai/nightly/cu129 \\\n --extra-index-url https://download.pytorch.org/whl/cu129 \\\n --index-strategy unsafe-best-match\n```\n\n### pip (AMD ROCm: MI300X, MI325X, MI350X, MI355X)\nRequires Python 3.12, ROCm 7.2.1, glibc >= 2.35 (Ubuntu 22.04+).\n```bash\nuv venv --python 3.12\nsource .venv/bin/activate\nuv pip install vllm --pre \\\n --extra-index-url https://wheels.vllm.ai/rocm/nightly/rocm721 --upgrade\n```\n\n### pip (Intel Xeon 6 CPUs)\nFor Intel and AMD x86 CPUs, follow the [CPU pre-built wheels](https://docs.vllm.ai/en/latest/getting_started/installation/cpu/#pre-built-wheels) installation instructions.\n\n### Docker\n```bash\ndocker pull vllm/vllm-openai:gemma4-0505-cu129 # NVIDIA Hopper (H100/H200, CUDA 12.9)\ndocker pull vllm/vllm-openai:gemma4-0505-cu130 # NVIDIA Blackwell (B200/B300, CUDA 13.0)\ndocker pull vllm/vllm-openai-rocm:latest # AMD\ndocker pull vllm/vllm-openai-cpu:latest-x86_64 # For Intel Xeon 6\ndocker pull vllm/vllm-openai-xpu:latest # Intel XPU (B60 / B70)\n```\nTPU images are published separately by [vllm-project/tpu-inference](https://github.com/vllm-project/tpu-inference); see the Trillium / Ironwood tpu-recipes below for the pinned tag.\n\n## Deployment Configurations\n\n### 26B MoE on 1x A100/H100 (BF16)\n```bash\nvllm serve google/gemma-4-26B-A4B-it \\\n --max-model-len 32768 \\\n --gpu-memory-utilization 0.90\n```\n\n### Full-Featured Server Launch\nEnables text, image, thinking, and tool calling:\n```bash\nvllm serve google/gemma-4-26B-A4B-it \\\n --max-model-len 16384 \\\n --gpu-memory-utilization 0.90 \\\n --enable-auto-tool-choice \\\n --reasoning-parser gemma4 \\\n --tool-call-parser gemma4 \\\n --chat-template examples/tool_chat_template_gemma4.jinja \\\n --limit-mm-per-prompt.image 4 \\\n --async-scheduling \\\n --host 0.0.0.0 \\\n --port 8000\n```\n\n### Docker (NVIDIA)\n```bash\ndocker run -itd --name gemma4-moe \\\n --ipc=host --network host --shm-size 16G --gpus all \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:gemma4-0505-cu129 \\\n --model google/gemma-4-26B-A4B-it \\\n --max-model-len 32768 \\\n --gpu-memory-utilization 0.90 \\\n --host 0.0.0.0 --port 8000\n```\nSwap `vllm/vllm-openai:gemma4-0505-cu129` for `vllm/vllm-openai:gemma4-0505-cu130` on Blackwell (B200/B300).\n\n### Docker (AMD MI300X/MI325X/MI350X/MI355X)\n```bash\ndocker run -itd --name gemma4-rocm \\\n --ipc=host --network=host --privileged \\\n --cap-add=CAP_SYS_ADMIN --device=/dev/kfd --device=/dev/dri \\\n --group-add=video --cap-add=SYS_PTRACE \\\n --security-opt=seccomp=unconfined --shm-size 16G \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai-rocm:latest \\\n --model google/gemma-4-26B-A4B-it \\\n --host 0.0.0.0 --port 8000\n```\n\n### Docker (Cloud TPU — Trillium / Ironwood)\nTPU uses the separate `vllm/vllm-tpu` image (no pip wheel). Pull the tag specified by the upstream [Trillium](https://github.com/AI-Hypercomputer/tpu-recipes/tree/main/inference/trillium/vLLM/Gemma4) or [Ironwood](https://github.com/AI-Hypercomputer/tpu-recipes/blob/main/inference/ironwood/vLLM/Gemma4/) recipe, then run:\n```bash\ndocker run -itd --name gemma4-tpu \\\n --privileged --network host --shm-size 16G \\\n -v /dev/shm:/dev/shm -e HF_TOKEN=$HF_TOKEN \\\n vllm/vllm-tpu:latest \\\n --model google/gemma-4-26B-A4B-it \\\n --tensor-parallel-size 8 \\\n --max-model-len 16384 \\\n --disable_chunked_mm_input \\\n --host 0.0.0.0 --port 8000\n```\nTrillium requires a 4-chip slice minimum; Ironwood runs on a single chip.\n\n### Intel Xeon 6 Deployment via Docker\n\nLaunch the x86 CPU vLLM Docker container for `google/gemma-4-26B-A4B-it`:\n\n```bash\ndocker run \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai-cpu:latest-x86_64 google/gemma-4-26B-A4B-it \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser gemma4 \\\n --chat-template examples/tool_chat_template_gemma4.jinja \\\n --reasoning-parser gemma4\n```\n\nFor additional Intel Xeon 6 deployment details, see the Intel Software Catalog entries for [Gemma 4 26B-A4B IT](https://aiswcatalog.intel.com/models/google-gemma-4-26b-a4b-it).\n\nThe following settings are intentionally not prescribed as portable CPU model\ndefaults because they depend on the workload, hardware, or runtime environment:\n\n- `--max-num-batched-tokens`: scheduler/throughput tuning.\n- `--max-num-seqs`: concurrency and scheduler-capacity tuning.\n- `--gpu-memory-utilization`: platform memory-budget tuning.\n- `--no-enable-prefix-caching`: workload/benchmark cache-policy tuning.\n- `VLLM_ENGINE_ITERATION_TIMEOUT_S`: operational runtime timeout.\n\nThese settings may still appear in validated hardware-specific overrides. Tune\nthem at deployment time based on platform resources, workload shape, and\nlatency/throughput goals.\n\n### Docker (Intel XPU B60 / B70, TP=4, Full-Featured)\n\nValidated on 4x Intel Arc Pro B60 / B70 with the official vLLM XPU image.\n\n```bash\ndocker run --device /dev/dri \\\n -v /dev/dri/by-path:/dev/dri/by-path --shm-size=16g \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai-xpu:latest \\\n google/gemma-4-26B-A4B-it \\\n --enforce-eager \\\n --attention-backend flash_attn \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --enable-auto-tool-choice \\\n --reasoning-parser gemma4 \\\n --tool-call-parser gemma4 \\\n --chat-template examples/tool_chat_template_gemma4.jinja \\\n --limit-mm-per-prompt.image 4 \\\n --async-scheduling \\\n --gpu-memory-utilization 0.9 \\\n --no-enable-prefix-caching \\\n --max-num-batched-tokens 8192 \\\n --max-model-len 5120 \\\n --tensor-parallel-size 4\n```\n\n## Client Usage\n\n### Text Generation\n```python\nfrom openai import OpenAI\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresponse = client.chat.completions.create(\n model=\"google/gemma-4-26B-A4B-it\",\n messages=[{\"role\": \"user\", \"content\": \"Write a poem about the ocean.\"}],\n max_tokens=512, temperature=0.7,\n)\nprint(response.choices[0].message.content)\n```\n\n### Image Understanding\n```python\nresponse = client.chat.completions.create(\n model=\"google/gemma-4-26B-A4B-it\",\n messages=[{\"role\": \"user\", \"content\": [\n {\"type\": \"image_url\", \"image_url\": {\"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg\"}},\n {\"type\": \"text\", \"text\": \"Describe this image in detail.\"},\n ]}],\n max_tokens=1024,\n)\n```\n\n### Thinking Mode\n```bash\nvllm serve google/gemma-4-26B-A4B-it \\\n --max-model-len 16384 \\\n --reasoning-parser gemma4 \\\n --tool-call-parser gemma4 \\\n --enable-auto-tool-choice \\\n --chat-template examples/tool_chat_template_gemma4.jinja\n```\nEnable per-request via `extra_body={\"chat_template_kwargs\": {\"enable_thinking\": True}}`.\n\n### Dynamic Vision Resolution\nSupported values: 70, 140, 280 (default), 560, 1120 tokens/image.\n```bash\nvllm serve google/gemma-4-26B-A4B-it \\\n --mm-processor-kwargs '{\"max_soft_tokens\": 560}'\n```\n\n## Configuration Tips\n\n- Set `--max-model-len` to match your workload.\n- `--gpu-memory-utilization 0.90-0.95` maximizes KV cache.\n- Text-only workloads: `--limit-mm-per-prompt '{\"image\": 0, \"audio\": 0}'`.\n- `--async-scheduling` improves throughput.\n- FP8 KV cache (`--kv-cache-dtype fp8`) saves ~50% KV memory.\n- For MoE, TEP (tensor-expert parallelism) and DEP (data-expert parallelism) strategies scale better than pure TP at large node counts.\n\n## Quantized Variants\n\nThree pre-quantized checkpoints are available:\n\n- [`RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic`](https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic) — FP8 (E4M3) weights with dynamic per-token activation quantization; runs on Hopper and Blackwell.\n- [`RedHatAI/gemma-4-26B-A4B-it-NVFP4`](https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-NVFP4) — NVFP4 (4-bit) weights; requires Blackwell (B200/B300).\n- [`nvidia/Gemma-4-26B-A4B-NVFP4`](https://huggingface.co/nvidia/Gemma-4-26B-A4B-NVFP4) — NVIDIA ModelOpt NVFP4 (4-bit) weights; this is the checkpoint validated on the single-GPU Blackwell desktops below.\n\nPick them from the **Variant** dropdown above, or pass the repo id directly to `vllm serve`.\n\n### NVFP4 on the Blackwell desktops\n\n[`nvidia/Gemma-4-26B-A4B-NVFP4`](https://huggingface.co/nvidia/Gemma-4-26B-A4B-NVFP4)\nhas been run end to end on DGX Spark (GB10), DGX Station (GB300) and RTX PRO\n6000, in each case on one GPU at TP1 with MTP drafting against\n[`google/gemma-4-26B-A4B-it-assistant`](https://huggingface.co/google/gemma-4-26B-A4B-it-assistant).\n\nSelect the **NVIDIA NVFP4** variant, enable **Spec Decoding**, and switch the\nhardware pill — memory utilization, batch size, prefill scheduling and the\ndrafter's MoE backend all differ per box and are encoded per hardware, so the\ngenerated command is the tested one. Highlights:\n\n- **DGX Spark (GB10)** — 128 GB shared with the CPU, so `--gpu-memory-utilization 0.8`\n and an 8-way batch. Serve from `eugr/spark-vllm:nightly-20260704`: vLLM 0.24\n and the generic nightly both fail on this checkpoint, so the pip tab is\n hidden on this hardware.\n- **DGX Station (GB300)** — 252 GB on one GPU, so `--gpu-memory-utilization 0.92`\n with chunked prefill and a 64-way batch.\n- **RTX PRO 6000** — 96 GB, `--gpu-memory-utilization 0.92` and an 8-way batch.\n\nAll three run the full 262,144-token context, above the 131,072 in\n`config.json`, and pass `\"moe_backend\":\"triton\"` in `--speculative-config`\nbecause the drafter's default fused MoE path is not built in these images.\n\nThe GB300 and RTX PRO 6000 runs used an internal NVIDIA nightly vLLM build.\nAs noted under Speculative Decoding below, Gemma 4 MTP has not landed in a\nstable release, so the default `vllm/vllm-openai:latest` tag will not serve\nthese commands with Spec Decoding enabled — use a nightly wheel or a nightly\nimage on those two boxes.\n\n## Throughput vs Latency\n\n| Goal | TP | `--max-num-seqs` | Notes |\n|------|----|------------------|-------|\n| Max throughput | 1-2 | 256-512 | Best tok/s per GPU |\n| Min latency | 4-8 | 8-16 | Best TTFT/TPOT |\n| Balanced | 2 | 128 | Mixed workloads |\n\n## Speculative Decoding (MTP)\n\nEnable the **Spec Decoding** feature toggle (above) or add `--speculative-config` manually to use MTP drafting with the [assistant model](https://huggingface.co/google/gemma-4-26B-A4B-it-assistant). Recommended `num_speculative_tokens`: 4 for this model. See the [Gemma 4 usage guide](../../Google/Gemma4) for details and benchmarks.\n\n> **Note:** MTP speculative decoding for Gemma 4 is only available on the vLLM nightly build — it has not yet landed in a stable release. Install via the nightly wheel (`uv pip install -U vllm --pre --extra-index-url https://wheels.vllm.ai/nightly/cu129 …`) or use the `vllm/vllm-openai:gemma4-0505-cu129` / `vllm/vllm-openai:gemma4-0505-cu130` images above; the standard `:latest` stable tag does not include this feature.\n\n## References\n\n- [Model card](https://huggingface.co/google/gemma-4-26B-A4B-it)\n- [FP8 variant](https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic)\n- [NVFP4 variant](https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-NVFP4)\n- [NVIDIA ModelOpt NVFP4 variant](https://huggingface.co/nvidia/Gemma-4-26B-A4B-NVFP4)\n- [Gemma docs](https://ai.google.dev/gemma/docs)\n- [vLLM Gemma 4 tool-call template](https://github.com/vllm-project/vllm/blob/main/examples/tool_chat_template_gemma4.jinja)\n- [TPU recipes: Trillium](https://github.com/AI-Hypercomputer/tpu-recipes/tree/main/inference/trillium/vLLM/Gemma4)\n- [TPU recipes: Ironwood](https://github.com/AI-Hypercomputer/tpu-recipes/blob/main/inference/ironwood/vLLM/Gemma4/)\n- [Xeon 6 recipe: Gemma 4 26B-A4B IT](https://aiswcatalog.intel.com/models/google-gemma-4-26b-a4b-it)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "389ffebe555705c85337025d5df108d0384d365e", "path": "recipes/models/Google/gemma-4-26B-A4B-it.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/Google/gemma-4-26B-A4B-it.yaml", "overlays": [] } }