{
  "hf_id": "Google/gemma-4-12B-it",
  "meta": {
    "title": "Gemma 4 12B IT",
    "slug": "gemma-4-12b-it",
    "provider": "Google",
    "description": "Google's encoder-free unified Gemma 4 dense model (12B) with native text, image, and audio, plus thinking mode and tool-use protocol.",
    "date_added": "2026-06-03",
    "date_updated": "2026-09-09",
    "tasks": [
      "multimodal",
      "text"
    ],
    "performance_headline": "Encoder-free unified multimodal model with audio, structured thinking, and function calling — runs on a single 40 GB+ GPU",
    "related_recipes": [
      "google/gemma-4-E4B-it",
      "google/gemma-4-26B-A4B-it",
      "google/gemma-4-31B-it"
    ],
    "hardware": {
      "h100": "verified",
      "arc_pro_b60": "verified",
      "arc_pro_b70": "verified"
    }
  },
  "recommended_command": {
    "hardware": "h200",
    "strategy": "single_node_tp",
    "variant": "default",
    "node_count": 1,
    "deploy_type": "single_node",
    "env": {},
    "docker_image": "vllm/vllm-openai:gemma4-unified",
    "command": "vllm serve google/gemma-4-12B-it \\\n  --tensor-parallel-size 1 \\\n  --enable-auto-tool-choice \\\n  --tool-call-parser gemma4 \\\n  --chat-template examples/tool_chat_template_gemma4.jinja \\\n  --reasoning-parser gemma4",
    "argv": [
      "vllm",
      "serve",
      "google/gemma-4-12B-it",
      "--tensor-parallel-size",
      "1",
      "--enable-auto-tool-choice",
      "--tool-call-parser",
      "gemma4",
      "--chat-template",
      "examples/tool_chat_template_gemma4.jinja",
      "--reasoning-parser",
      "gemma4"
    ],
    "docker_command": "docker run --gpus all \\\n  --privileged --ipc=host -p 8000:8000 \\\n  -v ~/.cache/huggingface:/root/.cache/huggingface \\\n  vllm/vllm-openai:gemma4-unified google/gemma-4-12B-it \\\n  --tensor-parallel-size 1 \\\n  --enable-auto-tool-choice \\\n  --tool-call-parser gemma4 \\\n  --chat-template examples/tool_chat_template_gemma4.jinja \\\n  --reasoning-parser gemma4",
    "docker_argv": [
      "docker",
      "run",
      "--gpus",
      "all",
      "--privileged",
      "--ipc=host",
      "-p",
      "8000:8000",
      "-v",
      "~/.cache/huggingface:/root/.cache/huggingface",
      "vllm/vllm-openai:gemma4-unified",
      "google/gemma-4-12B-it",
      "--tensor-parallel-size",
      "1",
      "--enable-auto-tool-choice",
      "--tool-call-parser",
      "gemma4",
      "--chat-template",
      "examples/tool_chat_template_gemma4.jinja",
      "--reasoning-parser",
      "gemma4"
    ],
    "strategy_spec": {
      "name": "single_node_tp",
      "deploy_type": "single_node",
      "display_name": "Tensor Parallel",
      "short_label": "TP",
      "orientation": "latency",
      "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n",
      "hardware_match": {
        "min_gpus": 1,
        "max_gpus": 8,
        "multi_node": false
      },
      "vllm_args": [],
      "parallel_flag": "--tensor-parallel-size"
    },
    "hardware_profile": {
      "brand": "NVIDIA",
      "generation": "hopper",
      "compute_arch": "sm90",
      "display_name": "H200",
      "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node",
      "gpu_count": 8,
      "vram_gb": 1128,
      "multi_node": false
    },
    "alternatives": {
      "multi_node_tp": "/Google/gemma-4-12B-it/strategies/multi_node_tp.json",
      "kv_store_centralized_mooncake": "/Google/gemma-4-12B-it/strategies/kv_store_centralized_mooncake.json",
      "kv_store_distributed_mooncake": "/Google/gemma-4-12B-it/strategies/kv_store_distributed_mooncake.json"
    },
    "by_hardware": {
      "h200": "/Google/gemma-4-12B-it/hw/h200.json",
      "h100": "/Google/gemma-4-12B-it/hw/h100.json",
      "b200": "/Google/gemma-4-12B-it/hw/b200.json",
      "gb200": "/Google/gemma-4-12B-it/hw/gb200.json",
      "b300": "/Google/gemma-4-12B-it/hw/b300.json",
      "gb300": "/Google/gemma-4-12B-it/hw/gb300.json",
      "mi300x": "/Google/gemma-4-12B-it/hw/mi300x.json",
      "mi325x": "/Google/gemma-4-12B-it/hw/mi325x.json",
      "mi350x": "/Google/gemma-4-12B-it/hw/mi350x.json",
      "mi355x": "/Google/gemma-4-12B-it/hw/mi355x.json",
      "arc_pro_b60": "/Google/gemma-4-12B-it/hw/arc_pro_b60.json",
      "arc_pro_b70": "/Google/gemma-4-12B-it/hw/arc_pro_b70.json"
    }
  },
  "model": {
    "model_id": "google/gemma-4-12B-it",
    "min_vllm_version": "0.23.0",
    "docker_image": "vllm/vllm-openai:gemma4-unified",
    "architecture": "dense",
    "parameter_count": "12B",
    "active_parameters": "12B",
    "context_length": 131072,
    "base_args": [],
    "base_env": {},
    "install": {
      "pip": {
        "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto"
      },
      "docker": {
        "command": "docker pull vllm/vllm-openai:gemma4-unified"
      },
      "extras": [
        {
          "note": "Audio extras — only needed when serving the audio modality",
          "command": "uv pip install \"vllm[audio]\"",
          "optional": true
        }
      ]
    }
  },
  "features": {
    "tool_calling": {
      "description": "Enable automatic tool choice with Gemma 4 parser and chat template",
      "args": [
        "--enable-auto-tool-choice",
        "--tool-call-parser",
        "gemma4",
        "--chat-template",
        "examples/tool_chat_template_gemma4.jinja"
      ]
    },
    "reasoning": {
      "description": "Enable structured thinking/reasoning output",
      "args": [
        "--reasoning-parser",
        "gemma4"
      ]
    },
    "spec_decoding": {
      "description": "MTP speculative decoding for accelerated inference",
      "args": [
        "--speculative-config",
        "{\"model\":\"google/gemma-4-12B-it-assistant\",\"num_speculative_tokens\":4}"
      ]
    }
  },
  "opt_in_features": [
    "spec_decoding"
  ],
  "variants": {
    "default": {
      "precision": "bf16",
      "vram_minimum_gb": 29,
      "description": "Full BF16 — single 40 GB+ NVIDIA GPU"
    },
    "nvfp4": {
      "model_id": "RedHatAI/gemma-4-12B-it-NVFP4",
      "precision": "nvfp4",
      "vram_minimum_gb": 8,
      "description": "RedHatAI compressed-tensors NVFP4 quantization for Blackwell GPUs",
      "extra_args": [
        "--kv-cache-dtype",
        "fp8"
      ],
      "json": "/RedHatAI/gemma-4-12B-it-NVFP4.json"
    },
    "w4a16": {
      "model_id": "google/gemma-4-12B-it-qat-w4a16-ct",
      "precision": "int4",
      "vram_minimum_gb": 9,
      "description": "W4A16 QAT (4-bit weights, 16-bit activations, group_size=32) via compressed-tensors — runs on any GPU",
      "json": "/google/gemma-4-12B-it-qat-w4a16-ct.json"
    },
    "fp8": {
      "model_id": "RedHatAI/gemma-4-12B-it-FP8-Dynamic",
      "precision": "fp8",
      "vram_minimum_gb": 15,
      "description": "RedHatAI FP8 (E4M3) with dynamic per-token activation quantization",
      "extra_args": [
        "--kv-cache-dtype",
        "fp8"
      ],
      "json": "/RedHatAI/gemma-4-12B-it-FP8-Dynamic.json"
    }
  },
  "hardware_overrides": {
    "xpu": {
      "extra_args": [
        "--enforce-eager",
        "--attention-backend",
        "flash_attn",
        "--trust-remote-code",
        "--gpu-memory-utilization",
        "0.9",
        "--no-enable-prefix-caching",
        "--max-num-batched-tokens",
        "8192",
        "--max-model-len",
        "5120",
        "--tensor-parallel-size",
        "2"
      ]
    }
  },
  "guide": "## Overview\n\n[Gemma 4](https://ai.google.dev/gemma/docs) is Google's most capable open model family, featuring a unified multimodal architecture that natively processes text, images, and audio. The **12B Unified** model is *encoder-free*: instead of dedicated vision/audio encoders, it projects raw image patches and audio waveforms directly into the decoder's embedding space through lightweight linear layers, so every modality flows into a single decoder-only transformer. Gemma 4 models support structured thinking/reasoning, function calling with a custom tool-use protocol, and dynamic vision resolution — all available through vLLM's OpenAI-compatible API.\n\n### Key Features\n- **Encoder-free multimodality**: Text, image, and audio inputs handled by a single decoder-only transformer — no separate encoders to load, reducing multimodal latency.\n- **Dual Attention**: Alternating sliding-window (local, 1024 tokens) and global attention; the final layer is always global, with unified Keys/Values and Proportional RoPE on global layers for long-context efficiency.\n- **Thinking Mode**: Structured reasoning via `<|channel>thought\\n...<channel|>` delimiters.\n- **Function Calling**: Custom tool-call protocol with dedicated special tokens.\n- **Native System Prompt Support**: First-class `system` role.\n\n### Supported Variants\n\nDense:\n- `google/gemma-4-E2B-it` (effective 2B)\n- `google/gemma-4-E4B-it` (effective 4B)\n- `google/gemma-4-12B-it` (11.95B, encoder-free)\n- `google/gemma-4-31B-it` (31B)\n\nMoE:\n- `google/gemma-4-26B-A4B-it` (26B total / 4B active)\n\n> **Context length**: this recipe pins `context_length` to the value in `config.json` (`max_position_embeddings = 131072`, i.e. 128K). The Gemma 4 model card markets the 12B at up to 256K — raise `--max-model-len` only if your vLLM build accepts it.\n\nTPU support is provided through [vLLM TPU](https://github.com/vllm-project/tpu-inference) with recipes for [Trillium](https://github.com/AI-Hypercomputer/tpu-recipes/tree/main/inference/trillium/vLLM/Gemma4) and [Ironwood](https://github.com/AI-Hypercomputer/tpu-recipes/blob/main/inference/ironwood/vLLM/Gemma4/).\n\n> **Nightly required**: support for the encoder-free 12B Unified model landed in [vllm-project/vllm#44429](https://github.com/vllm-project/vllm/pull/44429) and has not yet shipped in a stable release. Use the pinned Docker image (recommended) or a nightly pip wheel.\n\n## Prerequisites\n\n### Docker (recommended)\n```bash\ndocker pull vllm/vllm-openai:gemma4-unified            # NVIDIA (CUDA 13; append -cu129 for CUDA 12.9 hosts)\ndocker pull vllm/vllm-openai-xpu:latest                # Intel XPU (B60 / B70)\n```\nTPU images are published separately by [vllm-project/tpu-inference](https://github.com/vllm-project/tpu-inference); see the Trillium / Ironwood tpu-recipes below for the pinned tag.\n\n### pip (nightly, NVIDIA CUDA)\n```bash\nuv venv\nsource .venv/bin/activate\nuv pip install -U vllm --pre \\\n  --extra-index-url https://wheels.vllm.ai/nightly/cu129 \\\n  --extra-index-url https://download.pytorch.org/whl/cu129 \\\n  --index-strategy unsafe-best-match\n```\n\n## Deployment Configurations\n\n### Quick Start (Single GPU, BF16)\nThe 12B fits comfortably on a single 40 GB+ GPU.\n```bash\nvllm serve google/gemma-4-12B-it \\\n  --max-model-len 32768 \\\n  --gpu-memory-utilization 0.90\n```\n\n### Full-Featured Server Launch\nEnables text, image, audio, thinking, and tool calling:\n```bash\nvllm serve google/gemma-4-12B-it \\\n  --max-model-len 16384 \\\n  --gpu-memory-utilization 0.90 \\\n  --enable-auto-tool-choice \\\n  --reasoning-parser gemma4 \\\n  --tool-call-parser gemma4 \\\n  --chat-template examples/tool_chat_template_gemma4.jinja \\\n  --limit-mm-per-prompt '{\"image\": 4, \"audio\": 1}' \\\n  --async-scheduling \\\n  --host 0.0.0.0 \\\n  --port 8000\n```\n\n### Docker (NVIDIA)\n```bash\ndocker run -itd --name gemma4 \\\n  --ipc=host --network host --shm-size 16G --gpus all \\\n  -v ~/.cache/huggingface:/root/.cache/huggingface \\\n  vllm/vllm-openai:gemma4-unified \\\n    --model google/gemma-4-12B-it \\\n    --max-model-len 32768 \\\n    --gpu-memory-utilization 0.90 \\\n    --host 0.0.0.0 --port 8000\n```\nOn CUDA 12.9 hosts, use the `vllm/vllm-openai:gemma4-unified-cu129` tag instead.\n\n### Docker (Intel XPU B60 / B70, TP=2, Full-Featured)\n\nValidated on 2× Intel Arc Pro B60 / B70 with the official vLLM XPU image.\n\n```bash\ndocker run --device /dev/dri \\\n  -v /dev/dri/by-path:/dev/dri/by-path --shm-size=16g \\\n  --privileged --ipc=host -p 8000:8000 \\\n  -v ~/.cache/huggingface:/root/.cache/huggingface \\\n  vllm/vllm-openai-xpu:latest \\\n  google/gemma-4-12B-it \\\n  --enforce-eager \\\n  --attention-backend flash_attn \\\n  --host 0.0.0.0 \\\n  --port 8000 \\\n  --trust-remote-code \\\n  --enable-auto-tool-choice \\\n  --reasoning-parser gemma4 \\\n  --tool-call-parser gemma4 \\\n  --chat-template examples/tool_chat_template_gemma4.jinja \\\n  --limit-mm-per-prompt '{\"image\": 4, \"audio\": 1}' \\\n  --async-scheduling \\\n  --gpu-memory-utilization 0.9 \\\n  --no-enable-prefix-caching \\\n  --max-num-batched-tokens 8192 \\\n  --max-model-len 5120 \\\n  --tensor-parallel-size 2\n```\n\n### Docker (Cloud TPU — Trillium / Ironwood)\nTPU uses the separate `vllm/vllm-tpu` image (no pip wheel). Pull the tag specified by the upstream [Trillium](https://github.com/AI-Hypercomputer/tpu-recipes/tree/main/inference/trillium/vLLM/Gemma4) or [Ironwood](https://github.com/AI-Hypercomputer/tpu-recipes/blob/main/inference/ironwood/vLLM/Gemma4/) recipe, then run:\n```bash\ndocker run -itd --name gemma4-tpu \\\n  --privileged --network host --shm-size 16G \\\n  -v /dev/shm:/dev/shm -e HF_TOKEN=$HF_TOKEN \\\n  vllm/vllm-tpu:latest \\\n    --model google/gemma-4-12B-it \\\n    --max-model-len 16384 \\\n    --disable_chunked_mm_input \\\n    --host 0.0.0.0 --port 8000\n```\n\n## Client Usage\n\n### Text Generation\n```python\nfrom openai import OpenAI\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresponse = client.chat.completions.create(\n    model=\"google/gemma-4-12B-it\",\n    messages=[{\"role\": \"user\", \"content\": \"Write a poem about the ocean.\"}],\n    max_tokens=512, temperature=0.7,\n)\nprint(response.choices[0].message.content)\n```\n\n### Image Understanding\n```python\nresponse = client.chat.completions.create(\n    model=\"google/gemma-4-12B-it\",\n    messages=[{\"role\": \"user\", \"content\": [\n        {\"type\": \"image_url\", \"image_url\": {\"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg\"}},\n        {\"type\": \"text\", \"text\": \"Describe this image in detail.\"},\n    ]}],\n    max_tokens=1024,\n)\n```\n\n### Audio\nRequires `uv pip install \"vllm[audio]\"`.\n```bash\nvllm serve google/gemma-4-12B-it \\\n  --max-model-len 8192 \\\n  --limit-mm-per-prompt '{\"image\": 4, \"audio\": 1}'\n```\n\n### Thinking Mode\n```bash\nvllm serve google/gemma-4-12B-it \\\n  --max-model-len 16384 \\\n  --enable-auto-tool-choice \\\n  --reasoning-parser gemma4 \\\n  --tool-call-parser gemma4 \\\n  --chat-template examples/tool_chat_template_gemma4.jinja\n```\nEnable thinking per-request via `extra_body={\"chat_template_kwargs\": {\"enable_thinking\": True}}`, or default-on with `--default-chat-template-kwargs '{\"enable_thinking\": true}'`.\n\n### Structured Outputs\nvLLM guided decoding constrains output to a JSON schema. Include semantic instructions in the system prompt — the model does not see schema descriptions.\n\n## Configuration Tips\n\n- Set `--max-model-len` to match your workload.\n- `--gpu-memory-utilization 0.90–0.95` maximizes KV cache.\n- Image-only workloads: pass `--limit-mm-per-prompt.audio 0`.\n- Text-only workloads: pass `--limit-mm-per-prompt '{\"image\": 0, \"audio\": 0}'` to skip MM profiling.\n- `--async-scheduling` improves throughput.\n- FP8 KV cache (`--kv-cache-dtype fp8`) saves ~50% KV memory.\n\n## Speculative Decoding (MTP)\n\nEnable the **Spec Decoding** feature toggle (above) or add `--speculative-config` manually to use MTP drafting with the [assistant model](https://huggingface.co/google/gemma-4-12B-it-assistant). Recommended `num_speculative_tokens`: 4–8 for this model.\n\n> **Note:** MTP speculative decoding for Gemma 4 requires the nightly build — the pinned `vllm/vllm-openai:gemma4-unified` image or a nightly pip wheel. The standard `:latest` stable tag does not include this feature.\n\n## References\n\n- [Model card](https://huggingface.co/google/gemma-4-12B-it)\n- [vLLM support PR (#44429)](https://github.com/vllm-project/vllm/pull/44429)\n- [Assistant (MTP draft) model](https://huggingface.co/google/gemma-4-12B-it-assistant)\n- [Gemma docs](https://ai.google.dev/gemma/docs)\n- [vLLM Gemma 4 tool-call template](https://github.com/vllm-project/vllm/blob/main/examples/tool_chat_template_gemma4.jinja)\n- [TPU recipes: Trillium](https://github.com/AI-Hypercomputer/tpu-recipes/tree/main/inference/trillium/vLLM/Gemma4)\n- [TPU recipes: Ironwood](https://github.com/AI-Hypercomputer/tpu-recipes/blob/main/inference/ironwood/vLLM/Gemma4/)\n",
  "source": {
    "source": "vllm",
    "repo": "vllm-project/vllm",
    "ref": "main",
    "sha": "389ffebe555705c85337025d5df108d0384d365e",
    "path": "recipes/models/Google/gemma-4-12B-it.yaml",
    "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/Google/gemma-4-12B-it.yaml",
    "overlays": []
  }
}