{ "hf_id": "Google/diffusiongemma-26B-A4B-it", "meta": { "title": "DiffusionGemma 26B-A4B IT", "slug": "diffusiongemma-26b-a4b-it", "provider": "Google", "description": "Google's DiffusionGemma — a block-diffusion language model built on Gemma 4's MoE backbone (26B total / 4B active). Generates tokens via iterative denoising over a fixed-length canvas rather than left-to-right autoregressive decoding, enabling higher throughput with parallel block generation.", "date_added": "2026-06-11", "date_updated": "2026-09-23", "tasks": [ "multimodal", "text" ], "performance_headline": "Block-diffusion MoE — 26B total / 4B active, canvas-based parallel generation with ~1.9x throughput vs autoregressive baseline", "related_recipes": [ "google/gemma-4-26B-A4B-it", "google/gemma-4-31B-it" ], "hardware": { "h100": "verified", "h200": "verified", "dgx_spark_gb10": "verified", "xeon6": "verified", "arc_pro_b60": "verified", "arc_pro_b70": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": {}, "docker_image": "vllm/vllm-openai:gemma", "command": "vllm serve google/diffusiongemma-26B-A4B-it \\\n --max-num-seqs 4 \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser gemma4 \\\n --chat-template examples/tool_chat_template_gemma4.jinja \\\n --reasoning-parser gemma4", "argv": [ "vllm", "serve", "google/diffusiongemma-26B-A4B-it", "--max-num-seqs", "4", "--tensor-parallel-size", "1", "--enable-auto-tool-choice", "--tool-call-parser", "gemma4", "--chat-template", "examples/tool_chat_template_gemma4.jinja", "--reasoning-parser", "gemma4" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:gemma google/diffusiongemma-26B-A4B-it \\\n --max-num-seqs 4 \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser gemma4 \\\n --chat-template examples/tool_chat_template_gemma4.jinja \\\n --reasoning-parser gemma4", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "vllm/vllm-openai:gemma", "google/diffusiongemma-26B-A4B-it", "--max-num-seqs", "4", "--tensor-parallel-size", "1", "--enable-auto-tool-choice", "--tool-call-parser", "gemma4", "--chat-template", "examples/tool_chat_template_gemma4.jinja", "--reasoning-parser", "gemma4" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "kv_store_centralized_mooncake": "/Google/diffusiongemma-26B-A4B-it/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/Google/diffusiongemma-26B-A4B-it/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/Google/diffusiongemma-26B-A4B-it/hw/h200.json", "h100": "/Google/diffusiongemma-26B-A4B-it/hw/h100.json", "dgx_spark_gb10": "/Google/diffusiongemma-26B-A4B-it/hw/dgx_spark_gb10.json", "xeon6": "/Google/diffusiongemma-26B-A4B-it/hw/xeon6.json" } }, "model": { "model_id": "google/diffusiongemma-26B-A4B-it", "min_vllm_version": "0.24.0", "docker_image": "vllm/vllm-openai:gemma", "architecture": "moe", "parameter_count": "26B", "active_parameters": "4B", "context_length": 262144, "base_args": [ "--max-num-seqs", "4" ], "base_env": {}, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:gemma" } } }, "features": { "tool_calling": { "description": "Enable automatic tool choice with Gemma 4 parser and chat template", "args": [ "--enable-auto-tool-choice", "--tool-call-parser", "gemma4", "--chat-template", "examples/tool_chat_template_gemma4.jinja" ] }, "reasoning": { "description": "Enable structured thinking/reasoning output", "args": [ "--reasoning-parser", "gemma4" ] } }, "opt_in_features": [], "variants": { "default": { "precision": "bf16", "vram_minimum_gb": 64, "description": "Full BF16 — single 80 GB NVIDIA GPU (H100/H200). Requires --max-num-seqs 4 to avoid OOM from diffusion state buffers.", "supported_hardware": [ "h100", "h200", "dgx_spark_gb10", "xeon6" ] }, "fp8": { "model_id": "RedHatAI/diffusiongemma-26B-A4B-it-FP8-dynamic", "precision": "fp8", "vram_minimum_gb": 32, "description": "FP8 (E4M3) weights with dynamic per-token activation quantization", "supported_hardware": [ "h100", "h200", "dgx_spark_gb10" ], "json": "/RedHatAI/diffusiongemma-26B-A4B-it-FP8-dynamic.json" }, "nvfp4": { "model_id": "RedHatAI/diffusiongemma-26B-A4B-it-NVFP4", "precision": "nvfp4", "label": "NVFP4 - RedHatAI", "vram_minimum_gb": 24, "description": "NVFP4 (4-bit) quantized weights", "supported_hardware": [ "dgx_spark_gb10" ], "json": "/RedHatAI/diffusiongemma-26B-A4B-it-NVFP4.json" }, "nvidia_nvfp4": { "model_id": "nvidia/diffusiongemma-26B-A4B-it-NVFP4", "precision": "nvfp4", "label": "NVFP4 - NVIDIA", "vram_minimum_gb": 24, "description": "NVIDIA NVFP4 (4-bit) quantized weights", "supported_hardware": [ "dgx_spark_gb10" ], "hardware_overrides": { "dgx_spark_gb10": { "extra_args": [ "--gpu-memory-utilization", "0.8", "--max-num-seqs", "8", "--enable-auto-tool-choice", "--tool-call-parser", "gemma4", "--load-format", "fastsafetensors", "--attention-backend", "TRITON_ATTN", "--enable-prefix-caching", "--diffusion-config", "{\"canvas_length\":256}", "--override-generation-config", "{\"max_new_tokens\": null}", "--reasoning-parser", "gemma4", "--default-chat-template-kwargs", "{\"enable_thinking\": true}" ] } }, "json": "/nvidia/diffusiongemma-26B-A4B-it-NVFP4.json" } }, "hardware_overrides": { "cpu": {}, "xpu": { "extra_args": [ "--enforce-eager", "--attention-backend", "TRITON_ATTN", "--tensor-parallel-size", "4" ] } }, "guide": "## Overview\n\n[DiffusionGemma 26B-A4B](https://huggingface.co/google/diffusiongemma-26B-A4B-it) is a block-diffusion language model built on Gemma 4's MoE backbone. Instead of generating tokens one at a time (autoregressive), it generates blocks of 256 tokens simultaneously via iterative denoising over a fixed canvas — trading higher time-to-first-token for significantly higher per-request generation throughput.\n\n### Key Architecture Features\n- **Block Diffusion**: Generates tokens in 256-token canvas blocks via iterative denoising (up to 48 steps per block).\n- **MoE Backbone**: Same Gemma 4 architecture — 128 fine-grained experts with top-8 routing, 26B total / 4B active parameters.\n- **Entropy-Bound Sampling**: Uses an entropy-based sampler (`diffusion_sampler: entropy_bound`, `entropy_bound: 0.1`) for the denoising process.\n- **Multimodal**: Supports text + images via the Gemma 4 vision encoder.\n- **Thinking Mode**: Supports structured reasoning via `<|channel>thought\\n...` delimiters.\n- **Function Calling**: Supports Gemma 4's tool-call protocol (works best in thinking mode).\n\n### Important Deployment Notes\n\nDiffusionGemma requires several specific flags that differ from standard Gemma 4:\n\n| Flag | Why |\n|------|-----|\n| `--max-num-seqs 4` | The diffusion state buffers (`self_conditioning_probs`) pre-allocate `max_seqs × canvas_length × vocab_size` tensors. With Gemma's 262K vocab and canvas_length=256, higher values cause OOM. |\n| `--generation-config vllm` | The checkpoint's `generation_config.json` sets `max_tokens: 256`, which would override per-request limits. This flag ignores it. |\n| `--gpu-memory-utilization 0.85` | Leaves headroom for activation memory during denoising. |\n| `--hf-overrides '{\"diffusion_sampler\":\"entropy_bound\",\"diffusion_entropy_bound\":0.1}'` | Configures the entropy-bound denoising sampler. |\n| `--diffusion-config '{\"canvas_length\": 256}'` | Sets the canvas block size for generation. |\n\n## Prerequisites\n\nDiffusionGemma requires a vLLM build with diffusion model support, available in the\nGemma docker image:\n\n```bash\ndocker pull vllm/vllm-openai:gemma\n```\n\n## Deployment\n\n### Single GPU (H100/H200, BF16)\n\n```bash\ndocker run -itd --name diffusiongemma \\\n --ipc=host --network host --gpus all \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:gemma \\\n --model google/diffusiongemma-26B-A4B-it \\\n --max-model-len 262144 \\\n --max-num-seqs 4 \\\n --gpu-memory-utilization 0.85 \\\n --host 0.0.0.0 --port 8000\n```\n\n### Intel Xeon 6 (CPU, BF16)\n\nDiffusionGemma runs on Intel Xeon 6 CPUs using vLLM's CPU backend with AMX BF16 acceleration.\nThe generated example uses TP=1. Adjust `--tensor-parallel-size` based on the\ndetected NUMA topology and deployment requirements.\n\n```bash\ndocker run \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai-cpu:latest-x86_64 google/diffusiongemma-26B-A4B-it \\\n --max-num-seqs 4 \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser gemma4 \\\n --chat-template examples/tool_chat_template_gemma4.jinja \\\n --reasoning-parser gemma4\n```\n\n\n### Intel XPU (B60 / B70, BF16)\n\nValidated on 4x Intel Arc Pro B60 / B70 (B60 24 GB, B70 32 GB per card)\nwith the official vLLM XPU image `vllm/vllm-openai-xpu:latest`. `--enforce-eager`\nand `--attention-backend TRITON_ATTN` are required on this backend. Select Arc Pro\nB60/B70 in the hardware picker above for the exact command.\n\n### Full-Featured Server (text + image + thinking + tools)\n\n```bash\ndocker run -itd --name diffusiongemma \\\n --ipc=host --network host --gpus all \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:gemma \\\n --model google/diffusiongemma-26B-A4B-it \\\n --max-model-len 262144 \\\n --max-num-seqs 4 \\\n --gpu-memory-utilization 0.85 \\\n --mm-processor-kwargs '{\"max_soft_tokens\": 1120}' \\\n --limit-mm-per-prompt '{\"image\": 7}' \\\n --enable-auto-tool-choice \\\n --tool-call-parser gemma4 \\\n --reasoning-parser gemma4 \\\n --host 0.0.0.0 --port 8000\n```\n\n### Jev-Style Structured Decisions (Nightly)\n\nvLLM's structured-read support for DiffusionGemma can answer yes/no, multiple-choice,\nand scored questions with a confidence value. The example server seeds a canvas with\nthe response template, leaves only the answer slots noisy, and reads a probability\ndistribution from every slot after one denoising step. It exposes the Jev-compatible\n`/v1/systemone` endpoint; this is an example server rather than a standard vLLM endpoint.\n\nUse the pinned nightly image that contains the merged structured-read support:\n\n```bash\ndocker run -d --name dgemma \\\n --gpus '\"device=0\"' \\\n --ipc=host \\\n -p 8000:8000 \\\n -p 8011:8011 \\\n -v \"$HOME/.cache/huggingface:/root/.cache/huggingface\" \\\n vllm/vllm-openai:nightly-e9757321527ca1ecd514c07c1418dd2c53da3d19 \\\n google/diffusiongemma-26B-A4B-it \\\n --served-model-name dgemma \\\n --diffusion-config '{\"canvas_length\":64}' \\\n --max-logprobs 32 \\\n --enable-prefix-caching\n```\n\nWait for the vLLM server to become healthy, then start the example server in the same\ncontainer:\n\n```bash\nuntil curl -fsS localhost:8000/health >/dev/null; do sleep 2; done\n\ndocker exec -d dgemma \\\n python /vllm-workspace/examples/features/structured_diffusion/structured_server.py \\\n --upstream http://127.0.0.1:8000 \\\n --tokenizer google/diffusiongemma-26B-A4B-it \\\n --canvas 64\n```\n\nAfter the example server becomes healthy, send a Jev-style decision request:\n\n```bash\nuntil curl -fsS localhost:8011/health >/dev/null; do sleep 2; done\n\ncurl -sS localhost:8011/v1/systemone \\\n -H 'content-type: application/json' \\\n -d '{\n \"model\": \"jev-latest\",\n \"state\": {\"ticket\": \"Everything is down, demo at noon.\"},\n \"questions\": {\n \"urgent\": {\n \"type\": \"noul\",\n \"instructions\": \"Needs a reply within the hour?\"\n }\n }\n }'\n```\n\n## Client Usage\n\n### Text Generation\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresponse = client.chat.completions.create(\n model=\"google/diffusiongemma-26B-A4B-it\",\n messages=[{\"role\": \"user\", \"content\": \"Write a poem about the ocean.\"}],\n max_tokens=512,\n temperature=0.7,\n)\nprint(response.choices[0].message.content)\n```\n\n### Offline Inference\n\n```python\nfrom vllm import LLM, SamplingParams\nfrom transformers import AutoTokenizer\n\nmodel_path = \"google/diffusiongemma-26B-A4B-it\"\ntokenizer = AutoTokenizer.from_pretrained(model_path)\n\nllm = LLM(\n model=model_path,\n max_num_seqs=4,\n gpu_memory_utilization=0.85,\n hf_overrides={\n \"diffusion_sampler\": \"entropy_bound\",\n \"diffusion_entropy_bound\": 0.1,\n },\n diffusion_config={\"canvas_length\": 256},\n)\n\nmessages = [{\"role\": \"user\", \"content\": \"Explain quantum computing in simple terms.\"}]\nprompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\noutputs = llm.generate(prompt, SamplingParams(temperature=0.0, max_tokens=1024))\n\nprint(outputs[0].outputs[0].text)\n```\n\n### Thinking Mode\n\nLaunch the server with `--reasoning-parser gemma4`, then enable per-request:\n\n```python\nresponse = client.chat.completions.create(\n model=\"google/diffusiongemma-26B-A4B-it\",\n messages=[{\"role\": \"user\", \"content\": \"What is the derivative of x^3 * ln(x)?\"}],\n max_tokens=32768,\n extra_body={\"chat_template_kwargs\": {\"enable_thinking\": True}},\n)\n```\n\n## Performance Characteristics\n\nCompared to the autoregressive Gemma 4 26B-A4B baseline (single H100, concurrency=1, SPEED-Bench):\n\n| Metric | Gemma 4 26B-A4B (baseline) | DiffusionGemma 26B-A4B |\n|--------|---------------------------|------------------------|\n| Output TPS | 199 tok/s | 375 tok/s (**1.9×**) |\n| E2E Request Time (mean) | 2.87s | 0.88s (**3.3× faster**) |\n| TTFT (mean) | 53ms | 489ms (higher due to canvas denoising setup) |\n| Per-request Gen TPS (mean) | 205 tok/s | 1,282 tok/s (**6.2×**) |\n\nThe diffusion model trades higher time-to-first-token for significantly higher generation throughput. It generates fewer total tokens per prompt on average (shorter outputs) but completes requests much faster.\n\n## Known Limitations\n\n- **TTFT**: ~10× higher than autoregressive baseline — the model must denoise an entire canvas block before emitting the first token.\n- **`--max-num-seqs`**: Must be kept low (≤4) due to the large diffusion state tensors. Higher values cause CUDA OOM.\n- **Audio**: Not supported — the diffusion checkpoints do not include an audio encoder.\n- **Function Calling**: Works best in thinking mode. Non-thinking mode may answer directly instead of emitting tool calls.\n\n## References\n\n- [Model card](https://huggingface.co/google/diffusiongemma-26B-A4B-it)\n- [vLLM structured-read support for DiffusionGemma](https://github.com/vllm-project/vllm/pull/57250)\n- [djev-run Cloud Run deployment](https://github.com/taeold/djev-run)\n- [Gemma 4 26B-A4B (autoregressive baseline)](https://huggingface.co/google/gemma-4-26B-A4B-it)\n- [Gemma docs](https://ai.google.dev/gemma/docs)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "389ffebe555705c85337025d5df108d0384d365e", "path": "recipes/models/Google/diffusiongemma-26B-A4B-it.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/Google/diffusiongemma-26B-A4B-it.yaml", "overlays": [] } }