{ "hf_id": "bharatgenai/Param2-17B-A2.4B-Thinking", "meta": { "title": "Param-2-17B-A2.4B-Thinking", "slug": "param2-17b-a2-4b-thinking", "provider": "BharatGen", "description": "BharatGen's hybrid-MoE reasoning model for English, Hindi and 21 Indian languages, with thinking traces and Hermes-style tool calling", "date_added": "2026-09-20", "date_updated": "2026-09-20", "tasks": [ "text" ], "performance_headline": "17B total / 2.4B active parameters; the full BF16 checkpoint serves at TP1 on a single H100 or a single 48 GB L40S", "related_recipes": [], "default_hardware": "h100", "hardware": { "h100": "verified", "l40s": "verified" } }, "recommended_command": { "hardware": "h100", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": {}, "docker_image": "vllm/vllm-openai:latest", "command": "vllm serve bharatgenai/Param2-17B-A2.4B-Thinking \\\n --trust-remote-code \\\n --async-scheduling \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser hermes \\\n --reasoning-parser qwen3", "argv": [ "vllm", "serve", "bharatgenai/Param2-17B-A2.4B-Thinking", "--trust-remote-code", "--async-scheduling", "--tensor-parallel-size", "1", "--enable-auto-tool-choice", "--tool-call-parser", "hermes", "--reasoning-parser", "qwen3" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:latest bharatgenai/Param2-17B-A2.4B-Thinking \\\n --trust-remote-code \\\n --async-scheduling \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser hermes \\\n --reasoning-parser qwen3", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "vllm/vllm-openai:latest", "bharatgenai/Param2-17B-A2.4B-Thinking", "--trust-remote-code", "--async-scheduling", "--tensor-parallel-size", "1", "--enable-auto-tool-choice", "--tool-call-parser", "hermes", "--reasoning-parser", "qwen3" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H100", "description": "NVIDIA H100 80GB NVLink (8-GPU node)", "gpu_count": 8, "vram_gb": 640, "multi_node": false }, "alternatives": { "kv_store_centralized_mooncake": "/bharatgenai/Param2-17B-A2.4B-Thinking/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/bharatgenai/Param2-17B-A2.4B-Thinking/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h100": "/bharatgenai/Param2-17B-A2.4B-Thinking/hw/h100.json", "h200": "/bharatgenai/Param2-17B-A2.4B-Thinking/hw/h200.json", "b200": "/bharatgenai/Param2-17B-A2.4B-Thinking/hw/b200.json", "gb200": "/bharatgenai/Param2-17B-A2.4B-Thinking/hw/gb200.json", "b300": "/bharatgenai/Param2-17B-A2.4B-Thinking/hw/b300.json", "gb300": "/bharatgenai/Param2-17B-A2.4B-Thinking/hw/gb300.json", "l40s": "/bharatgenai/Param2-17B-A2.4B-Thinking/hw/l40s.json", "mi300x": "/bharatgenai/Param2-17B-A2.4B-Thinking/hw/mi300x.json", "mi325x": "/bharatgenai/Param2-17B-A2.4B-Thinking/hw/mi325x.json", "mi350x": "/bharatgenai/Param2-17B-A2.4B-Thinking/hw/mi350x.json", "mi355x": "/bharatgenai/Param2-17B-A2.4B-Thinking/hw/mi355x.json" } }, "model": { "model_id": "bharatgenai/Param2-17B-A2.4B-Thinking", "min_vllm_version": "0.20.0", "architecture": "moe", "parameter_count": "17B", "active_parameters": "2.4B", "context_length": 32768, "base_args": [ "--trust-remote-code", "--async-scheduling" ], "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:latest" } } }, "features": { "tool_calling": { "description": "Parse the model's JSON blocks into OpenAI tool calls with the Hermes parser", "args": [ "--enable-auto-tool-choice", "--tool-call-parser", "hermes" ] }, "reasoning": { "description": "Split the model's traces into the reasoning field with the Qwen3 parser", "args": [ "--reasoning-parser", "qwen3" ] } }, "opt_in_features": [], "variants": { "default": { "precision": "bf16", "vram_minimum_gb": 41, "tp": 1, "description": "Official BF16 checkpoint; approximately 34.3 GB of repository weights, served with TP1" } }, "hardware_overrides": {}, "guide": "## Overview\n\n`bharatgenai/Param2-17B-A2.4B-Thinking` is BharatGen's early post-training\ncheckpoint of Param-2-17B, a hybrid Mixture-of-Experts model with **17B total\nparameters** and **2.4B activated per token**. Each sparse block routes to 6\nof 64 experts and additionally keeps **2 shared experts** always active; the\nfirst layer is dense (`first_k_dense_replace: 1`) before the MoE stack begins.\nThe shared-expert design is aimed at stable cross-lingual representations for\nIndian languages.\n\nBeyond English and Hindi it covers 21 scheduled Indian languages — Assamese,\nBengali, Bodo, Dogri, Gujarati, Kannada, Konkani, Kashmiri, Maithili,\nMalayalam, Manipuri, Marathi, Nepali, Oriya, Punjabi, Sanskrit, Santali,\nSindhi, Tamil, Telugu and Urdu — and was pretrained on roughly 22.6T tokens.\n\nThe checkpoint is a **thinking** model: its chat template always opens an\nassistant turn with a `` block, and tool calls are emitted as\nHermes-style `{\"name\": ..., \"arguments\": ...}` JSON.\nThat is why the two Features above map onto the `qwen3` reasoning parser and\nthe `hermes` tool-call parser.\n\n## Prerequisites\n\n- **vLLM:** 0.20.0 or newer. `Param2MoEForCausalLM` was registered upstream in\n [PR #38000](https://github.com/vllm-project/vllm/pull/38000) and the\n tensor-parallel attention-head fix landed in\n [PR #39707](https://github.com/vllm-project/vllm/pull/39707); both ship in\n v0.20.0. Nothing extra needs to be installed.\n- **`--trust-remote-code` is required.** vLLM implements the model natively\n (`vllm/model_executor/models/param2moe.py`), but `model_type: param2moe` is\n not registered with Transformers, so the config still loads from the repo's\n `configuration_param2moe.py`.\n- **Hardware:** one GPU is enough. The BF16 weights are ~34.3 GB, so an H100\n 80GB leaves ample room for the 32,768-token KV cache. A single 48 GB\n **L40S** also runs the same configuration: at the default\n `--gpu-memory-utilization 0.9` roughly 8–9 GB remains for KV, and this\n model's KV cache is small (21 layers × 8 KV heads × 64 head dim ≈ 42 KB per\n token), so about 6 concurrent sequences fit at the full 32,768-token\n window — more if you cap `--max-model-len`.\n- **License:** the checkpoint is released under the BharatGen\n **Non-Commercial** license — review it before deploying.\n\n## Launching the Server\n\nThis is the exact configuration verified on both H100 and L40S:\n\n```bash\nvllm serve bharatgenai/Param2-17B-A2.4B-Thinking \\\n --trust-remote-code \\\n --async-scheduling \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser hermes \\\n --reasoning-parser qwen3\n```\n\nTo pin the server to one specific GPU on a multi-GPU node, prefix the command\nwith `CUDA_VISIBLE_DEVICES=`; add `--port ` if 8000 is taken.\n\n`--async-scheduling` overlaps scheduling with the model forward pass and cuts\nhost overhead between decode steps. It is not required — drop it if you hit an\nincompatibility with another feature you enable.\n\nTP1 is the recommended layout: the model fits on one GPU and a 2.4B\nactive-parameter forward pass gains little from sharding. To raise aggregate\nthroughput, run several independent TP1 replicas behind a load balancer rather\nthan increasing TP — especially on L40S, which has no NVLink, so cross-GPU TP\nwould fall back to PCIe.\n\n## Client Usage\n\nReasoning traces arrive in `message.reasoning` once `--reasoning-parser qwen3`\nis active:\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresponse = client.chat.completions.create(\n model=\"bharatgenai/Param2-17B-A2.4B-Thinking\",\n messages=[\n {\"role\": \"user\", \"content\": \"एक कक्षा में 30 छात्र हैं। पूरी कक्षा का औसत 70 है। शीर्ष 10 का औसत 85 और निचले 10 का औसत 50 है। मध्य 10 छात्रों का औसत क्या है?\"}\n ],\n temperature=0,\n)\nmsg = response.choices[0].message\nprint(msg.reasoning)\nprint(msg.content)\n```\n\nThe model card recommends **deterministic decoding** (`temperature=0`) for\nevaluations, structured tasks and production use; sampling defaults of\n`temperature=0.7`, `top_p=0.9`, `top_k=50` are suggested for open-ended\ngeneration. The card also notes the model is strong at structured output\n(Markdown, tables, JSON), and that clear task/format/constraint instructions\nin the prompt matter noticeably.\n\n### Skipping the thinking block\n\nThe chat template accepts `enable_thinking`. Setting it to `false` prefills an\nempty `` block so the model answers directly:\n\n```python\nresponse = client.chat.completions.create(\n model=\"bharatgenai/Param2-17B-A2.4B-Thinking\",\n messages=[{\"role\": \"user\", \"content\": \"Name the capital of Maharashtra.\"}],\n extra_body={\"chat_template_kwargs\": {\"enable_thinking\": False}},\n)\n```\n\n### Tool calling\n\nWith the Tool Calling feature on, pass `tools` as usual and read\n`message.tool_calls`:\n\n```python\nresponse = client.chat.completions.create(\n model=\"bharatgenai/Param2-17B-A2.4B-Thinking\",\n messages=[{\"role\": \"user\", \"content\": \"What is the weather in Pune?\"}],\n tools=[{\n \"type\": \"function\",\n \"function\": {\n \"name\": \"get_weather\",\n \"description\": \"Get the current weather for a city\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\"city\": {\"type\": \"string\"}},\n \"required\": [\"city\"],\n },\n },\n }],\n tool_choice=\"auto\",\n)\nprint(response.choices[0].message.tool_calls)\n```\n\n## Benchmarking\n\n```bash\nvllm bench serve \\\n --model bharatgenai/Param2-17B-A2.4B-Thinking \\\n --dataset-name random \\\n --random-input-len 2048 \\\n --random-output-len 1024 \\\n --num-prompts 200\n```\n\nBecause thinking traces count toward the output budget, keep\n`--random-output-len` generous when benchmarking with reasoning enabled, or\nbenchmark with `enable_thinking: false` to measure the non-reasoning path.\n\n## Notes and Limitations\n\n- **Context:** the native window is 32,768 tokens. The model card states a\n 128k-context variant is planned but not yet released, so do not raise\n `--max-model-len` past 32,768 on this checkpoint.\n- **No MTP head:** `num_nextn_predict_layers: 0`, so there is no\n speculative-decoding draft shipped with this checkpoint.\n- **Early post-training checkpoint:** BharatGen describes the release as an\n early post-training snapshot intended for downstream use and further\n fine-tuning.\n- **Verification scope:** the `verified` H100 and L40S badges cover the BF16\n checkpoint at TP1 with the launch command above. Other GPUs are untested\n rather than known-bad.\n- **L40S headroom:** at 48 GB the weights alone take ~34.3 GB, so KV cache is\n the binding constraint rather than weight capacity. Lower\n `--max-model-len` if you need more concurrency than the full context window\n allows.\n\n## References\n\n- [Param2-17B-A2.4B-Thinking on Hugging Face](https://huggingface.co/bharatgenai/Param2-17B-A2.4B-Thinking)\n- [BharatGen on Hugging Face](https://huggingface.co/bharatgenai)\n- [vLLM support PR #38000 — Add support for BharatGen's Param2MoE model](https://github.com/vllm-project/vllm/pull/38000)\n- [vLLM PR #39707 — Fix attention-head mismatch under tensor parallelism for param2moe](https://github.com/vllm-project/vllm/pull/39707)\n- [vLLM supported models — `Param2MoEForCausalLM`](https://docs.vllm.ai/en/latest/models/supported_models.html)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "389ffebe555705c85337025d5df108d0384d365e", "path": "recipes/models/bharatgenai/Param2-17B-A2.4B-Thinking.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/bharatgenai/Param2-17B-A2.4B-Thinking.yaml", "overlays": [] } }