1:"$Sreact.fragment"
2:I[6529,["619","static/chunks/619-ba102abea3e3d0e4.js","177","static/chunks/app/layout-13fa3fd02f6eb5db.js"],"default"]
3:I[9766,[],""]
4:I[8924,[],""]
5:I[2619,["619","static/chunks/619-ba102abea3e3d0e4.js","953","static/chunks/app/blog/%5Bslug%5D/page-f25a122e9ccf798d.js"],""]
d:I[7150,[],""]
:HL["/_next/static/css/1d1f6bc532e5f43f.css","style"]
:HL["/_next/static/css/eb87e4f7aea490c6.css","style"]
0:{"P":null,"b":"DQJo8iKQxHubJM4JvsRbH","p":"","c":["","blog","training-gym",""],"i":false,"f":[[["",{"children":["blog",{"children":[["slug","training-gym","d"],{"children":["__PAGE__",{}]}]}]},"$undefined","$undefined",true],["",["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/1d1f6bc532e5f43f.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":[["$","head",null,{"children":[["$","link",null,{"rel":"icon","type":"image/svg+xml","href":"/favicon.svg"}],["$","link",null,{"rel":"alternate icon","href":"/favicon.png"}]]}],["$","body",null,{"children":[["$","$L2",null,{}],["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}],["$","footer",null,{"children":["$","div",null,{"className":"container","children":[["$","div",null,{"className":"footer-content","children":[["$","div",null,{"className":"footer-section","children":[["$","h4",null,{"children":"Zen LM"}],["$","p",null,{"children":"95 open Zen models across Zen3, Zen4, and Zen5. Chat, code, vision, audio, image, embeddings, rerankers, and safety. OpenAI- and Anthropic-compatible API."}]]}],["$","div",null,{"className":"footer-section","children":[["$","h4",null,{"children":"Zen 5"}],["$","ul",null,{"children":[["$","li",null,{"children":["$","$L5",null,{"href":"/models#zen5","children":"Zen5 Nano (0.8B - 9B)"}]}],["$","li",null,{"children":["$","$L5",null,{"href":"/models#zen5","children":"Zen5 Flash"}]}],["$","li",null,{"children":["$","$L5",null,{"href":"/models#zen5","children":"Zen5 Mini"}]}],["$","li",null,{"children":["$","$L5",null,{"href":"/models#zen5","children":"Zen5 (default)"}]}],["$","li",null,{"children":["$","$L5",null,{"href":"/models#zen5","children":"Zen5 Coder"}]}],["$","li",null,{"children":["$","$L5",null,{"href":"/models#zen5","children":"Zen5 Pro"}]}],["$","li",null,{"children":["$","$L5",null,{"href":"/models#zen5","children":"Zen5 Max"}]}]]}]]}],["$","div",null,{"className":"footer-section","children":[["$","h4",null,{"children":"Zen 4"}],["$","ul",null,{"children":[["$","li",null,{"children":["$","$L5",null,{"href":"/models#zen4","children":"Zen4 / Zen4.1"}]}],["$","li",null,{"children":["$","$L5",null,{"href":"/models#zen4","children":"Zen4 Ultra / Max / Pro"}]}],["$","li",null,{"children":["$","$L5",null,{"href":"/models#zen4","children":"Zen4 Mini / Thinking"}]}],["$","li",null,{"children":["$","$L5",null,{"href":"/models#zen4","children":"Zen4 Coder / Pro / Flash"}]}]]}]]}],["$","div",null,{"className":"footer-section","children":[["$","h4",null,{"children":"Zen 3 Multimodal"}],["$","ul",null,{"children":[["$","li",null,{"children":["$","$L5",null,{"href":"/models#zen3","children":"Zen3 Omni / VL / Web"}]}],["$","li",null,{"children":["$","$L5",null,{"href":"/models#zen3","children":"Zen3 Nano / Guard"}]}],["$","li",null,{"children":["$","$L5",null,{"href":"/models#zen3","children":"Zen3 Embedding / Reranker"}]}],["$","li",null,{"children":["$","$L5",null,{"href":"/models#zen3","children":"Zen3 Image / ASR / TTS"}]}]]}]]}],["$","div",null,{"className":"footer-section","children":[["$","h4",null,{"children":"Resources"}],["$","ul",null,{"children":[["$","li",null,{"children":["$","$L5",null,{"href":"/datasets","children":"Training Data"}]}],["$","li",null,{"children":["$","a",null,{"href":"https://huggingface.co/zenlm","target":"_blank","rel":"noopener noreferrer","children":"HuggingFace"}]}],["$","li",null,{"children":["$","a",null,{"href":"https://github.com/zenlm","target":"_blank","rel":"noopener noreferrer","children":"GitHub"}]}],["$","li",null,{"children":["$","$L5",null,{"href":"/research","children":"Research Papers"}]}],["$","li",null,{"children":["$","a",null,{"href":"https://api.hanzo.ai","target":"_blank","rel":"noopener noreferrer","children":"Zen API"}]}]]}]]}]]}],"$L6"]}]}],"$L7","$L8"]}]]}]]}],{"children":["blog","$L9",{"children":[["slug","training-gym","d"],"$La",{"children":["__PAGE__","$Lb",{},null,false]},null,false]},null,false]},null,false],"$Lc",false]],"m":"$undefined","G":["$d",[]],"s":false,"S":true}
e:I[7405,["619","static/chunks/619-ba102abea3e3d0e4.js","177","static/chunks/app/layout-13fa3fd02f6eb5db.js"],"default"]
10:I[4431,[],"OutletBoundary"]
12:I[5278,[],"AsyncMetadataOutlet"]
14:I[4431,[],"ViewportBoundary"]
16:I[4431,[],"MetadataBoundary"]
17:"$Sreact.suspense"
6:["$","div",null,{"className":"footer-bottom","children":["$","p",null,{"children":["© ",2026," Zen Authors. Open foundation models. Served on the Zen API."]}]}]
7:["$","$Le",null,{}]
8:["$","script",null,{"src":"/assets/js/main.js","async":true}]
9:["$","$1","c",{"children":[null,["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}]
a:["$","$1","c",{"children":[null,["$","$L3",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L4",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}]
b:["$","$1","c",{"children":["$Lf",[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/eb87e4f7aea490c6.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}]],["$","$L10",null,{"children":["$L11",["$","$L12",null,{"promise":"$@13"}]]}]]}]
c:["$","$1","h",{"children":[null,[["$","$L14",null,{"children":"$L15"}],null],["$","$L16",null,{"children":["$","div",null,{"hidden":true,"children":["$","$17",null,{"fallback":null,"children":"$L18"}]}]}]]}]
19:T202f,<p>Training large language models requires more than algorithms. It requires infrastructure: distributed training frameworks, data pipelines, experiment tracking, and evaluation harnesses. Today we open source Training Gym, our complete platform for model development.</p>
<h2 id="why-training-gym">Why Training Gym?</h2>
<p>Open AI development faces an infrastructure gap. Publishing model weights is valuable, but it's not enough. Researchers need:</p>
<ul>
<li>Reproducible training pipelines</li>
<li>Scalable distributed training</li>
<li>Standardized evaluation</li>
<li>Experiment management</li>
<li>Data processing tools</li>
</ul>
<p>Training Gym provides all of this in an integrated, open source package.</p>
<h2 id="architecture">Architecture</h2>
<pre><code>+------------------+     +------------------+     +------------------+
|   Data Pipeline  | --> |  Training Loop   | --> |   Evaluation     |
+------------------+     +------------------+     +------------------+
         |                       |                        |
         v                       v                        v
+------------------+     +------------------+     +------------------+
|   Data Registry  |     | Checkpoint Store |     |   Metrics DB     |
+------------------+     +------------------+     +------------------+
                                |
                                v
                    +------------------+
                    |   Experiment     |
                    |   Tracker        |
                    +------------------+
</code></pre>
<h3 id="data-pipeline">Data Pipeline</h3>
<p>The data pipeline handles:</p>
<ul>
<li><strong>Ingestion</strong>: Load from local files, cloud storage, or streaming sources</li>
<li><strong>Processing</strong>: Tokenization, filtering, deduplication</li>
<li><strong>Mixing</strong>: Combine multiple data sources with configurable ratios</li>
<li><strong>Streaming</strong>: Memory-efficient data loading for large corpora</li>
</ul>
<pre><code class="language-python">from training_gym.data import DataPipeline, MixedDataset

pipeline = DataPipeline(
    sources=[
        ("s3://data/books", 0.3),
        ("s3://data/web", 0.5),
        ("s3://data/code", 0.2),
    ],
    tokenizer="zoo-labs/zen-tokenizer",
    sequence_length=2048,
)

dataset = pipeline.build()
</code></pre>
<h3 id="distributed-training">Distributed Training</h3>
<p>Training Gym supports multiple distributed training strategies:</p>
<ul>
<li><strong>Data Parallel</strong>: Simple replication across devices</li>
<li><strong>Tensor Parallel</strong>: Split layers across devices</li>
<li><strong>Pipeline Parallel</strong>: Split model stages across devices</li>
<li><strong>ZeRO</strong>: Memory-efficient data parallelism</li>
<li><strong>FSDP</strong>: Fully sharded data parallel (PyTorch native)</li>
</ul>
<p>Configuration is declarative:</p>
<pre><code class="language-yaml">distributed:
  strategy: fsdp
  world_size: 64
  sharding_strategy: full_shard
  mixed_precision: bf16
  gradient_checkpointing: true
</code></pre>
<h3 id="training-loop">Training Loop</h3>
<p>The training loop is modular and extensible:</p>
<pre><code class="language-python">from training_gym import Trainer, TrainingConfig

config = TrainingConfig(
    model="zen-7b",
    optimizer="adamw",
    learning_rate=1e-4,
    batch_size=2048,
    max_steps=100000,
    warmup_steps=2000,
    weight_decay=0.1,
)

trainer = Trainer(config)
trainer.fit(dataset)
</code></pre>
<p>Built-in features:</p>
<ul>
<li>Learning rate scheduling</li>
<li>Gradient clipping</li>
<li>Mixed precision training</li>
<li>Automatic checkpointing</li>
<li>Loss spike detection and recovery</li>
</ul>
<h3 id="evaluation">Evaluation</h3>
<p>Standardized evaluation across common benchmarks:</p>
<pre><code class="language-python">from training_gym.eval import Evaluator

evaluator = Evaluator(
    benchmarks=["mmlu", "hellaswag", "winogrande", "arc"],
    model=model,
)

results = evaluator.run()
</code></pre>
<p>Supported benchmarks:</p>
<ul>
<li>MMLU (multitask language understanding)</li>
<li>HellaSwag (commonsense reasoning)</li>
<li>WinoGrande (coreference resolution)</li>
<li>ARC (science questions)</li>
<li>TruthfulQA (truthfulness)</li>
<li>HumanEval (code generation)</li>
<li>GSM8K (math reasoning)</li>
</ul>
<h3 id="experiment-tracking">Experiment Tracking</h3>
<p>Every training run is tracked:</p>
<pre><code class="language-python">from training_gym import Experiment

with Experiment("zen-7b-v2") as exp:
    exp.log_config(config)
    trainer.fit(dataset)
    exp.log_metrics(results)
    exp.log_artifacts(["model.pt", "tokenizer/"])
</code></pre>
<p>The experiment tracker records:</p>
<ul>
<li>Hyperparameters</li>
<li>Training metrics (loss, gradient norms, learning rates)</li>
<li>Evaluation results</li>
<li>System metrics (GPU utilization, memory)</li>
<li>Artifacts (checkpoints, configs)</li>
</ul>
<h2 id="reproducibility">Reproducibility</h2>
<p>Training Gym emphasizes reproducibility:</p>
<h3 id="deterministic-training">Deterministic Training</h3>
<pre><code class="language-yaml">reproducibility:
  seed: 42
  deterministic_algorithms: true
  cublas_workspace_config: ":4096:8"
</code></pre>
<p>Same seed, same results (within floating point precision).</p>
<h3 id="environment-capture">Environment Capture</h3>
<p>Every experiment records:</p>
<ul>
<li>Git commit hash</li>
<li>Package versions</li>
<li>Hardware configuration</li>
<li>CUDA/cuDNN versions</li>
</ul>
<h3 id="configuration-as-code">Configuration as Code</h3>
<p>All configs are versioned YAML:</p>
<pre><code class="language-yaml"># experiments/zen-7b-v2.yaml
model:
  architecture: llama
  hidden_size: 4096
  num_layers: 32
  num_heads: 32
  vocab_size: 32000

training:
  batch_size: 2048
  learning_rate: 3e-4
  max_steps: 150000
</code></pre>
<h2 id="community-features">Community Features</h2>
<p>Training Gym includes tools for collaborative development:</p>
<h3 id="model-registry">Model Registry</h3>
<p>Share and discover models:</p>
<pre><code class="language-python">from training_gym.registry import ModelRegistry

registry = ModelRegistry()

# Publish a model
registry.push("my-org/my-model", model, config)

# Load a model
model = registry.pull("zoo-labs/zen-7b")
</code></pre>
<h3 id="leaderboards">Leaderboards</h3>
<p>Automatic benchmark submission:</p>
<pre><code class="language-python">evaluator.submit_to_leaderboard(
    model_name="zen-7b-v2",
    organization="zoo-labs",
)
</code></pre>
<h3 id="dataset-sharing">Dataset Sharing</h3>
<pre><code class="language-python">from training_gym.data import DatasetRegistry

# Share processed datasets
DatasetRegistry.push("my-corpus", dataset, license="cc-by-4.0")

# Load shared datasets
dataset = DatasetRegistry.pull("zoo-labs/zen-pretrain-v1")
</code></pre>
<h2 id="getting-started">Getting Started</h2>
<h3 id="installation">Installation</h3>
<pre><code class="language-bash">pip install training-gym

# For distributed training
pip install training-gym[distributed]

# For evaluation suite
pip install training-gym[eval]
</code></pre>
<h3 id="quick-start">Quick Start</h3>
<pre><code class="language-python">from training_gym import quickstart

# Train a small model to verify setup
quickstart.train_tiny_model()

# Run evaluation suite
quickstart.evaluate_model("my-model")
</code></pre>
<h3 id="documentation">Documentation</h3>
<p>Full documentation at docs.training-gym.ai:</p>
<ul>
<li>Getting started guide</li>
<li>Architecture overview</li>
<li>API reference</li>
<li>Example configurations</li>
<li>Troubleshooting</li>
</ul>
<h2 id="roadmap">Roadmap</h2>
<p><strong>Q4 2023</strong>: Multi-modal training support
<strong>Q1 2024</strong>: Reinforcement learning from human feedback (RLHF) integration
<strong>Q2 2024</strong>: Federated training support
<strong>Q3 2024</strong>: Automated hyperparameter optimization</p>
<h2 id="conclusion">Conclusion</h2>
<p>Open AI development needs open infrastructure. Training Gym provides the tools to train, evaluate, and share models. Join us in building the future of open AI.</p>
<p>Repository: github.com/zoo-labs/training-gym</p>
<hr>
<p><em>Zach Kelling is a co-founder of Zoo Labs Foundation.</em></p>f:["$","main",null,{"children":["$","article",null,{"className":"blog-article","children":[["$","$L5",null,{"className":"blog-back","href":"/blog","children":"← Blog"}],["$","div",null,{"className":"blog-post-meta","children":["September 10, 2023"," ","·"," ",4," min read"]}],["$","h1",null,{"className":"blog-post-title","children":"Training Gym: A Platform for Open Model Development"}],["$","p",null,{"className":"blog-post-lede","children":"Announcing Training Gym, our open platform for collaborative large model training."}],["$","div",null,{"className":"blog-prose","dangerouslySetInnerHTML":{"__html":"$19"}}]]}]}]
15:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]
11:null
13:{"metadata":[["$","title","0",{"children":"Training Gym: A Platform for Open Model Development — Zen Blog"}],["$","meta","1",{"name":"description","content":"Announcing Training Gym, our open platform for collaborative large model training."}],["$","meta","2",{"name":"keywords","content":"AI, LLM, Agentic AI, Code Generation, Zen Coder, Multimodal, Open Source, Machine Learning"}]],"error":null,"digest":"$undefined"}
18:"$13:metadata"
