How to Run NVIDIA Dory 4B with llama.cpp: Loop by Loop Guide

In this tutorial, we work with Dory 1.0 4B, a small reasoning model from NVIDIA’s Nemotron Labs whose middle layers form a shared block that can be applied once, twice, three, or four times for every token.

Each extra pass, or loop, spends more computation per token without adding a single parameter, so one checkpoint can run at four different compute budgets with no retraining.We build the llama.cpp branch that NVIDIA’s model card pins, download the 3.

7 GB GGUF, and read the architecture straight from the file to predict what each loop should cost in parameters and cache memory.

We then check those predictions against the running server, measure decode speed at every loop count, test whether the cheaper last-loop cache changes the model’s predictions, and run a paired GSM8K evaluation to see how accuracy, answer length, and time per solved question move as the loops increase.

Copy CodeCopiedUse a different Browserimport os import re import sys import json import time import math import random import signal import shutil import hashlib import platform import statistics import traceback import subprocess import collections import urllib.

request from pathlib import Path RESULTS = {} # step -> one-line result, for the summary table CLAIMS = [] # (step, claim, held) CURRENT = ["1"] def banner(title): print("\n" + "=" 78) print(title) print("=" 78) def section(name): def wrap(fn): def run(a, kw): CURRENT[0] = name.split(".

")[0] banner(name) t0 = time.time() try: out = fn(a, kw) RESULTS[name] = out if isinstance(out, str) else "ok" return out except Exception as e: RESULTS[name] = f"SKIPPED / FAILED -> {type(e).name}: {e}" print(f"\n[!] {name} did not complete: {type(e).name}: {e}") traceback.

printexc(limit=3) return None finally: print(f"\n[{time.time() - t0:.1f} s]") return run return wrap def claim(text, held): CLAIMS.append((CURRENT[0], text, bool(held))) return held banner("1.Build the pinned Dory llama.cpp fork and record the environment") FORK = "https://github.

com/brianchmielNV/llama.cpp.git" FORKCOMMIT = "98a74c97ea09ac72f6ceda1559808fc0a2f447ce" # the commit NVIDIA's model card pins; ggml-org PR #30306 is still a draft GGUFREPO = "nvidia/NVIDIA-Nemotron-Labs-Dory1.

0-4B-GGUF" GGUFREV = "df53755cb8741843af8e019e088819fbef17e29b" GGUFFILE = "NVIDIA-Nemotron-Labs-Dory1.0-4B-Q5KM.gguf" # the card's commands call it dory-q5km.

gguf GGUFSIZE = 3673288864 GGUFSHA256 = "96080be49ffd67b916018b7bdef8c484762576fb52780e10ab98ac4978a660e0" GSM8KREV = "740312add88f781978c0658806c59bc2815b9866" WORK = Path(os.environ.get("DORYWORK") or ("/content/dory" if os.path.isdir("/content") else "dorywork")).resolve() WORK.

mkdir(parents=True, existok=True) (WORK / "logs").mkdir(existok=True) subprocess.run([sys.executable, "-m", "pip", "install", "-q", "gguf==0.19.0", "huggingfacehub", "pandas", "pyarrow"], check=True) SYSTEM = platform.system() GPU = "" if shutil.which("nvidia-smi"): q = subprocess.

run(["nvidia-smi", "--query-gpu=name,memory.total,computecap", "--format=csv,noheader"], captureoutput=True, text=True) GPU = q.stdout.strip().splitlines()[0] if q.returncode == 0 and q.stdout.

strip() else "" BACKEND = "metal" if SYSTEM == "Darwin" else ("cuda" if GPU else "cpu") SRC = WORK / "llama.cpp-dory" SERVERBIN = SRC / "build" / "bin" / "llama-server" freegb = shutil.diskusage(WORK).free / 1e9 print(f"{SYSTEM} {platform.machine()}, Python {platform.pythonversion()}, {os.

cpucount()} CPU cores, " f"backend {BACKEND}{' (' + GPU + ')' if GPU else ''}, {freegb:.1f} GB free disk") if not SERVERBIN.exists(): t0 = time.time() if not (SRC / ".git").exists(): subprocess.run(["git", "init", "-q", str(SRC)], check=True) subprocess.

run(["git", "-C", str(SRC), "remote", "add", "origin", FORK], check=True) subprocess.run(["git", "-C", str(SRC), "fetch", "-q", "--depth", "1", "origin", FORKCOMMIT], check=True) subprocess.

run(["git", "-C", str(SRC), "checkout", "-q", "--detach", "FETCHHEAD"], check=True) flags = ["-DCMAKEBUILDTYPE=Release", "-DLLAMABUILDSERVER=ON", "-DLLAMABUILDTESTS=OFF", "-DLLAMABUILDEXAMPLES=OFF", "-DLLAMABUILDAPP=OFF", "-DLLAMABUILDUI=OFF", "-DLLAMAUSEPREBUILTUI=ON", "-DLLAMAOPENSSL=OFF"] if BACKEND == "metal": # the card's Apple-silicon flags flags += ["-DCMAKEOSXARCHITECTURES=arm64", "-DGGMLMETAL=ON", "-DGGMLMETALEMBEDLIBRARY=ON", "-DGGMLBLAS=ON", "-DGGMLBLASVENDOR=Apple"] elif BACKEND == "cuda": # kernels for this GPU only (a T4 is 7.

5); flash-attention kernels are unused with --flash-attn off cap = GPU.split(",")[-1].strip().replace(".", "") flags += ["-DGGMLCUDA=ON", f"-DCMAKECUDAARCHITECTURES={cap}", "-DGGMLCUDAFA=OFF", "-DGGMLNATIVE=OFF"] env = dict(os.environ, HFUIVERSION="b11551") with open(WORK / "build.

log", "w") as log: subprocess.run(["cmake", "-S", str(SRC), "-B", str(SRC / "build")] + flags, check=True, env=env, stdout=log, stderr=log) subprocess.run(["cmake", "--build", str(SRC / "build"), "--parallel", str(os.

cpucount()), "--target", "llama-server"], check=True, env=env, stdout=log, stderr=log) print(f"built llama-server in {time.time() - t0:.0f} s") head = subprocess.run(["git", "-C", str(SRC), "rev-parse", "HEAD"], captureoutput=True, text=True).stdout.

strip() assert head == FORKCOMMIT, f"checked out {head}, expected {FORKCOMMIT}" version = subprocess.run([str(SERVERBIN), "--version"], captureoutput=True, text=True) print(f"fork commit {head[:12]}; {(version.stdout + version.stderr).strip().splitlines()[-1]}") RESULTS["1.

Build the pinned Dory llama.cpp fork and record the environment"] = f"fork {head[:12]}, backend {BACKEND}" We build llama-server from the exact commit of the Dory branch that NVIDIA’s model card pins, because Dory support in llama.

cpp is still an open draft pull request and no released binary can load the model.

The cell detects the platform and passes the card’s Metal flags on a Mac, or compiles CUDA kernels for the attached GPU alone on Colab, skipping the flash-attention kernels that the card’s –flash-attn off never uses, which keeps a T4 build manageable.

It then checks that the checked-out commit is the pinned one, so every later number traces to one known version of the code.Copy CodeCopiedUse a different Browser@section("2.

Download the Q5KM GGUF at a pinned revision and check its SHA-256") def downloadstep(): global GGUF from huggingfacehub import hfhubdownload t0 = time.time() GGUF = Path(hfhubdownload(GGUFREPO, GGUFFILE, revision=GGUFREV, localdir=str(WORK))) h = hashlib.

sha256() with open(GGUF, "rb") as f: for block in iter(lambda: f.read(1 << 24), b""): h.update(block) size, digest = GGUF.stat().stsize, h.hexdigest() print(f"{GGUF.name}: {size:,} bytes, sha256 {digest[:16]}...({time.time() - t0:.

0f} s)") assert size == GGUFSIZE and digest == GGUFSHA256, "the file does not match the card's size and checksum" claim("the GGUF matches the size and SHA-256 published on the card", True) return f"{size / 1e9:.

2f} GB, checksum matches the card" downloadstep() The Q5KM GGUF comes from Hugging Face at a fixed repository revision, and we hash it before anything else touches it.The card’s commands refer to the file as dory-q5km.gguf, but the repository names it NVIDIA-Nemotron-Labs-Dory1.0-4B-Q5KM.

gguf, so copying the card’s command verbatim fails; pinning the real name, the size, and the SHA-256 together catches both a wrong file and a corrupted download.The 3.67 GB file matches the card’s published checksum exactly, so every measurement that follows uses the same bytes NVIDIA published.

Copy CodeCopiedUse a different Browser@section("3.Read the architecture from the GGUF: three layer groups and a default of four loops") def architecturestep(): global META, GROUPS, PATTERN import gguf reader = gguf.GGUFReader(str(GGUF)) META = {k: f.contents() for k, f in reader.fields.items() if k.

startswith("dory.")} for k in ("dory.blockcount", "dory.inputlayercount", "dory.recurrentlayercount", "dory.outputlayercount