Deploying DINOv3 as an INT8 TensorRT engine

The self-supervised foundation model DINOv3 developed by Meta, is a universal vision backbone that a growing share of state-of-the-art models are now built on top of. DINOv3’s output is not boxes or masks, the features themselves are the product.

In this tutorial we step-by-step deploy DINOv3 with TensorRT using embedl-deploy[tensorrt] on NVIDIA GPUs.

Pipeline:

1. Load DINOv3 and fuse + INT8-quantize it with embedl_deploy, calibrating on imagenette (auto-downloaded).

2. Export FP32 and INT8 (QDQ) ONNX graphs and build one TensorRT engine from each.

3. Verify the deployment: patch-similarity maps, PCA feature maps, latency, and k-NN accuracy vs the FP32 reference.

Note

The engine build and everything after it require an NVIDIA GPU with TensorRT 10.x (10.16 pip wheel recommended); tested on CUDA 12.8. The first run downloads imagenette (~1.5 GB).

Install dependencies:

pip install torch torchvision transformers onnx onnxscript matplotlib pillow
pip install "tensorrt-cu12==10.16.*"
pip install "embedl-deploy[tensorrt]"

The DINOv3 checkpoint is gated: request access on its Hugging Face model page (approval from Meta is usually quick) and authenticate before running:

hf auth login

Step 0 — Configuration

import sys
import time
import urllib.request
from pathlib import Path
from typing import Any

import torch
import torch.nn.functional as F
from torch import nn
from torch.utils.data import DataLoader
from torchvision import datasets, transforms

sys.setrecursionlimit(5000)

ARTIFACTS_PATH = Path("dinov3_artifacts")
ARTIFACTS_PATH.mkdir(parents=True, exist_ok=True)

MODEL_ID = "facebook/dinov3-vitb16-pretrain-lvd1689m"
IMAGE_SIZE = 224
PATCH_SIZE = 16
PATCH_PREFIX = 5  # 1 CLS + 4 register tokens precede the patch tokens
GRID = IMAGE_SIZE // PATCH_SIZE  # 14×14 = 196 patch tokens
N_CALIB = 512  # frames used for PTQ calibration

# DINOv3 uses standard ImageNet normalization.
IMAGENET_MEAN = [0.485, 0.456, 0.406]
IMAGENET_STD = [0.229, 0.224, 0.225]

DEMO_URL = "http://images.cocodataset.org/val2017/000000039769.jpg"
DEMO_IMAGE = ARTIFACTS_PATH / "demo_coco_cats.jpg"

ONNX_FP32 = ARTIFACTS_PATH / "dinov3-vitb16_original.onnx"
ONNX_INT8 = ARTIFACTS_PATH / "dinov3-vitb16_int8.onnx"
ENGINE_FP16 = ARTIFACTS_PATH / "dinov3.fp16.engine"
ENGINE_INT8 = ARTIFACTS_PATH / "dinov3.int8.engine"
TIMING_CACHE = ARTIFACTS_PATH / "trt_timing.cache"
BUILDER_OPTIMIZATION_LEVEL = 3

DEVICE = torch.device("cuda")

Step 1 — Load DINOv3

A thin wrapper gives torch.export a positional tensor signature: image [B, 3, 224, 224] → tokens [B, 201, 768] (1 CLS + 4 register + 196 patch tokens). Eager attention keeps the exported graph free of data-dependent SDPA branches. The untouched FP32 model is exported to ONNX right away — it becomes the FP16 baseline engine.

from transformers import (  # type: ignore[import-not-found]
    AutoConfig,
    AutoModel,
)


class Dinov3Encoder(nn.Module):
    """Wrap the backbone to expose an image-to-tokens forward signature."""

    def __init__(self, backbone: nn.Module) -> None:
        """Store the DINOv3 backbone."""
        super().__init__()
        self.backbone = backbone

    def forward(self, image: torch.Tensor) -> torch.Tensor:
        """Return the full token sequence."""
        return self.backbone(pixel_values=image).last_hidden_state


def load_dinov3() -> Dinov3Encoder:
    """Load a fresh, export-friendly DINOv3 model in fp32 eval mode."""
    cfg = AutoConfig.from_pretrained(MODEL_ID)
    cfg.torch_dtype = torch.float32
    if hasattr(cfg, "_attn_implementation"):
        cfg._attn_implementation = "eager"  # noqa: SLF001  # SDPA branches break export
    if hasattr(cfg, "use_cache"):
        cfg.use_cache = False
    backbone = AutoModel.from_pretrained(
        MODEL_ID, config=cfg, torch_dtype=torch.float32
    )
    return Dinov3Encoder(backbone).eval().to(DEVICE)


model = load_dinov3()


def export_onnx(module: nn.Module, path: Path) -> None:
    """Export a module to ONNX via the dynamo path."""
    dummy = torch.randn(1, 3, IMAGE_SIZE, IMAGE_SIZE, device=DEVICE)
    with torch.no_grad():
        torch.onnx.export(
            module,
            (dummy,),
            str(path),
            input_names=["image"],
            output_names=["last_hidden_state"],
            do_constant_folding=True,
            dynamo=True,
        )
    print(f"  {path.name}: {path.stat().st_size / 1e6:.0f} MB")
    del dummy
    torch.cuda.empty_cache()


export_onnx(model, ONNX_FP32)

Step 2 — Calibration data

Post-training quantization (PTQ) needs real images to calibrate activation ranges. Imagenette (a 10-class ImageNet subset) auto-downloads via torchvision; its train split provides calibration images (and later the k-NN index), the val split is the held-out test set.

EVAL_TF = transforms.Compose(
    [
        transforms.Resize(256),
        transforms.CenterCrop(IMAGE_SIZE),
        transforms.ToTensor(),
        transforms.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD),
    ]
)


def imagenette(split: str) -> datasets.Imagenette:
    """Load imagenette, downloading it on the first run."""
    try:
        return datasets.Imagenette(
            ".", split=split, transform=EVAL_TF, download=True
        )
    except RuntimeError:
        return datasets.Imagenette(".", split=split, transform=EVAL_TF)


train_ds, val_ds = imagenette("train"), imagenette("val")
loader_kw: dict[str, Any] = {
    "batch_size": 64,
    "num_workers": 4,
    "pin_memory": True,
}
train_loader = DataLoader(train_ds, shuffle=False, **loader_kw)
val_loader = DataLoader(val_ds, shuffle=False, **loader_kw)
# The split is class-sorted — a shuffled loader (fixed seed) keeps a
# partial calibration set class-balanced and reproducible.
calib_loader = DataLoader(
    train_ds,
    shuffle=True,
    generator=torch.Generator().manual_seed(42),
    **loader_kw,
)
print(f"  {len(train_ds)} train / {len(val_ds)} val images")

Step 3 — Fuse and INT8-quantize with Embedl Deploy

transform exports the model and applies the TensorRT fusion patterns; quantize inserts INT8 quantizers and calibrates them. Three recipe choices matter for ViT-family encoders:

• HISTOGRAM calibration for activations — MINMAX lets a single outlier in a ViT MLP blow up a whole tensor’s quantization range.

• Skip the patch-embed stem — the first 3-channel conv stays in higher precision; quantizing the model boundary costs the most.

• Skip SmoothQuant for LayerNorm — its learnable affine should not be folded into quant scales.

from embedl_deploy import transform
from embedl_deploy.backend import set_backend
from embedl_deploy.quantize import (
    CalibrationMethod,
    ModulesToSkip,
    Precision,
    QuantConfig,
    TensorQuantConfig,
    quantize,
)
from embedl_deploy.tensorrt import TENSORRT_PATTERNS

set_backend("tensorrt")

dummy = torch.randn(1, 3, IMAGE_SIZE, IMAGE_SIZE, device=DEVICE)
with torch.no_grad():
    result = transform(model, (dummy,), TENSORRT_PATTERNS)
print(
    f"Patterns applied: {result.report['applied_count']}, "
    f"skipped: {result.report['skipped_count']}"
)
fused = result.model.eval().to(device=DEVICE, dtype=torch.float32)


def patch_embed_conv(module: nn.Module) -> nn.Conv2d | None:
    """Return the stem conv (3-channel input, patch-sized kernel) to skip."""
    for mod in module.modules():
        if (
            isinstance(mod, nn.Conv2d)
            and mod.in_channels == 3
            and max(mod.kernel_size) >= 7
        ):
            return mod
    return None


skip_mods: set[nn.Module] = set()
if (stem := patch_embed_conv(fused)) is not None:
    skip_mods.add(stem)

config = QuantConfig(
    activation=TensorQuantConfig(
        precision=Precision.INT8,
        symmetric=True,
        per_channel=False,
        calibration_method=CalibrationMethod.HISTOGRAM,
    ),
    weight=TensorQuantConfig(
        precision=Precision.INT8,
        symmetric=True,
        per_channel=True,
    ),
    skip=ModulesToSkip(
        stub=skip_mods,  # type: ignore[arg-type]
        weight=skip_mods,  # type: ignore[arg-type]
        smooth={nn.LayerNorm},
    ),
)


def calib_loop(m: nn.Module) -> None:
    """Feed calibration images one by one (the graph is batch-1)."""
    m.eval()
    seen = 0
    with torch.no_grad():
        for images, _ in calib_loader:
            for i in range(images.shape[0]):
                if seen >= N_CALIB:
                    return
                m(images[i : i + 1].to(DEVICE))
                seen += 1


print(f"Calibrating (HISTOGRAM) on {N_CALIB} images")
int8_model = quantize(
    fused,
    args=(dummy,),
    config=config,
    forward_loop=calib_loop,
    freeze_weights=True,
)

export_onnx(int8_model, ONNX_INT8)

del model, result, fused, int8_model, dummy
torch.cuda.empty_cache()

Step 4 — Build the TensorRT engines

One FP16 baseline engine from the FP32 ONNX, one INT8+FP16 engine from the QDQ ONNX. A shared timing cache makes the second build (and any rebuild) much faster.

import tensorrt as trt


def build_engine(onnx_path: Path, engine_path: Path, *, int8: bool) -> None:
    """Build a TRT engine. Always FP16; INT8 added if requested."""
    logger = trt.Logger(trt.Logger.WARNING)
    trt.init_libnvinfer_plugins(logger, "")
    builder = trt.Builder(logger)
    network = builder.create_network(
        1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)
    )
    parser = trt.OnnxParser(network, logger)
    if not parser.parse(onnx_path.read_bytes(), path=str(onnx_path)):
        for i in range(parser.num_errors):
            print(parser.get_error(i))
        raise RuntimeError("ONNX parse failed")

    config = builder.create_builder_config()
    config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, 4 * 1024**3)
    config.builder_optimization_level = BUILDER_OPTIMIZATION_LEVEL
    config.set_flag(trt.BuilderFlag.FP16)
    if int8:
        config.set_flag(trt.BuilderFlag.INT8)
    cache = config.create_timing_cache(
        TIMING_CACHE.read_bytes() if TIMING_CACHE.exists() else b""
    )
    config.set_timing_cache(cache, ignore_mismatch=False)

    tag = "INT8+FP16" if int8 else "FP16"
    print(f"Building {tag} engine from {onnx_path.name}")
    t0 = time.perf_counter()
    plan = builder.build_serialized_network(network, config)
    if plan is None:
        raise RuntimeError("build_serialized_network returned None")

    TIMING_CACHE.write_bytes(bytes(config.get_timing_cache().serialize()))
    engine_path.write_bytes(bytes(plan))
    print(
        f"  built in {time.perf_counter() - t0:.0f} s → {engine_path.name} "
        f"({len(bytes(plan)) / 1e6:.0f} MB)"
    )


build_engine(ONNX_FP32, ENGINE_FP16, int8=False)
build_engine(ONNX_INT8, ENGINE_INT8, int8=True)

Running the engines

A small helper used for each step below. The engine works on fixed GPU memory locations, so its input and output tensors are allocated once — each infer() just copies an image in, runs the engine, and copies the result out.

_TRT_TORCH = {
    trt.float32: torch.float32,
    trt.float16: torch.float16,
    trt.int32: torch.int32,
    trt.int64: torch.int64,
    trt.int8: torch.int8,
    trt.bool: torch.bool,
}


class TrtRunner:
    """TRT engine wrapper backed by torch.cuda tensors as I/O buffers."""

    def __init__(self, engine_path: Path) -> None:
        """Deserialize the engine and bind I/O buffers."""
        logger = trt.Logger(trt.Logger.WARNING)
        trt.init_libnvinfer_plugins(logger, "")
        self.engine = trt.Runtime(logger).deserialize_cuda_engine(
            engine_path.read_bytes()
        )
        self.ctx = self.engine.create_execution_context()
        self.stream = torch.cuda.Stream()
        self.bufs: dict[str, torch.Tensor] = {}
        self.outs: list[str] = []
        for i in range(self.engine.num_io_tensors):
            name = self.engine.get_tensor_name(i)
            shape = tuple(self.engine.get_tensor_shape(name))
            dtype = _TRT_TORCH[self.engine.get_tensor_dtype(name)]
            self.bufs[name] = torch.empty(shape, dtype=dtype, device="cuda")
            self.ctx.set_tensor_address(name, self.bufs[name].data_ptr())
            if self.engine.get_tensor_mode(name) == trt.TensorIOMode.OUTPUT:
                self.outs.append(name)

    def run(self) -> None:
        """Execute the engine on whatever is in the input buffers."""
        self.ctx.execute_async_v3(self.stream.cuda_stream)
        self.stream.synchronize()

    def infer(self, image: torch.Tensor) -> torch.Tensor:
        """Copy in → execute → return the token tensor on CPU, fp32."""
        buf = self.bufs["image"]
        buf.copy_(image.to(device=buf.device, dtype=buf.dtype))
        self.run()
        return self.bufs[self.outs[0]].detach().float().cpu()


fp16_runner = TrtRunner(ENGINE_FP16)
int8_runner = TrtRunner(ENGINE_INT8)

Step 5 — Feature inspection

Two views of the patch features, FP16 engine vs INT8 engine:

• Patch similarity — pick an anchor patch (the ✕), color every patch by cosine similarity to it. Semantically matching regions light up: anchoring on one cat highlights both cats.

• PCA features — project the 768-dim patch features onto their top 3 principal components and render them as RGB.

PCA features rendered as RGB — matching colors mean the quantized model preserves the feature geometry:

import matplotlib.pyplot as plt  # type: ignore[import-not-found]
from PIL import Image  # type: ignore[import-not-found]

ANCHOR_YX = (5, 4)  # left cat's head, in the 14×14 patch grid


def demo_image() -> Image.Image:
    """Download (once) and open the COCO cats demo image."""
    if not DEMO_IMAGE.exists():
        urllib.request.urlretrieve(DEMO_URL, DEMO_IMAGE)
    return Image.open(DEMO_IMAGE).convert("RGB")


def patch_grid(tokens: torch.Tensor) -> torch.Tensor:
    """Reshape ``[1, 201, 768]`` tokens into a ``[14, 14, 768]`` patch grid."""
    return tokens[0, PATCH_PREFIX:].reshape(GRID, GRID, -1)


def similarity_map(patches: torch.Tensor, yx: tuple[int, int]) -> torch.Tensor:
    """Compute cosine similarity of every patch to the anchor patch."""
    anchor = patches[yx[0], yx[1]]
    flat = patches.reshape(-1, patches.shape[-1])
    return F.cosine_similarity(flat, anchor[None], dim=-1).reshape(GRID, GRID)


def pca_rgb(
    patches: torch.Tensor, basis: torch.Tensor | None = None
) -> tuple[torch.Tensor, torch.Tensor]:
    """Project the patch features onto their top 3 principal components."""
    flat = patches.reshape(-1, patches.shape[-1])
    flat = flat - flat.mean(dim=0)
    if basis is None:
        _, _, basis = torch.pca_lowrank(flat, q=3)
    rgb = (flat @ basis).reshape(GRID, GRID, 3)
    lo = rgb.amin(dim=(0, 1))
    hi = rgb.amax(dim=(0, 1))
    return (rgb - lo) / (hi - lo).clamp_min(1e-8), basis


image = demo_image()
crop = transforms.CenterCrop(IMAGE_SIZE)(transforms.Resize(256)(image))
x = EVAL_TF(image).unsqueeze(0)

fp16_patches = patch_grid(fp16_runner.infer(x))
int8_patches = patch_grid(int8_runner.infer(x))


fig, axes = plt.subplots(2, 3, figsize=(10.5, 7))
for row in axes:
    row[0].imshow(crop)
ay, ax_ = ANCHOR_YX
axes[0, 0].plot(
    (ax_ + 0.5) * PATCH_SIZE, (ay + 0.5) * PATCH_SIZE, "rx", ms=12, mew=3
)
# Shared color scale so the two heatmaps are directly comparable.
sim_kw = {"cmap": "viridis", "vmin": -0.2, "vmax": 1.0}
axes[0, 1].imshow(similarity_map(fp16_patches, ANCHOR_YX), **sim_kw)
axes[0, 2].imshow(similarity_map(int8_patches, ANCHOR_YX), **sim_kw)
rgb_fp16, basis = pca_rgb(fp16_patches)
rgb_int8, _ = pca_rgb(int8_patches, basis)
axes[1, 1].imshow(rgb_fp16)
axes[1, 2].imshow(rgb_int8)
axes[0, 0].set_title("Input (✕ = anchor)")
for col, name in [(1, "TensorRT FP16"), (2, "Embedl INT8")]:
    axes[0, col].set_title(f"{name} — similarity")
    axes[1, col].set_title(f"{name} — PCA")
for a in axes.ravel():
    a.axis("off")
fig.tight_layout()
fig.savefig(ARTIFACTS_PATH / "feature_maps.png", dpi=120, bbox_inches="tight")
print(f"  → {ARTIFACTS_PATH / 'feature_maps.png'}")

Step 6 — Latency and k-NN accuracy

Benchmark both engines: forward-pass latency via CUDA events, and k-NN classification accuracy on imagenette against the FP32 reference.

N_WARMUP, N_ITERS = 20, 200


def bench(runner: TrtRunner) -> tuple[float, float]:
    """Compute mean and p95 engine latency in ms over ``N_ITERS`` runs."""
    for _ in range(N_WARMUP):
        runner.run()
    start = torch.cuda.Event(enable_timing=True)
    end = torch.cuda.Event(enable_timing=True)
    samples = []
    for _ in range(N_ITERS):
        start.record()
        runner.run()
        end.record()
        torch.cuda.synchronize()
        samples.append(start.elapsed_time(end))
    samples.sort()
    return sum(samples) / len(samples), samples[int(0.95 * len(samples)) - 1]


fp16_mean, fp16_p95 = bench(fp16_runner)
int8_mean, int8_p95 = bench(int8_runner)
print("Latency (DINOv3 ViT-B/16 @ 224x224, batch 1)")
print(f"  TensorRT FP16:  mean {fp16_mean:5.2f} ms  p95 {fp16_p95:5.2f} ms")
print(
    f"  Embedl INT8:    mean {int8_mean:5.2f} ms  p95 {int8_p95:5.2f} ms  "
    f"({fp16_mean / int8_mean:.2f}x vs FP16)"
)

KNN_K, KNN_TAU = 20, 0.07


@torch.no_grad()
def cls_features(
    model_fn: nn.Module, loader: DataLoader[Any]
) -> tuple[torch.Tensor, torch.Tensor]:
    """Return L2-normalized CLS features (``[N, D]``) and labels (``[N]``)."""
    model_fn.eval()
    feats, labels = [], []
    for images, targets in loader:
        hidden = model_fn(images.to(DEVICE))
        feats.append(F.normalize(hidden[:, 0, :].float(), dim=-1).cpu())
        labels.append(targets)
    return torch.cat(feats), torch.cat(labels)


@torch.no_grad()
def cls_features_trt(
    runner: TrtRunner, ds: datasets.Imagenette
) -> torch.Tensor:
    """Return CLS features from the batch-1 engine for each image in `ds`."""
    feats = []
    for image_t, _ in ds:
        tokens = runner.infer(image_t.unsqueeze(0))
        feats.append(F.normalize(tokens[:, 0, :], dim=-1))
    return torch.cat(feats)


def knn_top1(
    index_feats: torch.Tensor,
    index_labels: torch.Tensor,
    query_feats: torch.Tensor,
    query_labels: torch.Tensor,
    num_classes: int,
) -> float:
    """Compute k-NN top-1 accuracy (%) via weighted exp(sim/tau) votes."""
    index_gpu = index_feats.to(DEVICE)
    correct = 0
    for start in range(0, len(query_feats), 256):
        q = query_feats[start : start + 256].to(DEVICE)
        top_sim, top_idx = (q @ index_gpu.T).topk(KNN_K, dim=1)
        votes = torch.zeros(len(q), num_classes, device=DEVICE)
        votes.scatter_add_(
            1,
            index_labels[top_idx.cpu()].to(DEVICE),
            (top_sim / KNN_TAU).exp(),
        )
        pred = votes.argmax(dim=1).cpu()
        correct += int(
            (pred == query_labels[start : start + 256]).sum().item()
        )
    return 100.0 * correct / len(query_feats)


print("FP32 reference: extracting index (train) + query (val) features")
fp32 = load_dinov3()
index_feats, index_labels = cls_features(fp32, train_loader)
val_feats, val_labels = cls_features(fp32, val_loader)
del fp32
torch.cuda.empty_cache()

n_classes = len(train_ds.classes)
fp32_acc = knn_top1(
    index_feats, index_labels, val_feats, val_labels, n_classes
)
print(f"  FP32 knn_top1: {fp32_acc:.2f}%")

print("INT8 engine: extracting query features")
int8_feats = cls_features_trt(int8_runner, val_ds)
int8_acc = knn_top1(
    index_feats, index_labels, int8_feats, val_labels, n_classes
)
print(
    f"  INT8 knn_top1: {int8_acc:.2f}%  ({int8_acc - fp32_acc:+.2f} vs FP32)"
)

On an NVIDIA L4 with TensorRT 10.16:

Latency (DINOv3 ViT-B/16 @ 224x224, batch 1)
  TensorRT FP16:  mean  1.50 ms  p95  1.52 ms
  Embedl INT8:    mean  1.13 ms  p95  1.15 ms  (1.34x vs FP16)
FP32 reference: extracting index (train) + query (val) features
  FP32 knn_top1: 99.75%
INT8 engine: extracting query features
  INT8 knn_top1: 99.26%  (-0.48 vs FP32)

Summary

HuggingFace ─▶ Embedl Deploy fuse + INT8 PTQ ─▶ ONNX (QDQ)
            ─▶ TensorRT engine (INT8+FP16)

DINOv3 ViT-B/16 @ 224×224, batch 1, TensorRT 10.16:

Path	NVIDIA L4	Jetson AGX Orin	knn_top1 (imagenette)
TensorRT FP16	1.50 ms	2.71 ms	99.75 % (FP32 ref)
Embedl INT8	1.13 ms	2.25 ms	99.26 %

The quantized model, engines and full evaluation protocol are published at embedl/dinov3-quantized-tensorrt. For the same recipe on a detection model, see the SAM3 tutorial.