# Copyright (C) 2026 Embedl AB # mypy: disable-error-code="no-untyped-call,no-any-return" """ Deploying DINOv3 as an INT8 TensorRT engine =========================================== The self-supervised foundation model `DINOv3 `_ developed by Meta, is a universal vision backbone that a growing share of state-of-the-art models are now built on top of. DINOv3's output is not boxes or masks, the features themselves are the product. .. image:: https://storage.googleapis.com/embedl-deploy-artifacts/pca_video.gif :alt: Animated clip of a cockatoo next to its DINOv3 patch features, foreground-segmented and colored by their top-3 PCA components — the FP32 and Embedl INT8 columns track the moving subject in lockstep, one block per 16×16-px patch. :width: 100% | In this tutorial we step-by-step deploy DINOv3 with TensorRT using ``embedl-deploy[tensorrt]`` on NVIDIA GPUs. Pipeline: 1. Load DINOv3 and fuse + INT8-quantize it with ``embedl_deploy``, calibrating on imagenette (auto-downloaded). 2. Export FP32 and INT8 (QDQ) ONNX graphs and build one TensorRT engine from each. 3. Verify the deployment: patch-similarity maps, PCA feature maps, latency, and k-NN accuracy vs the FP32 reference. .. note:: The engine build and everything after it require an NVIDIA GPU with TensorRT 10.x (10.16 pip wheel recommended); tested on CUDA 12.8. The first run downloads imagenette (~1.5 GB). | Install dependencies: .. code-block:: bash pip install torch torchvision transformers onnx onnxscript matplotlib pillow pip install "tensorrt-cu12==10.16.*" pip install "embedl-deploy[tensorrt]" The DINOv3 checkpoint is gated: request access on its `Hugging Face model page `_ (approval from Meta is usually quick) and authenticate before running: .. code-block:: bash hf auth login """ # %% # Step 0 — Configuration # ---------------------- # sphinx_gallery_start_ignore # pylint: disable=wrong-import-position,wrong-import-order,ungrouped-imports,useless-suppression,no-member,import-error # sphinx_gallery_end_ignore import sys import time import urllib.request from pathlib import Path from typing import Any import torch import torch.nn.functional as F from torch import nn from torch.utils.data import DataLoader from torchvision import datasets, transforms sys.setrecursionlimit(5000) ARTIFACTS_PATH = Path("dinov3_artifacts") ARTIFACTS_PATH.mkdir(parents=True, exist_ok=True) MODEL_ID = "facebook/dinov3-vitb16-pretrain-lvd1689m" IMAGE_SIZE = 224 PATCH_SIZE = 16 PATCH_PREFIX = 5 # 1 CLS + 4 register tokens precede the patch tokens GRID = IMAGE_SIZE // PATCH_SIZE # 14×14 = 196 patch tokens N_CALIB = 512 # frames used for PTQ calibration # DINOv3 uses standard ImageNet normalization. IMAGENET_MEAN = [0.485, 0.456, 0.406] IMAGENET_STD = [0.229, 0.224, 0.225] DEMO_URL = "http://images.cocodataset.org/val2017/000000039769.jpg" DEMO_IMAGE = ARTIFACTS_PATH / "demo_coco_cats.jpg" ONNX_FP32 = ARTIFACTS_PATH / "dinov3-vitb16_original.onnx" ONNX_INT8 = ARTIFACTS_PATH / "dinov3-vitb16_int8.onnx" ENGINE_FP16 = ARTIFACTS_PATH / "dinov3.fp16.engine" ENGINE_INT8 = ARTIFACTS_PATH / "dinov3.int8.engine" TIMING_CACHE = ARTIFACTS_PATH / "trt_timing.cache" BUILDER_OPTIMIZATION_LEVEL = 3 DEVICE = torch.device("cuda") # %% # Step 1 — Load DINOv3 # -------------------- # # A thin wrapper gives ``torch.export`` a positional tensor signature: # ``image [B, 3, 224, 224] → tokens [B, 201, 768]`` (1 CLS + 4 register + # 196 patch tokens). Eager attention keeps the exported graph free of # data-dependent SDPA branches. The untouched FP32 model is exported to # ONNX right away — it becomes the FP16 baseline engine. from transformers import ( # type: ignore[import-not-found] AutoConfig, AutoModel, ) class Dinov3Encoder(nn.Module): """Wrap the backbone to expose an image-to-tokens forward signature.""" def __init__(self, backbone: nn.Module) -> None: """Store the DINOv3 backbone.""" super().__init__() self.backbone = backbone def forward(self, image: torch.Tensor) -> torch.Tensor: """Return the full token sequence.""" return self.backbone(pixel_values=image).last_hidden_state def load_dinov3() -> Dinov3Encoder: """Load a fresh, export-friendly DINOv3 model in fp32 eval mode.""" cfg = AutoConfig.from_pretrained(MODEL_ID) cfg.torch_dtype = torch.float32 if hasattr(cfg, "_attn_implementation"): cfg._attn_implementation = "eager" # noqa: SLF001 # SDPA branches break export if hasattr(cfg, "use_cache"): cfg.use_cache = False backbone = AutoModel.from_pretrained( MODEL_ID, config=cfg, torch_dtype=torch.float32 ) return Dinov3Encoder(backbone).eval().to(DEVICE) model = load_dinov3() def export_onnx(module: nn.Module, path: Path) -> None: """Export a module to ONNX via the dynamo path.""" dummy = torch.randn(1, 3, IMAGE_SIZE, IMAGE_SIZE, device=DEVICE) with torch.no_grad(): torch.onnx.export( module, (dummy,), str(path), input_names=["image"], output_names=["last_hidden_state"], do_constant_folding=True, dynamo=True, ) print(f" {path.name}: {path.stat().st_size / 1e6:.0f} MB") del dummy torch.cuda.empty_cache() export_onnx(model, ONNX_FP32) # %% # Step 2 — Calibration data # ------------------------- # # Post-training quantization (PTQ) needs real images to calibrate activation # ranges. Imagenette (a 10-class ImageNet subset) auto-downloads via # torchvision; its train split provides calibration images (and later the # k-NN index), the val split is the held-out test set. EVAL_TF = transforms.Compose( [ transforms.Resize(256), transforms.CenterCrop(IMAGE_SIZE), transforms.ToTensor(), transforms.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD), ] ) def imagenette(split: str) -> datasets.Imagenette: """Load imagenette, downloading it on the first run.""" try: return datasets.Imagenette( ".", split=split, transform=EVAL_TF, download=True ) except RuntimeError: return datasets.Imagenette(".", split=split, transform=EVAL_TF) train_ds, val_ds = imagenette("train"), imagenette("val") loader_kw: dict[str, Any] = { "batch_size": 64, "num_workers": 4, "pin_memory": True, } train_loader = DataLoader(train_ds, shuffle=False, **loader_kw) val_loader = DataLoader(val_ds, shuffle=False, **loader_kw) # The split is class-sorted — a shuffled loader (fixed seed) keeps a # partial calibration set class-balanced and reproducible. calib_loader = DataLoader( train_ds, shuffle=True, generator=torch.Generator().manual_seed(42), **loader_kw, ) print(f" {len(train_ds)} train / {len(val_ds)} val images") # %% # Step 3 — Fuse and INT8-quantize with Embedl Deploy # -------------------------------------------------- # # ``transform`` exports the model and applies the TensorRT fusion patterns; # ``quantize`` inserts INT8 quantizers and calibrates them. Three recipe # choices matter for ViT-family encoders: # # - **HISTOGRAM calibration for activations** — MINMAX lets a single # outlier in a ViT MLP blow up a whole tensor's quantization range. # - **Skip the patch-embed stem** — the first 3-channel conv stays in # higher precision; quantizing the model boundary costs the most. # - **Skip SmoothQuant for LayerNorm** — its learnable affine should not # be folded into quant scales. from embedl_deploy import transform from embedl_deploy.backend import set_backend from embedl_deploy.quantize import ( CalibrationMethod, ModulesToSkip, Precision, QuantConfig, TensorQuantConfig, quantize, ) from embedl_deploy.tensorrt import TENSORRT_PATTERNS set_backend("tensorrt") dummy = torch.randn(1, 3, IMAGE_SIZE, IMAGE_SIZE, device=DEVICE) with torch.no_grad(): result = transform(model, (dummy,), TENSORRT_PATTERNS) print( f"Patterns applied: {result.report['applied_count']}, " f"skipped: {result.report['skipped_count']}" ) fused = result.model.eval().to(device=DEVICE, dtype=torch.float32) def patch_embed_conv(module: nn.Module) -> nn.Conv2d | None: """Return the stem conv (3-channel input, patch-sized kernel) to skip.""" for mod in module.modules(): if ( isinstance(mod, nn.Conv2d) and mod.in_channels == 3 and max(mod.kernel_size) >= 7 ): return mod return None skip_mods: set[nn.Module] = set() if (stem := patch_embed_conv(fused)) is not None: skip_mods.add(stem) config = QuantConfig( activation=TensorQuantConfig( precision=Precision.INT8, symmetric=True, per_channel=False, calibration_method=CalibrationMethod.HISTOGRAM, ), weight=TensorQuantConfig( precision=Precision.INT8, symmetric=True, per_channel=True, ), skip=ModulesToSkip( stub=skip_mods, # type: ignore[arg-type] weight=skip_mods, # type: ignore[arg-type] smooth={nn.LayerNorm}, ), ) def calib_loop(m: nn.Module) -> None: """Feed calibration images one by one (the graph is batch-1).""" m.eval() seen = 0 with torch.no_grad(): for images, _ in calib_loader: for i in range(images.shape[0]): if seen >= N_CALIB: return m(images[i : i + 1].to(DEVICE)) seen += 1 print(f"Calibrating (HISTOGRAM) on {N_CALIB} images") int8_model = quantize( fused, args=(dummy,), config=config, forward_loop=calib_loop, freeze_weights=True, ) export_onnx(int8_model, ONNX_INT8) del model, result, fused, int8_model, dummy torch.cuda.empty_cache() # %% # Step 4 — Build the TensorRT engines # ----------------------------------- # # One FP16 baseline engine from the FP32 ONNX, one INT8+FP16 engine from # the QDQ ONNX. A shared timing cache makes the second build (and any # rebuild) much faster. import tensorrt as trt def build_engine(onnx_path: Path, engine_path: Path, *, int8: bool) -> None: """Build a TRT engine. Always FP16; INT8 added if requested.""" logger = trt.Logger(trt.Logger.WARNING) trt.init_libnvinfer_plugins(logger, "") builder = trt.Builder(logger) network = builder.create_network( 1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH) ) parser = trt.OnnxParser(network, logger) if not parser.parse(onnx_path.read_bytes(), path=str(onnx_path)): for i in range(parser.num_errors): print(parser.get_error(i)) raise RuntimeError("ONNX parse failed") config = builder.create_builder_config() config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, 4 * 1024**3) config.builder_optimization_level = BUILDER_OPTIMIZATION_LEVEL config.set_flag(trt.BuilderFlag.FP16) if int8: config.set_flag(trt.BuilderFlag.INT8) cache = config.create_timing_cache( TIMING_CACHE.read_bytes() if TIMING_CACHE.exists() else b"" ) config.set_timing_cache(cache, ignore_mismatch=False) tag = "INT8+FP16" if int8 else "FP16" print(f"Building {tag} engine from {onnx_path.name}") t0 = time.perf_counter() plan = builder.build_serialized_network(network, config) if plan is None: raise RuntimeError("build_serialized_network returned None") TIMING_CACHE.write_bytes(bytes(config.get_timing_cache().serialize())) engine_path.write_bytes(bytes(plan)) print( f" built in {time.perf_counter() - t0:.0f} s → {engine_path.name} " f"({len(bytes(plan)) / 1e6:.0f} MB)" ) build_engine(ONNX_FP32, ENGINE_FP16, int8=False) build_engine(ONNX_INT8, ENGINE_INT8, int8=True) # %% # Running the engines # ------------------- # # A small helper used for each step below. The engine works on fixed # GPU memory locations, so its input and output tensors are allocated # once — each ``infer()`` just copies an image in, runs the engine, and # copies the result out. _TRT_TORCH = { trt.float32: torch.float32, trt.float16: torch.float16, trt.int32: torch.int32, trt.int64: torch.int64, trt.int8: torch.int8, trt.bool: torch.bool, } class TrtRunner: """TRT engine wrapper backed by torch.cuda tensors as I/O buffers.""" def __init__(self, engine_path: Path) -> None: """Deserialize the engine and bind I/O buffers.""" logger = trt.Logger(trt.Logger.WARNING) trt.init_libnvinfer_plugins(logger, "") self.engine = trt.Runtime(logger).deserialize_cuda_engine( engine_path.read_bytes() ) self.ctx = self.engine.create_execution_context() self.stream = torch.cuda.Stream() self.bufs: dict[str, torch.Tensor] = {} self.outs: list[str] = [] for i in range(self.engine.num_io_tensors): name = self.engine.get_tensor_name(i) shape = tuple(self.engine.get_tensor_shape(name)) dtype = _TRT_TORCH[self.engine.get_tensor_dtype(name)] self.bufs[name] = torch.empty(shape, dtype=dtype, device="cuda") self.ctx.set_tensor_address(name, self.bufs[name].data_ptr()) if self.engine.get_tensor_mode(name) == trt.TensorIOMode.OUTPUT: self.outs.append(name) def run(self) -> None: """Execute the engine on whatever is in the input buffers.""" self.ctx.execute_async_v3(self.stream.cuda_stream) self.stream.synchronize() def infer(self, image: torch.Tensor) -> torch.Tensor: """Copy in → execute → return the token tensor on CPU, fp32.""" buf = self.bufs["image"] buf.copy_(image.to(device=buf.device, dtype=buf.dtype)) self.run() return self.bufs[self.outs[0]].detach().float().cpu() fp16_runner = TrtRunner(ENGINE_FP16) int8_runner = TrtRunner(ENGINE_INT8) # %% # Step 5 — Feature inspection # --------------------------- # # Two views of the patch features, FP16 engine vs INT8 engine: # # - **Patch similarity** — pick an anchor patch (the ✕), color every # patch by cosine similarity to it. Semantically matching regions light # up: anchoring on one cat highlights both cats. # - **PCA features** — project the 768-dim patch features onto their top # 3 principal components and render them as RGB. # # .. image:: https://huggingface.co/datasets/embedl/documentation-images/resolve/main/dinov3-quantized-tensorrt/patch_similarity_cats.png # :alt: Anchor-patch cosine-similarity maps for the COCO cats image — # TensorRT FP16 and Embedl INT8 produce near-identical heatmaps. # :width: 100% # # | # # PCA features rendered as RGB — matching colors mean the quantized # model preserves the feature geometry: # # .. image:: https://storage.googleapis.com/embedl-deploy-artifacts/pca_features.png # :alt: Cats-image patch features projected onto their top-3 PCA # components and rendered as RGB blocks; the DINOv3 FP32 and # Embedl INT8 columns are nearly identical. # :width: 100% # import matplotlib.pyplot as plt # type: ignore[import-not-found] from PIL import Image # type: ignore[import-not-found] ANCHOR_YX = (5, 4) # left cat's head, in the 14×14 patch grid def demo_image() -> Image.Image: """Download (once) and open the COCO cats demo image.""" if not DEMO_IMAGE.exists(): urllib.request.urlretrieve(DEMO_URL, DEMO_IMAGE) return Image.open(DEMO_IMAGE).convert("RGB") def patch_grid(tokens: torch.Tensor) -> torch.Tensor: """Reshape ``[1, 201, 768]`` tokens into a ``[14, 14, 768]`` patch grid.""" return tokens[0, PATCH_PREFIX:].reshape(GRID, GRID, -1) def similarity_map(patches: torch.Tensor, yx: tuple[int, int]) -> torch.Tensor: """Compute cosine similarity of every patch to the anchor patch.""" anchor = patches[yx[0], yx[1]] flat = patches.reshape(-1, patches.shape[-1]) return F.cosine_similarity(flat, anchor[None], dim=-1).reshape(GRID, GRID) def pca_rgb( patches: torch.Tensor, basis: torch.Tensor | None = None ) -> tuple[torch.Tensor, torch.Tensor]: """Project the patch features onto their top 3 principal components.""" flat = patches.reshape(-1, patches.shape[-1]) flat = flat - flat.mean(dim=0) if basis is None: _, _, basis = torch.pca_lowrank(flat, q=3) rgb = (flat @ basis).reshape(GRID, GRID, 3) lo = rgb.amin(dim=(0, 1)) hi = rgb.amax(dim=(0, 1)) return (rgb - lo) / (hi - lo).clamp_min(1e-8), basis image = demo_image() crop = transforms.CenterCrop(IMAGE_SIZE)(transforms.Resize(256)(image)) x = EVAL_TF(image).unsqueeze(0) fp16_patches = patch_grid(fp16_runner.infer(x)) int8_patches = patch_grid(int8_runner.infer(x)) fig, axes = plt.subplots(2, 3, figsize=(10.5, 7)) for row in axes: row[0].imshow(crop) ay, ax_ = ANCHOR_YX axes[0, 0].plot( (ax_ + 0.5) * PATCH_SIZE, (ay + 0.5) * PATCH_SIZE, "rx", ms=12, mew=3 ) # Shared color scale so the two heatmaps are directly comparable. sim_kw = {"cmap": "viridis", "vmin": -0.2, "vmax": 1.0} axes[0, 1].imshow(similarity_map(fp16_patches, ANCHOR_YX), **sim_kw) axes[0, 2].imshow(similarity_map(int8_patches, ANCHOR_YX), **sim_kw) rgb_fp16, basis = pca_rgb(fp16_patches) rgb_int8, _ = pca_rgb(int8_patches, basis) axes[1, 1].imshow(rgb_fp16) axes[1, 2].imshow(rgb_int8) axes[0, 0].set_title("Input (✕ = anchor)") for col, name in [(1, "TensorRT FP16"), (2, "Embedl INT8")]: axes[0, col].set_title(f"{name} — similarity") axes[1, col].set_title(f"{name} — PCA") for a in axes.ravel(): a.axis("off") fig.tight_layout() fig.savefig(ARTIFACTS_PATH / "feature_maps.png", dpi=120, bbox_inches="tight") print(f" → {ARTIFACTS_PATH / 'feature_maps.png'}") # %% # Step 6 — Latency and k-NN accuracy # ---------------------------------- # # Benchmark both engines: forward-pass latency via CUDA events, and # k-NN classification accuracy on imagenette against the FP32 reference. N_WARMUP, N_ITERS = 20, 200 def bench(runner: TrtRunner) -> tuple[float, float]: """Compute mean and p95 engine latency in ms over ``N_ITERS`` runs.""" for _ in range(N_WARMUP): runner.run() start = torch.cuda.Event(enable_timing=True) end = torch.cuda.Event(enable_timing=True) samples = [] for _ in range(N_ITERS): start.record() runner.run() end.record() torch.cuda.synchronize() samples.append(start.elapsed_time(end)) samples.sort() return sum(samples) / len(samples), samples[int(0.95 * len(samples)) - 1] fp16_mean, fp16_p95 = bench(fp16_runner) int8_mean, int8_p95 = bench(int8_runner) print("Latency (DINOv3 ViT-B/16 @ 224x224, batch 1)") print(f" TensorRT FP16: mean {fp16_mean:5.2f} ms p95 {fp16_p95:5.2f} ms") print( f" Embedl INT8: mean {int8_mean:5.2f} ms p95 {int8_p95:5.2f} ms " f"({fp16_mean / int8_mean:.2f}x vs FP16)" ) KNN_K, KNN_TAU = 20, 0.07 @torch.no_grad() def cls_features( model_fn: nn.Module, loader: DataLoader[Any] ) -> tuple[torch.Tensor, torch.Tensor]: """Return L2-normalized CLS features (``[N, D]``) and labels (``[N]``).""" model_fn.eval() feats, labels = [], [] for images, targets in loader: hidden = model_fn(images.to(DEVICE)) feats.append(F.normalize(hidden[:, 0, :].float(), dim=-1).cpu()) labels.append(targets) return torch.cat(feats), torch.cat(labels) @torch.no_grad() def cls_features_trt( runner: TrtRunner, ds: datasets.Imagenette ) -> torch.Tensor: """Return CLS features from the batch-1 engine for each image in `ds`.""" feats = [] for image_t, _ in ds: tokens = runner.infer(image_t.unsqueeze(0)) feats.append(F.normalize(tokens[:, 0, :], dim=-1)) return torch.cat(feats) def knn_top1( index_feats: torch.Tensor, index_labels: torch.Tensor, query_feats: torch.Tensor, query_labels: torch.Tensor, num_classes: int, ) -> float: """Compute k-NN top-1 accuracy (%) via weighted exp(sim/tau) votes.""" index_gpu = index_feats.to(DEVICE) correct = 0 for start in range(0, len(query_feats), 256): q = query_feats[start : start + 256].to(DEVICE) top_sim, top_idx = (q @ index_gpu.T).topk(KNN_K, dim=1) votes = torch.zeros(len(q), num_classes, device=DEVICE) votes.scatter_add_( 1, index_labels[top_idx.cpu()].to(DEVICE), (top_sim / KNN_TAU).exp(), ) pred = votes.argmax(dim=1).cpu() correct += int( (pred == query_labels[start : start + 256]).sum().item() ) return 100.0 * correct / len(query_feats) print("FP32 reference: extracting index (train) + query (val) features") fp32 = load_dinov3() index_feats, index_labels = cls_features(fp32, train_loader) val_feats, val_labels = cls_features(fp32, val_loader) del fp32 torch.cuda.empty_cache() n_classes = len(train_ds.classes) fp32_acc = knn_top1( index_feats, index_labels, val_feats, val_labels, n_classes ) print(f" FP32 knn_top1: {fp32_acc:.2f}%") print("INT8 engine: extracting query features") int8_feats = cls_features_trt(int8_runner, val_ds) int8_acc = knn_top1( index_feats, index_labels, int8_feats, val_labels, n_classes ) print( f" INT8 knn_top1: {int8_acc:.2f}% ({int8_acc - fp32_acc:+.2f} vs FP32)" ) # %% # On an NVIDIA L4 with TensorRT 10.16: # # .. code-block:: text # # Latency (DINOv3 ViT-B/16 @ 224x224, batch 1) # TensorRT FP16: mean 1.50 ms p95 1.52 ms # Embedl INT8: mean 1.13 ms p95 1.15 ms (1.34x vs FP16) # FP32 reference: extracting index (train) + query (val) features # FP32 knn_top1: 99.75% # INT8 engine: extracting query features # INT8 knn_top1: 99.26% (-0.48 vs FP32) # # %% # Summary # ------- # # :: # # HuggingFace ─▶ Embedl Deploy fuse + INT8 PTQ ─▶ ONNX (QDQ) # ─▶ TensorRT engine (INT8+FP16) # # DINOv3 ViT-B/16 @ 224×224, batch 1, TensorRT 10.16: # # ============== =========== ================ ===================== # Path NVIDIA L4 Jetson AGX Orin knn_top1 (imagenette) # ============== =========== ================ ===================== # TensorRT FP16 1.50 ms 2.71 ms 99.75 % (FP32 ref) # Embedl INT8 1.13 ms 2.25 ms 99.26 % # ============== =========== ================ ===================== # # The quantized model, engines and full evaluation protocol are published # at `embedl/dinov3-quantized-tensorrt # `_. For the # same recipe on a detection model, see the SAM3 tutorial.