Skip to content

vllm.models.deepseek_v41.common.engram

Engram: n-gram hash lookups gated into the hyper-connection stream.

Port of the reference inference/engram.py + Engram / ParallelEngramEmbedding from inference/model.py (DeepSeek V4.1 checkpoint layout). Engram modules live on the backbone layers listed in engram_layer_ids only.

An n-gram at position p needs the token ids at p-1..p-3 while vLLM streams tokens chunk-by-chunk:

  • token_map: token id -> compressed vocab id, built once from the model's tokenizer at init (deterministic; asserted against engram_compressed_vocab_size).
  • lookback_token_ids: the ids just before each request's chunk start, which the runner reads from its device-resident token history.

Classes:

  • DPSharedEngramStorage –

    Host weights shared by a node-local DP group with one writer: host-NUMA

  • Engram –

    Writes an n-gram lookup into the residual stream, gated by how well it

  • EngramBatch –

    Stage every Engram layer's WKV input as MXFP8 with one exchange.

  • EngramLayout –

    Bucket layout of the n-gram hash tables.

  • EngramPeerGroup –

    GPUs of several engines that split each Engram table over their HBM.

  • NgramHashState –

    Maps each position to the hash ids of the n-grams ending there.

  • ParallelEngramEmbedding –

    Hash heads with FP8 or packed MXFP4 rows and per-block E8M0 scales.

Functions:

DPSharedEngramStorage

Host weights shared by a node-local DP group with one writer: host-NUMA cuMem where FABRIC handles work, else a registered /dev/shm mapping.

Methods:

  • share_shm_table –

    Map and register one /dev/shm file across a node-local DP group.

Source code in vllm/models/deepseek_v41/common/engram.py
class DPSharedEngramStorage:
    """Host weights shared by a node-local DP group with one writer: host-NUMA
    cuMem where FABRIC handles work, else a registered /dev/shm mapping."""

    def __init__(
        self,
        num_rows: int,
        dim: int,
        block_size: int,
        group: GroupCoordinator,
        packed_fp4: bool = False,
    ) -> None:
        self.group = group
        row_bytes = dim // 2 if packed_fp4 else dim
        weight_dtype = torch.uint8 if packed_fp4 else torch.float8_e4m3fn
        weight_bytes = num_rows * row_bytes
        num_bytes = weight_bytes + num_rows * (dim // block_size)

        def split_table(table: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
            weight = table[:weight_bytes].view(weight_dtype).view(-1, row_bytes)
            return weight, table[weight_bytes:].view(-1, dim // block_size)

        views = None
        if current_platform.is_cuda():
            from vllm.models.deepseek_v41.nvidia.ops import engram_fabric_table

            views = engram_fabric_table.share_host_table(num_bytes, group)
        # The GPU maps cuMem host memory with 2M pages, /dev/shm with 64K pages.
        self.huge_pages = views is not None
        host, gpu = views or self.share_shm_table(num_bytes, group)
        self.weight, self.weight_scale_inv = split_table(host)
        self._views = split_table(gpu)

    @staticmethod
    def share_shm_table(
        num_bytes: int, group: GroupCoordinator
    ) -> tuple[torch.Tensor, torch.Tensor]:
        """Map and register one /dev/shm file across a node-local DP group.

        Returns CPU and GPU views of the file. Raises on every rank, leaving
        nothing mapped, if any rank fails.
        """
        with ExitStack() as stack:
            path = error = None
            if group.rank_in_group == 0 and not (error := _shm_shortfall(num_bytes)):
                try:
                    backing_file = stack.enter_context(
                        tempfile.NamedTemporaryFile(prefix="vllm_engram_", dir=SHM_PATH)
                    )
                    backing_file.truncate(num_bytes)
                    path = backing_file.name
                except Exception as exc:
                    error = describe_error(exc)
            path, error = group.broadcast_object((path, error))
            if error is not None:
                raise RuntimeError(
                    "Engram shared-memory creation failed on EDP rank 0: " + error
                )

            mapping = None
            try:
                with open(path, "r+b") as file:
                    mapping = mmap.mmap(file.fileno(), num_bytes, flags=mmap.MAP_SHARED)
            except Exception as exc:
                error = describe_error(exc)
            if failures := gather_errors(group, error):
                del mapping
                raise RuntimeError(
                    "Engram shared-memory initialization failed: " + failures
                )
            # Unlink while the file is still sparse, so a crash during the slow
            # registration below cannot orphan the table.
            stack.close()
            assert mapping is not None

            owner = tensor = None
            try:
                owner = np.frombuffer(mapping, dtype=np.uint8)
                pointer = owner.ctypes.data
                _allow_any_numa_node(pointer, num_bytes)
                tensor = torch.from_numpy(owner)
                result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0)
                if result.value != 0:
                    raise RuntimeError(f"cudaHostRegister failed: {result}")
                finalizer = weakref.finalize(
                    owner, DPSharedEngramStorage._unregister, mapping, pointer
                )
                finalizer.atexit = False  # type: ignore[misc]
                # The UVA helper otherwise allocates a private pinned copy.
                if not tensor.is_pinned():
                    raise RuntimeError(
                        "cudaHostRegister did not pin the shared Engram mapping"
                    )
            except Exception as exc:
                error = describe_error(exc)

            if failures := gather_errors(group, error):
                # Dropping the owner runs the finalizer; the mapping then unmaps.
                del tensor, owner, mapping
                raise RuntimeError(
                    "Engram shared-memory initialization failed: " + failures
                )
            assert tensor is not None
            return tensor, get_accelerator_view_from_cpu_tensor(tensor)

    @staticmethod
    def _unregister(mapping: mmap.mmap, pointer: int) -> None:
        # Torch storage retains the numpy owner, including through cached UVA views.
        # Keep its mmap alive until CUDA has released the registration.
        result = torch.cuda.cudart().cudaHostUnregister(pointer)
        if result.value != 0:
            logger.warning("Engram cudaHostUnregister failed: %s", result)

    def load_weight(
        self, param: torch.nn.Parameter, loaded_weight: torch.Tensor
    ) -> None:
        if self.group.rank_in_group == 0:
            _engram_head_shard_weight_loader(param, loaded_weight)
        # Read order may differ across ranks. Equal load counts ensure all shared
        # weights are ready after the last weight-loader call returns.
        torch.distributed.barrier(group=self.group.cpu_group)

    def get_views(
        self, weight: torch.Tensor, scales: torch.Tensor
    ) -> tuple[torch.Tensor, torch.Tensor]:
        if (weight.data_ptr(), scales.data_ptr()) != (
            self.weight.data_ptr(),
            self.weight_scale_inv.data_ptr(),
        ):
            raise RuntimeError("Shared Engram parameter storage must not be replaced")
        return self._views

share_shm_table(num_bytes, group) staticmethod

Map and register one /dev/shm file across a node-local DP group.

Returns CPU and GPU views of the file. Raises on every rank, leaving nothing mapped, if any rank fails.

Source code in vllm/models/deepseek_v41/common/engram.py
@staticmethod
def share_shm_table(
    num_bytes: int, group: GroupCoordinator
) -> tuple[torch.Tensor, torch.Tensor]:
    """Map and register one /dev/shm file across a node-local DP group.

    Returns CPU and GPU views of the file. Raises on every rank, leaving
    nothing mapped, if any rank fails.
    """
    with ExitStack() as stack:
        path = error = None
        if group.rank_in_group == 0 and not (error := _shm_shortfall(num_bytes)):
            try:
                backing_file = stack.enter_context(
                    tempfile.NamedTemporaryFile(prefix="vllm_engram_", dir=SHM_PATH)
                )
                backing_file.truncate(num_bytes)
                path = backing_file.name
            except Exception as exc:
                error = describe_error(exc)
        path, error = group.broadcast_object((path, error))
        if error is not None:
            raise RuntimeError(
                "Engram shared-memory creation failed on EDP rank 0: " + error
            )

        mapping = None
        try:
            with open(path, "r+b") as file:
                mapping = mmap.mmap(file.fileno(), num_bytes, flags=mmap.MAP_SHARED)
        except Exception as exc:
            error = describe_error(exc)
        if failures := gather_errors(group, error):
            del mapping
            raise RuntimeError(
                "Engram shared-memory initialization failed: " + failures
            )
        # Unlink while the file is still sparse, so a crash during the slow
        # registration below cannot orphan the table.
        stack.close()
        assert mapping is not None

        owner = tensor = None
        try:
            owner = np.frombuffer(mapping, dtype=np.uint8)
            pointer = owner.ctypes.data
            _allow_any_numa_node(pointer, num_bytes)
            tensor = torch.from_numpy(owner)
            result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0)
            if result.value != 0:
                raise RuntimeError(f"cudaHostRegister failed: {result}")
            finalizer = weakref.finalize(
                owner, DPSharedEngramStorage._unregister, mapping, pointer
            )
            finalizer.atexit = False  # type: ignore[misc]
            # The UVA helper otherwise allocates a private pinned copy.
            if not tensor.is_pinned():
                raise RuntimeError(
                    "cudaHostRegister did not pin the shared Engram mapping"
                )
        except Exception as exc:
            error = describe_error(exc)

        if failures := gather_errors(group, error):
            # Dropping the owner runs the finalizer; the mapping then unmaps.
            del tensor, owner, mapping
            raise RuntimeError(
                "Engram shared-memory initialization failed: " + failures
            )
        assert tensor is not None
        return tensor, get_accelerator_view_from_cpu_tensor(tensor)

Engram

Bases: Module

Writes an n-gram lookup into the residual stream, gated by how well it matches that stream.

The hash ids fetch n_hash_cols rows; wkv turns them into one key per hc copy plus a shared value. The gate is a normalized dot product of the stream against the key, signed-sqrt'ed before the sigmoid (matching the training kernel).

Methods:

  • embed –

    Gather heads, returning only local tokens when SP is enabled.

  • forward –

    hidden_states: [T, hc_mult, dim]; hash_ids: [T, n_hash_cols] (all

  • prepare_embeddings –

    Look up, or prefetch from host memory, rows before the decoder layers

Source code in vllm/models/deepseek_v41/common/engram.py
class Engram(nn.Module):
    """Writes an n-gram lookup into the residual stream, gated by how well it
    matches that stream.

    The hash ids fetch `n_hash_cols` rows; `wkv` turns them into one key per
    hc copy plus a shared value. The gate is a normalized dot product of the
    stream against the key, signed-sqrt'ed before the sigmoid (matching the
    training kernel).
    """

    _prefetch_stream: torch.cuda.Stream | None = None
    _prefetch_done: torch.cuda.Event | None = None
    _batch: EngramBatch | None = None

    def __init__(
        self,
        config,
        quant_config: QuantizationConfig | None,
        layout: EngramLayout,
        layer_hash_index: int,
        use_sequence_parallel: bool,
        prefix: str,
        prefetch_stream: torch.cuda.Stream | None = None,
    ) -> None:
        super().__init__()
        # Layers sharing one stream serialize their offloaded lookups, so they
        # take turns instead of jointly starving decoder compute of SMs.
        self._prefetch_stream = prefetch_stream
        self.layer_hash_index = layer_hash_index
        self.dim = config.hidden_size
        self.hc_mult = config.hc_mult
        self.eps = config.rms_norm_eps
        self.clamp_value = 1e-6
        self.use_sequence_parallel = use_sequence_parallel

        # Named ``embed_tokens`` so the checkpoint's ``engram.embed.weight``
        # survives the mapper's ``embed.weight`` -> ``embed_tokens.weight``
        # suffix rule.
        self.embed_tokens = self._create_embedding(layout, layer_hash_index)
        n_hash_cols = (layout.max_ngram_size - 1) * layout.n_heads
        self.wkv = self._create_wkv(
            n_hash_cols * layout.head_dim, quant_config, f"{prefix}.wkv"
        )
        self.q_weight = nn.Parameter(
            torch.empty(self.hc_mult, self.dim, dtype=torch.bfloat16),
            requires_grad=False,
        )
        self.k_weight = nn.Parameter(
            torch.empty(self.hc_mult, self.dim, dtype=torch.bfloat16),
            requires_grad=False,
        )

        max_tokens = get_current_vllm_config().scheduler_config.max_num_batched_tokens
        self._init_staging(max_tokens, layout.head_dim)

    def _create_wkv(
        self, input_size: int, quant_config: QuantizationConfig | None, prefix: str
    ) -> ColumnParallelLinear:
        # Without sequence parallelism every TP rank holds every token, so
        # shard the output columns instead of replicating the projection.
        return ColumnParallelLinear(
            input_size,
            self.dim * (self.hc_mult + 1),
            bias=False,
            gather_output=True,
            quant_config=quant_config,
            return_bias=False,
            prefix=prefix,
            disable_tp=self.use_sequence_parallel,
        )

    def _create_embedding(
        self, layout: EngramLayout, layer_hash_index: int
    ) -> ParallelEngramEmbedding:
        engram_config = get_current_vllm_config().engram_config
        assert engram_config is not None
        peer_group = None
        if engram_config.peer_group_store is not None:
            peer_group = join_engram_peer_group(
                engram_config.peer_group_size, engram_config.peer_group_store
            )
        return ParallelEngramEmbedding(
            layout.num_embeddings[layer_hash_index],
            layout.head_dim,
            tuple(size for order in layout.primes[layer_hash_index] for size in order),
            cpu_offload=engram_config.cpu_offload,
            dp_shared_memory=bool(engram_config.dp_shared_memory),
            use_thp=engram_config.use_thp,
            packed_fp4=layout.packed_fp4,
            peer_group=peer_group,
        )

    def _init_staging(self, max_tokens: int, head_dim: int) -> None:
        # Persistent storage keeps lookup addresses stable across graph replays.
        self.staged_rows = torch.empty(
            max_tokens * self.embed_tokens.dp_size,
            self.embed_tokens.part_n_hash_cols,
            head_dim,
            dtype=torch.bfloat16,
        )
        if not self.embed_tokens.cpu_offload:
            self._prefetch_stream = None
            return
        if self._prefetch_stream is None:
            self._prefetch_stream = torch.cuda.Stream(device=self.staged_rows.device)
        self._prefetch_done = torch.cuda.Event()

    def prepare_embeddings(self, hash_ids: torch.Tensor) -> None:
        """Look up, or prefetch from host memory, rows before the decoder layers
        consume them; DP-sharded tables take the group's gathered hash IDs."""
        rows = self.staged_rows[: hash_ids.shape[0]]
        assert rows.shape[0] == hash_ids.shape[0], "engram staging buffer too small"
        if self._prefetch_stream is None:
            self.embed_tokens.lookup(hash_ids, rows)
        else:
            self._start_prefetch(hash_ids, rows)

    @eager_break_during_capture
    def _start_prefetch(self, hash_ids: torch.Tensor, rows: torch.Tensor) -> None:
        # Eager boundaries let the lookup span piecewise graph segments, and
        # decide per replay (on the replay stream) whether it runs inline.
        num_tokens = hash_ids.shape[0]
        # All DP ranks must agree, or EP collectives wait on the slowest one.
        if is_forward_context_available() and (dp := get_forward_context().dp_metadata):
            num_tokens = int(dp.num_tokens_across_dp_cpu.max())
        current = stream = torch.cuda.current_stream()
        # Big lookups stall persistent main-stream kernels anyway: run them inline.
        _, inline_min_tokens = _engram_lookup_thresholds(hash_ids.device)
        if inline_min_tokens is not None and num_tokens >= inline_min_tokens:
            self.embed_tokens.lookup(hash_ids, rows)
        else:
            stream = self._prefetch_stream
            assert stream is not None
            stream.wait_stream(current)
            # Keep temporary hash storage alive until lookup finishes reading it.
            hash_ids.record_stream(stream)
            with torch.cuda.stream(stream):
                self.embed_tokens.lookup(hash_ids, rows, background=True)
        assert self._prefetch_done is not None
        self._prefetch_done.record(stream)

    @eager_break_during_capture
    def _finish_prefetch(self, event: torch.cuda.Event) -> None:
        # Wait for this layer's lookup only, not for later layers on the stream.
        torch.cuda.current_stream().wait_event(event)

    def _ready_rows(self, num_tokens: int) -> torch.Tensor:
        if self._prefetch_done is not None:
            self._finish_prefetch(self._prefetch_done)
        if self.embed_tokens.dp_size > 1:
            slot = engram_gathered_num_tokens()
            staged = self.staged_rows[: slot * self.embed_tokens.dp_size]
            return _gather_engram_rows(staged, num_tokens)
        return self.staged_rows[:num_tokens]

    def embed(self, hash_ids: torch.Tensor) -> torch.Tensor:
        """Gather heads, returning only local tokens when SP is enabled."""
        rows = self._ready_rows(hash_ids.shape[0])
        if self.embed_tokens.tp_size == 1:
            return rows[:, : self.embed_tokens.n_hash_cols]
        if self.use_sequence_parallel:
            tp_size = self.embed_tokens.tp_size
            num_tokens, local_heads, dim = rows.shape
            gathered = tensor_model_parallel_all_gather(rows, dim=0)
            chunk = (num_tokens + tp_size - 1) // tp_size
            out = rows.new_empty((chunk, self.embed_tokens.n_hash_cols, dim))
            _engram_select_rows(
                gathered,
                out,
                num_tokens,
                get_tensor_model_parallel_rank() * chunk,
                local_heads * dim,
            )
            return out
        rows = tensor_model_parallel_all_gather(rows, dim=1)
        return rows[:, : self.embed_tokens.n_hash_cols]

    def forward(
        self,
        hidden_states: torch.Tensor,
        hash_ids: torch.Tensor,
        token_mask: torch.Tensor | None = None,
    ) -> torch.Tensor:
        """hidden_states: [T, hc_mult, dim]; hash_ids: [T, n_hash_cols] (all
        tokens, pre sequence-parallel shard); token_mask: [T], False shuts
        the gate so those positions pass through untouched."""
        if self._batch is not None:
            kv = self.wkv(self._batch.wkv_input(self, hash_ids.shape[0]))
        else:
            kv = self.wkv(self.embed(hash_ids).flatten(-2))
        num_kv_tokens = hash_ids.shape[0]
        assert token_mask is None or token_mask.shape == (num_kv_tokens,)
        if self.use_sequence_parallel:
            tp_size = get_tensor_model_parallel_world_size()
            tp_rank = get_tensor_model_parallel_rank()
            shard_size = (num_kv_tokens + tp_size - 1) // tp_size
            assert hidden_states.shape[0] == shard_size
            start = min(tp_rank * shard_size, num_kv_tokens)
            num_kv_tokens = min(shard_size, num_kv_tokens - start)
            if token_mask is not None:
                token_mask = token_mask[start : start + num_kv_tokens]

        return self._post_wkv(hidden_states, kv, token_mask, num_kv_tokens)

    def _post_wkv(
        self,
        hidden_states: torch.Tensor,
        kv: torch.Tensor,
        token_mask: torch.Tensor | None,
        num_kv_tokens: int,
    ) -> torch.Tensor:
        """The gated update from the `wkv` output of `num_kv_tokens` tokens."""
        num_tokens, hc_mult, dim = hidden_states.shape
        assert hc_mult == self.hc_mult and dim == self.dim
        assert kv.ndim == 2 and kv.shape[1] == (hc_mult + 1) * dim
        output = torch.empty_like(hidden_states)
        if num_tokens == 0:
            return output

        block_size = triton.next_power_of_2(dim)
        num_warps = 8 if block_size >= 2048 else 4
        mask = token_mask if token_mask is not None else hidden_states
        _fused_engram_post_wkv_kernel[(num_tokens * hc_mult,)](
            hidden_states,
            kv,
            self.q_weight,
            self.k_weight,
            mask,
            output,
            num_kv_tokens,
            hidden_states.stride(0),
            hidden_states.stride(1),
            hidden_states.stride(2),
            kv.stride(0),
            kv.stride(1),
            self.q_weight.stride(0),
            self.q_weight.stride(1),
            self.k_weight.stride(0),
            self.k_weight.stride(1),
            token_mask.stride(0) if token_mask is not None else 0,
            output.stride(0),
            output.stride(1),
            output.stride(2),
            self.eps,
            self.clamp_value,
            DIM=dim,
            HC_MULT=hc_mult,
            BLOCK_SIZE=block_size,
            HAS_MASK=token_mask is not None,
            num_warps=num_warps,
        )
        return output

_post_wkv(hidden_states, kv, token_mask, num_kv_tokens)

The gated update from the wkv output of num_kv_tokens tokens.

Source code in vllm/models/deepseek_v41/common/engram.py
def _post_wkv(
    self,
    hidden_states: torch.Tensor,
    kv: torch.Tensor,
    token_mask: torch.Tensor | None,
    num_kv_tokens: int,
) -> torch.Tensor:
    """The gated update from the `wkv` output of `num_kv_tokens` tokens."""
    num_tokens, hc_mult, dim = hidden_states.shape
    assert hc_mult == self.hc_mult and dim == self.dim
    assert kv.ndim == 2 and kv.shape[1] == (hc_mult + 1) * dim
    output = torch.empty_like(hidden_states)
    if num_tokens == 0:
        return output

    block_size = triton.next_power_of_2(dim)
    num_warps = 8 if block_size >= 2048 else 4
    mask = token_mask if token_mask is not None else hidden_states
    _fused_engram_post_wkv_kernel[(num_tokens * hc_mult,)](
        hidden_states,
        kv,
        self.q_weight,
        self.k_weight,
        mask,
        output,
        num_kv_tokens,
        hidden_states.stride(0),
        hidden_states.stride(1),
        hidden_states.stride(2),
        kv.stride(0),
        kv.stride(1),
        self.q_weight.stride(0),
        self.q_weight.stride(1),
        self.k_weight.stride(0),
        self.k_weight.stride(1),
        token_mask.stride(0) if token_mask is not None else 0,
        output.stride(0),
        output.stride(1),
        output.stride(2),
        self.eps,
        self.clamp_value,
        DIM=dim,
        HC_MULT=hc_mult,
        BLOCK_SIZE=block_size,
        HAS_MASK=token_mask is not None,
        num_warps=num_warps,
    )
    return output

embed(hash_ids)

Gather heads, returning only local tokens when SP is enabled.

Source code in vllm/models/deepseek_v41/common/engram.py
def embed(self, hash_ids: torch.Tensor) -> torch.Tensor:
    """Gather heads, returning only local tokens when SP is enabled."""
    rows = self._ready_rows(hash_ids.shape[0])
    if self.embed_tokens.tp_size == 1:
        return rows[:, : self.embed_tokens.n_hash_cols]
    if self.use_sequence_parallel:
        tp_size = self.embed_tokens.tp_size
        num_tokens, local_heads, dim = rows.shape
        gathered = tensor_model_parallel_all_gather(rows, dim=0)
        chunk = (num_tokens + tp_size - 1) // tp_size
        out = rows.new_empty((chunk, self.embed_tokens.n_hash_cols, dim))
        _engram_select_rows(
            gathered,
            out,
            num_tokens,
            get_tensor_model_parallel_rank() * chunk,
            local_heads * dim,
        )
        return out
    rows = tensor_model_parallel_all_gather(rows, dim=1)
    return rows[:, : self.embed_tokens.n_hash_cols]

forward(hidden_states, hash_ids, token_mask=None)

hidden_states: [T, hc_mult, dim]; hash_ids: [T, n_hash_cols] (all tokens, pre sequence-parallel shard); token_mask: [T], False shuts the gate so those positions pass through untouched.

Source code in vllm/models/deepseek_v41/common/engram.py
def forward(
    self,
    hidden_states: torch.Tensor,
    hash_ids: torch.Tensor,
    token_mask: torch.Tensor | None = None,
) -> torch.Tensor:
    """hidden_states: [T, hc_mult, dim]; hash_ids: [T, n_hash_cols] (all
    tokens, pre sequence-parallel shard); token_mask: [T], False shuts
    the gate so those positions pass through untouched."""
    if self._batch is not None:
        kv = self.wkv(self._batch.wkv_input(self, hash_ids.shape[0]))
    else:
        kv = self.wkv(self.embed(hash_ids).flatten(-2))
    num_kv_tokens = hash_ids.shape[0]
    assert token_mask is None or token_mask.shape == (num_kv_tokens,)
    if self.use_sequence_parallel:
        tp_size = get_tensor_model_parallel_world_size()
        tp_rank = get_tensor_model_parallel_rank()
        shard_size = (num_kv_tokens + tp_size - 1) // tp_size
        assert hidden_states.shape[0] == shard_size
        start = min(tp_rank * shard_size, num_kv_tokens)
        num_kv_tokens = min(shard_size, num_kv_tokens - start)
        if token_mask is not None:
            token_mask = token_mask[start : start + num_kv_tokens]

    return self._post_wkv(hidden_states, kv, token_mask, num_kv_tokens)

prepare_embeddings(hash_ids)

Look up, or prefetch from host memory, rows before the decoder layers consume them; DP-sharded tables take the group's gathered hash IDs.

Source code in vllm/models/deepseek_v41/common/engram.py
def prepare_embeddings(self, hash_ids: torch.Tensor) -> None:
    """Look up, or prefetch from host memory, rows before the decoder layers
    consume them; DP-sharded tables take the group's gathered hash IDs."""
    rows = self.staged_rows[: hash_ids.shape[0]]
    assert rows.shape[0] == hash_ids.shape[0], "engram staging buffer too small"
    if self._prefetch_stream is None:
        self.embed_tokens.lookup(hash_ids, rows)
    else:
        self._start_prefetch(hash_ids, rows)

EngramBatch

Stage every Engram layer's WKV input as MXFP8 with one exchange.

Methods:

  • create –

    Batch DP-sharded tables whose WKV takes FlashInfer MXFP8 input.

  • prepare_embeddings –

    Stage every layer's WKV input before the decoder layers.

Source code in vllm/models/deepseek_v41/common/engram.py
class EngramBatch:
    """Stage every Engram layer's WKV input as MXFP8 with one exchange."""

    def __init__(self, layers: list["Engram"], max_tokens: int) -> None:
        first = layers[0].layer_hash_index
        hash_indices = [layer.layer_hash_index for layer in layers]
        assert hash_indices == list(range(first, first + len(layers)))
        self.layers = layers
        table = layers[0].embed_tokens
        device = layers[0].staged_rows.device
        self.packed_rows = torch.empty(
            max_tokens * table.dp_size,
            len(layers),
            table.part_n_hash_cols,
            table.dim + table.dim // 32,
            dtype=torch.uint8,
            device=device,
        )
        # Same allocator as the send buffer: NCCL hangs this all-to-all when its
        # buffers mix the cumem pool and the caching allocator.
        self.exchanged_rows = torch.empty_like(self.packed_rows)
        width = table.n_hash_cols * table.dim
        self.fp8_values = torch.empty(
            len(layers), max_tokens, width, dtype=torch.uint8, device=device
        )
        self.fp8_scales = torch.empty(
            len(layers),
            mxfp8_scale_bytes(max_tokens, width),
            dtype=torch.uint8,
            device=device,
        )
        max_peer_slot = min(max_tokens, ENGRAM_A2A_MIN_SLOT - 1)
        self.peer_exchange = None
        if table.cpu_offload:
            from vllm.models.deepseek_v41.nvidia.ops.engram_peer import (
                EngramPeerExchange,
            )

            self.peer_exchange = EngramPeerExchange.create(layers, max_peer_slot)
        for layer in layers:
            layer._batch = self
            # The layers read this batch's staging instead of their BF16 rows.
            layer.staged_rows = layer.staged_rows.new_empty(
                (0, *layer.staged_rows.shape[1:])
            )

    @classmethod
    def create(cls, layers: list["Engram"], max_tokens: int) -> "EngramBatch | None":
        """Batch DP-sharded tables whose WKV takes FlashInfer MXFP8 input."""
        if not layers:
            return None
        table = layers[0].embed_tokens
        if (
            table.dp_size == 1
            or any(layer.embed_tokens.packed_fp4 for layer in layers)
            or table.tp_size > 1
            or table.dim % 128
            or get_current_vllm_config().lora_config is not None
            or not can_fuse_query_quant([layer.wkv for layer in layers])
        ):
            return None
        return cls(layers, max_tokens)

    def prepare_embeddings(self, hashes: torch.Tensor) -> None:
        """Stage every layer's WKV input before the decoder layers.

        DP-sharded tables decide on the group's slot, so every rank agrees.
        """
        slot = engram_gathered_num_tokens()
        peer = self.peer_exchange is not None and slot < ENGRAM_A2A_MIN_SLOT
        self._stage(hashes if peer else gather_engram_hashes(hashes), slot, peer)

    @eager_break_during_capture
    def _stage(self, hashes: torch.Tensor, slot: int, peer: bool) -> None:
        if slot == 0:
            return
        if peer:
            assert self.peer_exchange is not None
            rows = self.peer_exchange.exchange(hashes, slot)
            rank = self.peer_exchange.dp_rank
        else:
            rows, rank = self.packed_rows[: hashes.shape[0]], 0
            for index, layer in enumerate(self.layers):
                layer.embed_tokens.lookup(
                    hashes[:, layer.layer_hash_index], rows[:, index]
                )
            group = get_engram_dp_group()
            assert group is not None
            sent, rows = rows, self.exchanged_rows[: rows.shape[0]]
            torch.distributed.all_to_all_single(rows, sent, group=group.device_group)
        table = self.layers[0].embed_tokens
        heads = table.n_hash_cols
        grid = (triton.cdiv(slot, 128) * 128 * heads // 16, len(self.layers))
        _engram_unpack_fp8_kernel[grid](
            rows,
            self.fp8_values,
            self.fp8_scales,
            slot,
            self.fp8_values.stride(0),
            self.fp8_scales.stride(0),
            HEADS=heads,
            LOCAL_HEADS=table.part_n_hash_cols,
            LAYERS=len(self.layers),
            RANK=rank,
            DIM=table.dim,
            BLOCK=16,
            PEER=peer,
        )

    def wkv_input(self, layer: "Engram", num_tokens: int) -> QuantizedActivation:
        index = layer.layer_hash_index - self.layers[0].layer_hash_index
        values = self.fp8_values[index, :num_tokens].view(torch.float8_e4m3fn)
        return QuantizedActivation(
            values,
            self.fp8_scales[index, : mxfp8_scale_bytes(num_tokens, values.shape[1])],
            torch.bfloat16,
            values.shape,
            kMxfp8Dynamic,
        )

create(layers, max_tokens) classmethod

Batch DP-sharded tables whose WKV takes FlashInfer MXFP8 input.

Source code in vllm/models/deepseek_v41/common/engram.py
@classmethod
def create(cls, layers: list["Engram"], max_tokens: int) -> "EngramBatch | None":
    """Batch DP-sharded tables whose WKV takes FlashInfer MXFP8 input."""
    if not layers:
        return None
    table = layers[0].embed_tokens
    if (
        table.dp_size == 1
        or any(layer.embed_tokens.packed_fp4 for layer in layers)
        or table.tp_size > 1
        or table.dim % 128
        or get_current_vllm_config().lora_config is not None
        or not can_fuse_query_quant([layer.wkv for layer in layers])
    ):
        return None
    return cls(layers, max_tokens)

prepare_embeddings(hashes)

Stage every layer's WKV input before the decoder layers.

DP-sharded tables decide on the group's slot, so every rank agrees.

Source code in vllm/models/deepseek_v41/common/engram.py
def prepare_embeddings(self, hashes: torch.Tensor) -> None:
    """Stage every layer's WKV input before the decoder layers.

    DP-sharded tables decide on the group's slot, so every rank agrees.
    """
    slot = engram_gathered_num_tokens()
    peer = self.peer_exchange is not None and slot < ENGRAM_A2A_MIN_SLOT
    self._stage(hashes if peer else gather_engram_hashes(hashes), slot, peer)

EngramLayout

Bucket layout of the n-gram hash tables.

A position is hashed as max_ngram_size - 1 n-grams (2-gram .. max), each split over n_heads heads. Every (n-gram size, head) pair owns its own prime-sized bucket range in the layer's table; the primes are drawn in order and never reused, which keeps the ranges disjoint.

Source code in vllm/models/deepseek_v41/common/engram.py
class EngramLayout:
    """Bucket layout of the n-gram hash tables.

    A position is hashed as `max_ngram_size - 1` n-grams (2-gram .. max), each
    split over `n_heads` heads. Every (n-gram size, head) pair owns its own
    prime-sized bucket range in the layer's table; the primes are drawn in
    order and never reused, which keeps the ranges disjoint.
    """

    def __init__(self, config) -> None:
        self.layer_ids: tuple[int, ...] = tuple(config.engram_layer_ids)
        self.num_embeddings: tuple[int, ...] = tuple(config.engram_num_embeddings)
        self.max_ngram_size: int = config.engram_max_ngram_size
        self.n_heads: int = config.engram_n_heads
        self.head_dim: int = config.engram_head_dim
        self.compressed_vocab_size: int = config.engram_compressed_vocab_size
        self.pad_token_id: int = config.engram_pad_token_id
        assert len(self.layer_ids) == len(self.num_embeddings)

        quant = getattr(config, "quantization_config", None) or {}
        embed_quant = quant.get("engram_embed_quant", "fp8")
        if embed_quant not in ("fp8", "mxfp4"):
            raise ValueError(f"Unsupported engram_embed_quant {embed_quant!r}")
        for key, expected in (
            ("engram_embed_block_size", 32),
            ("engram_embed_scale_dtype", "ue8m0"),
        ):
            if quant.get(key, expected) != expected:
                raise ValueError(f"{key} must be {expected!r}")
        embed_layers = quant.get("engram_embed_layers", self.layer_ids)
        if set(embed_layers) != set(self.layer_ids):
            raise ValueError("engram_embed_layers must match engram_layer_ids")
        self.packed_fp4 = embed_quant == "mxfp4"
        self.row_bytes = self.head_dim // 2 if self.packed_fp4 else self.head_dim

        primes = []
        seen: set[int] = set()
        for _ in self.layer_ids:
            per_ngram = []
            for _ in range(self.max_ngram_size - 1):
                sizes, current = [], config.engram_vocab_size - 1
                for _ in range(self.n_heads):
                    current = find_next_prime(current, seen)
                    seen.add(current)
                    sizes.append(current)
                per_ngram.append(tuple(sizes))
            primes.append(tuple(per_ngram))
        self.primes: tuple[tuple[tuple[int, ...], ...], ...] = tuple(primes)
        self.n_hash_cols = (self.max_ngram_size - 1) * self.n_heads
        flat = [[p for per_ngram in layer for p in per_ngram] for layer in primes]
        offsets = [np.cumsum([0, *sizes[:-1]]) for sizes in flat]
        self.offsets = torch.tensor(np.array(offsets))  # [n_layers, n_hash_cols]

    @classmethod
    def from_config(cls, config) -> "EngramLayout | None":
        if not getattr(config, "engram_layer_ids", None):
            return None
        return cls(config)

EngramPeerGroup

GPUs of several engines that split each Engram table over their HBM.

They meet only at startup, through a file store; lookups then read peers' rows directly, so no engine's step ever waits on another's. Every GPU builds and loads the same tables in the same order, so its n-th exchange is every peer's n-th.

Methods:

  • all_gather –

    Every GPU's value for the next exchange in rank order, once all have

Source code in vllm/models/deepseek_v41/common/engram.py
class EngramPeerGroup:
    """GPUs of several engines that split each Engram table over their HBM.

    They meet only at startup, through a file store; lookups then read peers'
    rows directly, so no engine's step ever waits on another's. Every GPU
    builds and loads the same tables in the same order, so its n-th exchange
    is every peer's n-th.
    """

    # Engines load their weights at their own pace before the group can serve.
    TIMEOUT = timedelta(minutes=30)

    def __init__(self, size: int, path: str) -> None:
        self.world_size = size
        os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
        self.store = torch.distributed.FileStore(path, size)
        self.store.set_timeout(self.TIMEOUT)
        # Any assignment works: a rank only decides which rows a GPU holds.
        self.rank_in_group = self.store.add("next_rank", 1) - 1
        if self.rank_in_group >= size:
            raise RuntimeError(
                f"More than {size} GPUs joined the Engram peer group at {path}; "
                "check peer_group_size and use a fresh store path per launch"
            )
        self._step = 0

    def all_gather(self, value: bytes) -> list[bytes]:
        """Every GPU's value for the next exchange in rank order, once all have
        set theirs."""
        self._step += 1
        key = f"step{self._step}"
        self.store.set(f"{key}/{self.rank_in_group}", value)
        keys = [f"{key}/{rank}" for rank in range(self.world_size)]
        try:
            self.store.wait(keys)
        except RuntimeError as exc:
            missing = [
                rank for rank, name in enumerate(keys) if not self.store.check([name])
            ]
            raise TimeoutError(
                f"Engram peer GPUs {missing} did not reach startup step "
                f"{self._step} within {self.TIMEOUT}; every engine of the group "
                "must start, and none may restart alone"
            ) from exc
        return [self.store.get(name) for name in keys]

    def barrier(self) -> None:
        self.all_gather(b"")

all_gather(value)

Every GPU's value for the next exchange in rank order, once all have set theirs.

Source code in vllm/models/deepseek_v41/common/engram.py
def all_gather(self, value: bytes) -> list[bytes]:
    """Every GPU's value for the next exchange in rank order, once all have
    set theirs."""
    self._step += 1
    key = f"step{self._step}"
    self.store.set(f"{key}/{self.rank_in_group}", value)
    keys = [f"{key}/{rank}" for rank in range(self.world_size)]
    try:
        self.store.wait(keys)
    except RuntimeError as exc:
        missing = [
            rank for rank, name in enumerate(keys) if not self.store.check([name])
        ]
        raise TimeoutError(
            f"Engram peer GPUs {missing} did not reach startup step "
            f"{self._step} within {self.TIMEOUT}; every engine of the group "
            "must start, and none may restart alone"
        ) from exc
    return [self.store.get(name) for name in keys]

NgramHashState

Bases: Module

Maps each position to the hash ids of the n-grams ending there.

Stateless: the runner supplies every lookback token id.

Methods:

  • dummy_hashes –

    Participate in DP lookups without valid rows.

  • ensure_cache –

    Return False while the KV cache is unbound (profile run); the caller

  • forward –

    Compute [tokens, layers, hash columns] int32 n-gram hashes.

Source code in vllm/models/deepseek_v41/common/engram.py
class NgramHashState(nn.Module):
    """Maps each position to the hash ids of the n-grams ending there.

    Stateless: the runner supplies every lookback token id.
    """

    def __init__(
        self,
        vllm_config: VllmConfig,
        layout: EngramLayout,
        swa_cache_module: nn.Module,
    ) -> None:
        super().__init__()
        self.layout = layout
        self.swa_cache_module = swa_cache_module
        self.lookback_depth: int = layout.max_ngram_size - 1

        model_config = vllm_config.model_config
        from transformers import AutoTokenizer

        tokenizer = AutoTokenizer.from_pretrained(
            model_config.tokenizer,
            trust_remote_code=model_config.trust_remote_code,
            revision=model_config.revision,
        )
        token_map, vocab_size = build_compressed_token_map(tokenizer)
        if vocab_size != layout.compressed_vocab_size:
            raise ValueError(
                f"Compressed vocab size mismatch: built {vocab_size} from the "
                f"tokenizer, config expects {layout.compressed_vocab_size}; "
                "every hash multiplier derives from it, so the engram tables "
                "would be silently rehashed."
            )
        self.pad_id = token_map[layout.pad_token_id]
        multipliers = compute_hash_multipliers(
            layout.layer_ids, layout.max_ngram_size, vocab_size
        )
        self.register_buffer(
            "token_map", torch.tensor(token_map, dtype=torch.int32), persistent=False
        )
        self.register_buffer("primes", torch.tensor(layout.primes), persistent=False)
        self.register_buffer("offsets", layout.offsets, persistent=False)
        self.register_buffer("multipliers", multipliers, persistent=False)
        logger.info(
            "Built engram token map (%d -> %d ids) for layers %s",
            len(token_map),
            vocab_size,
            layout.layer_ids,
        )

    def ensure_cache(self) -> bool:
        """Return False while the KV cache is unbound (profile run); the caller
        skips engram hashing then."""
        return self.swa_cache_module.kv_cache.numel() > 0

    def dummy_hashes(
        self, input_ids: torch.Tensor
    ) -> tuple[torch.Tensor, torch.Tensor]:
        """Participate in DP lookups without valid rows."""
        num_tokens = input_ids.shape[0]
        num_layers, max_ngram = self.multipliers.shape
        num_heads = self.primes.shape[-1]
        hashes = input_ids.new_full(
            (num_tokens, num_layers, (max_ngram - 1) * num_heads),
            DEAD_ID,
            dtype=torch.int32,
        )
        keep = torch.zeros(num_tokens, dtype=torch.bool, device=input_ids.device)
        return hashes, keep

    def forward(
        self,
        input_ids: torch.Tensor,
        positions: torch.Tensor,
        query_start_loc: torch.Tensor,
        dead_mask: torch.Tensor,
        lookback_token_ids: torch.Tensor,
        lookback_dead_mask: torch.Tensor,
    ) -> torch.Tensor:
        """Compute [tokens, layers, hash columns] int32 n-gram hashes.

        History comes from the current chunk, then the runner's lookback
        window.
        """
        num_tokens = input_ids.shape[0]
        num_layers, max_ngram = self.multipliers.shape
        num_heads = self.primes.shape[-1]
        output = input_ids.new_empty(
            (num_tokens, num_layers, (max_ngram - 1) * num_heads), dtype=torch.int32
        )
        if num_tokens == 0:
            return output
        _hash_ids_kernel[(triton.cdiv(num_tokens, 32), num_layers)](
            input_ids,
            self.token_map,
            dead_mask,
            positions,
            query_start_loc,
            self.multipliers,
            self.primes,
            self.offsets,
            lookback_token_ids,
            lookback_dead_mask,
            output,
            num_tokens,
            self.pad_id,
            input_stride=input_ids.stride(0),
            mask_stride=dead_mask.stride(0),
            position_stride=positions.stride(0),
            query_stride=query_start_loc.stride(0),
            num_query_rows=query_start_loc.numel() - 1,
            MAX_NGRAM=max_ngram,
            num_heads=num_heads,
            BLOCK_T=32,
            BLOCK_H=triton.next_power_of_2(num_heads),
            dead_id=DEAD_ID,
            lookback_depth=lookback_token_ids.shape[1],
            lookback_row_stride=lookback_token_ids.stride(0),
            lookback_col_stride=lookback_token_ids.stride(1),
            lookback_mask_row_stride=lookback_dead_mask.stride(0),
            lookback_mask_col_stride=lookback_dead_mask.stride(1),
            num_warps=4,
        )
        return output

dummy_hashes(input_ids)

Participate in DP lookups without valid rows.

Source code in vllm/models/deepseek_v41/common/engram.py
def dummy_hashes(
    self, input_ids: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
    """Participate in DP lookups without valid rows."""
    num_tokens = input_ids.shape[0]
    num_layers, max_ngram = self.multipliers.shape
    num_heads = self.primes.shape[-1]
    hashes = input_ids.new_full(
        (num_tokens, num_layers, (max_ngram - 1) * num_heads),
        DEAD_ID,
        dtype=torch.int32,
    )
    keep = torch.zeros(num_tokens, dtype=torch.bool, device=input_ids.device)
    return hashes, keep

ensure_cache()

Return False while the KV cache is unbound (profile run); the caller skips engram hashing then.

Source code in vllm/models/deepseek_v41/common/engram.py
def ensure_cache(self) -> bool:
    """Return False while the KV cache is unbound (profile run); the caller
    skips engram hashing then."""
    return self.swa_cache_module.kv_cache.numel() > 0

forward(input_ids, positions, query_start_loc, dead_mask, lookback_token_ids, lookback_dead_mask)

Compute [tokens, layers, hash columns] int32 n-gram hashes.

History comes from the current chunk, then the runner's lookback window.

Source code in vllm/models/deepseek_v41/common/engram.py
def forward(
    self,
    input_ids: torch.Tensor,
    positions: torch.Tensor,
    query_start_loc: torch.Tensor,
    dead_mask: torch.Tensor,
    lookback_token_ids: torch.Tensor,
    lookback_dead_mask: torch.Tensor,
) -> torch.Tensor:
    """Compute [tokens, layers, hash columns] int32 n-gram hashes.

    History comes from the current chunk, then the runner's lookback
    window.
    """
    num_tokens = input_ids.shape[0]
    num_layers, max_ngram = self.multipliers.shape
    num_heads = self.primes.shape[-1]
    output = input_ids.new_empty(
        (num_tokens, num_layers, (max_ngram - 1) * num_heads), dtype=torch.int32
    )
    if num_tokens == 0:
        return output
    _hash_ids_kernel[(triton.cdiv(num_tokens, 32), num_layers)](
        input_ids,
        self.token_map,
        dead_mask,
        positions,
        query_start_loc,
        self.multipliers,
        self.primes,
        self.offsets,
        lookback_token_ids,
        lookback_dead_mask,
        output,
        num_tokens,
        self.pad_id,
        input_stride=input_ids.stride(0),
        mask_stride=dead_mask.stride(0),
        position_stride=positions.stride(0),
        query_stride=query_start_loc.stride(0),
        num_query_rows=query_start_loc.numel() - 1,
        MAX_NGRAM=max_ngram,
        num_heads=num_heads,
        BLOCK_T=32,
        BLOCK_H=triton.next_power_of_2(num_heads),
        dead_id=DEAD_ID,
        lookback_depth=lookback_token_ids.shape[1],
        lookback_row_stride=lookback_token_ids.stride(0),
        lookback_col_stride=lookback_token_ids.stride(1),
        lookback_mask_row_stride=lookback_dead_mask.stride(0),
        lookback_mask_col_stride=lookback_dead_mask.stride(1),
        num_warps=4,
    )
    return output

ParallelEngramEmbedding

Bases: Module

Hash heads with FP8 or packed MXFP4 rows and per-block E8M0 scales.

Heads are TP-sharded, and additionally DP-sharded across a node-local group unless the table lives in (DP-shared) pinned host memory or its rows are split over a peer group's HBM. MXFP4 rows are E2M1 values, two per byte (even column in the low nibble), stored as uint8 since torch has no 4-bit dtype.

Methods:

  • collapse_huge_pages –

    Best-effort MADV_COLLAPSE (Linux >= 6.1) of pages that faulted small.

  • forward –

    indices: [num_tokens, n_hash_cols] -> [num_tokens, n_hash_cols, dim]

  • lookup –

    Look up local heads into BF16, or packed FP8 bytes followed by scales.

  • process_weights_after_loading –

    Peers read this table's rows directly, so serve only once every GPU

Source code in vllm/models/deepseek_v41/common/engram.py
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
class ParallelEngramEmbedding(nn.Module):
    """Hash heads with FP8 or packed MXFP4 rows and per-block E8M0 scales.

    Heads are TP-sharded, and additionally DP-sharded across a node-local
    group unless the table lives in (DP-shared) pinned host memory or its rows
    are split over a peer group's HBM. MXFP4 rows are E2M1 values, two per
    byte (even column in the low nibble), stored as uint8 since torch has no
    4-bit dtype.
    """

    _weight_loader = staticmethod(_engram_head_shard_weight_loader)
    _shared_memory: DPSharedEngramStorage | None = None
    _packed: torch.Tensor | None = None
    _loaded = False
    packed_fp4: bool = False

    def __init__(
        self,
        num_embeddings: int,
        dim: int,
        head_sizes: tuple[int, ...],
        block_size: int = 32,
        cpu_offload: bool = False,
        dp_shared_memory: bool = False,
        use_thp: bool = False,
        packed_fp4: bool = False,
        peer_group: EngramPeerGroup | None = None,
    ) -> None:
        super().__init__()
        assert head_sizes and all(size > 0 for size in head_sizes)
        assert sum(head_sizes) <= num_embeddings
        assert not packed_fp4 or dim % 2 == 0
        self.packed_fp4 = packed_fp4
        self.row_bytes = dim // 2 if packed_fp4 else dim
        self.cpu_offload = cpu_offload
        self.dp_shared_memory = dp_shared_memory
        self.use_thp = use_thp
        self.dp_size = get_engram_dp_size()
        if dp_shared_memory:
            if not cpu_offload:
                raise ValueError("dp_shared_memory requires cpu_offload=True")
            if self.dp_size <= 1:
                raise ValueError(
                    "dp_shared_memory requires a node-local Engram DP "
                    f"group with size > 1; effective Engram DP size is {self.dp_size}. "
                    "Check that the node layout and rank placement allow complete "
                    "DP replicas to be co-located."
                )
            self.dp_size = 1
        # GPU tables split rows over a peer group (the given GPUs of several
        # engines, else the DP group) and every rank reads its own tokens' rows
        # from the owning peer's HBM, so lookups need no collective.
        self._peer_group: EngramPeerGroup | GroupCoordinator | None = peer_group
        if peer_group is not None:
            self.dp_size = 1
        elif (
            not cpu_offload
            and self.dp_size > 1
            and get_tensor_model_parallel_world_size() == 1
        ):
            group = get_engram_dp_group()
            assert group is not None
            # Cross-node groups require embedding_across_dp's single NVLink domain;
            # node-local groups without NVLink P2P keep the DP collectives.
            peer = not all(in_the_same_node_as(group.cpu_group, source_rank=0))
            if not peer:
                physical_id = current_platform.visible_device_id_to_physical_device_id(
                    torch.accelerator.current_device_index()
                )
                physical_ids = [None] * group.world_size
                torch.distributed.all_gather_object(
                    physical_ids, physical_id, group=group.cpu_group
                )
                peer = current_platform.is_fully_connected(physical_ids)
            if peer:
                self._peer_group, self.dp_size = group, 1
        if cpu_offload and not is_uva_available():
            raise RuntimeError("Engram CPU offload requires UVA support")
        self._views: tuple[torch.Tensor, torch.Tensor] | None = None
        self._view_src: tuple[int, int] | None = None
        self.num_embeddings = num_embeddings
        self.dim = dim
        self.block_size = block_size
        self.n_hash_cols = len(head_sizes)
        self.tp_size = get_tensor_model_parallel_world_size()
        num_shards, head_rank = self._get_shard_info()
        self.part_n_hash_cols = triton.cdiv(self.n_hash_cols, num_shards)
        # TODO: Support row-wise sharding when there are too few hash heads.
        assert (num_shards - 1) * self.part_n_hash_cols < self.n_hash_cols, (
            f"Engram sharding leaves ranks without hash heads: "
            f"{self.n_hash_cols} heads over {num_shards} shards"
        )
        self.head_start = head_rank * self.part_n_hash_cols
        head_end = self.head_start + self.part_n_hash_cols
        self.vocab_start_idx = sum(head_sizes[: self.head_start])
        self.vocab_end_idx = sum(head_sizes[:head_end])
        self.part_num_embeddings = self.vocab_end_idx - self.vocab_start_idx
        # Peer tables keep the full lookup range but store one row slice.
        self.peer_rows, row_start = 0, self.vocab_start_idx
        if (peers := self._peer_group) is not None:
            self.peer_rows = triton.cdiv(self.vocab_end_idx, peers.world_size)
            row_start = min(peers.rank_in_group * self.peer_rows, self.vocab_end_idx)
            self.part_num_embeddings = min(
                self.peer_rows, self.vocab_end_idx - row_start
            )
        self._num_sms = torch.cuda.get_device_properties(
            torch.accelerator.current_device_index()
        ).multi_processor_count
        weight, scales = self._allocate_weights()
        self.weight = nn.Parameter(weight, requires_grad=False)
        self.weight_scale_inv = nn.Parameter(scales, requires_grad=False)
        for param in (self.weight, self.weight_scale_inv):
            set_weight_attrs(
                param,
                {
                    "weight_loader": self._weight_loader,
                    "engram_vocab_start": row_start,
                },
            )
        if cpu_offload and not self.weight.is_meta:
            # Constant dummy values avoid randomizing huge CPU lookup tables.
            set_weight_attrs(self.weight, {"dummy_weight_value": 1.0})
            # The ue8m0 encoding of scale 1.0 is exponent byte 127.
            set_weight_attrs(self.weight_scale_inv, {"dummy_weight_value": 127})
            logger.info(
                "Engram table offloaded to pinned host memory: %d rows x %d, "
                "%.2f GiB %s",
                self.part_num_embeddings,
                dim,
                self.part_num_embeddings
                * (self.row_bytes + dim // block_size)
                / 1024**3,
                "per rank"
                if self._shared_memory is None
                else "shared across DP replicas via "
                + ("cuMem" if self._shared_memory.huge_pages else "/dev/shm"),
            )

    def _get_shard_info(self) -> tuple[int, int]:
        if self.dp_size == 1:
            return self.tp_size, get_tensor_model_parallel_rank()
        return self.tp_size * self.dp_size, engram_head_shard_rank()

    def _allocate_weights(self) -> tuple[torch.Tensor, torch.Tensor]:
        scale_dim = self.dim // self.block_size
        weight_dtype = torch.uint8 if self.packed_fp4 else torch.float8_e4m3fn
        if torch.get_default_device().type == "meta" or not (
            self.cpu_offload or self.peer_rows
        ):
            return (
                torch.empty(
                    self.part_num_embeddings, self.row_bytes, dtype=weight_dtype
                ),
                torch.empty(self.part_num_embeddings, scale_dim, dtype=torch.uint8),
            )
        if self.peer_rows:
            peers = self._peer_group
            values = self.peer_rows * self.row_bytes
            num_bytes = values + self.peer_rows * scale_dim
            device = torch.device("cuda", torch.accelerator.current_device_index())
            if isinstance(peers, EngramPeerGroup):
                # Symmetric memory cannot span engines: its rendezvous rejects
                # equal device ordinals on one node, and each engine's start at 0.
                from vllm.models.deepseek_v41.nvidia.ops.engram_fabric_table import (
                    share_peer_table,
                )

                # Held here like the symmetric handle, so lookups keep every
                # slice mapped even if a loader replaces the parameters' data.
                buffer, self._peer_mappings = share_peer_table(num_bytes, peers)
                pointers = self._peer_mappings.addresses
            else:
                import torch.distributed._symmetric_memory as symm_mem

                assert peers is not None
                buffer = symm_mem.empty(num_bytes, dtype=torch.uint8, device=device)
                self._symm_handle = symm_mem.rendezvous(buffer, peers.device_group)
                pointers = self._symm_handle.buffer_ptrs
            self.register_buffer(
                "peer_ptrs",
                torch.tensor(pointers, dtype=torch.int64, device=device),
                persistent=False,
            )
            rows = self.part_num_embeddings
            return (
                buffer[: rows * self.row_bytes]
                .view(weight_dtype)
                .view(rows, self.row_bytes),
                buffer[values : values + rows * scale_dim].view(rows, scale_dim),
            )
        if self.dp_shared_memory:
            group = get_engram_dp_group()
            assert group is not None
            storage = DPSharedEngramStorage(
                self.part_num_embeddings,
                self.dim,
                self.block_size,
                group,
                self.packed_fp4,
            )
            self._shared_memory = storage
            self._weight_loader = storage.load_weight
            return storage.weight, storage.weight_scale_inv
        # pin_memory rounds up to a power of two. ROCm pins huge pages either way.
        if self.use_thp or current_platform.is_rocm():
            weight_bytes = self.part_num_embeddings * self.row_bytes
            scale_bytes = self.part_num_embeddings * scale_dim
            packed = _allocate_huge_page_storage(weight_bytes + scale_bytes)
            if packed is not None:
                self._packed = packed
                return (
                    packed[:weight_bytes].view(weight_dtype).view(-1, self.row_bytes),
                    packed[weight_bytes:].view(-1, scale_dim),
                )
        # Model initialization may be inside a CUDA device context.
        return (
            torch.empty(
                self.part_num_embeddings,
                self.row_bytes,
                dtype=weight_dtype,
                device="cpu",
                pin_memory=True,
            ),
            torch.empty(
                self.part_num_embeddings,
                scale_dim,
                dtype=torch.uint8,
                device="cpu",
                pin_memory=True,
            ),
        )

    def process_weights_after_loading(self) -> None:
        """Peers read this table's rows directly, so serve only once every GPU
        of the group has finished loading its own."""
        if not self.peer_rows:
            return
        assert self._peer_group is not None
        if self._loaded and isinstance(self._peer_group, EngramPeerGroup):
            # Other engines read these rows at any time, also while they reload.
            raise RuntimeError(
                "Engram tables split across engines (peer_group_size) cannot "
                "reload weights; restart the peer group instead"
            )
        torch.accelerator.synchronize()
        self._peer_group.barrier()
        self._loaded = True

    def collapse_huge_pages(self) -> None:
        """Best-effort MADV_COLLAPSE (Linux >= 6.1) of pages that faulted small."""
        if self._packed is None:
            return
        addr, num_bytes = self._packed.data_ptr(), self._packed.nbytes
        libc = ctypes.CDLL(None, use_errno=True)
        libc.madvise.argtypes = (ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int)
        if libc.madvise(addr, num_bytes, 25) != 0:
            logger.warning(
                "Engram MADV_COLLAPSE failed; keeping existing pages: %s",
                os.strerror(ctypes.get_errno()),
            )

    @property
    def _uses_small_host_pages(self) -> bool:
        """Whether the GPU maps this host table with pages too small for its TLB."""
        if self._shared_memory is not None:
            return not self._shared_memory.huge_pages
        return self.cpu_offload and self._packed is None

    def _storage(self) -> tuple[torch.Tensor, torch.Tensor]:
        if self._shared_memory is not None:
            return self._shared_memory.get_views(self.weight, self.weight_scale_inv)
        if not self.cpu_offload:
            return self.weight.data, self.weight_scale_inv.data
        src = (self.weight.data_ptr(), self.weight_scale_inv.data_ptr())
        if self._view_src != src:
            self._views = (
                get_accelerator_view_from_cpu_tensor(self.weight.data),
                get_accelerator_view_from_cpu_tensor(self.weight_scale_inv.data),
            )
            self._view_src = src
        assert self._views is not None
        return self._views

    def lookup(
        self, indices: torch.Tensor, out: torch.Tensor, background: bool = False
    ) -> None:
        """Look up local heads into BF16, or packed FP8 bytes followed by scales.

        `background` limits the grid to leave SMs for concurrent work.
        """
        if self.packed_fp4 and out.dtype == torch.uint8:
            raise ValueError("MXFP4 Engram tables do not support packed FP8 output")
        rows = indices.shape[0] * self.part_n_hash_cols
        if not rows:
            return
        weight, scales = self._storage()
        ids_stride_t, ids_stride_h = indices.stride()
        sort_min_rows, _ = _engram_lookup_thresholds(indices.device)
        sort = (
            self._uses_small_host_pages
            and sort_min_rows is not None
            and rows >= sort_min_rows
        )
        dst = indices
        if sort:
            # Small-page host tables miss the TLB per row; in table order, rows on
            # one page share it.
            head_end = self.head_start + self.part_n_hash_cols
            local = indices[:, self.head_start : head_end] - self.vocab_start_idx
            if (pad := self.part_n_hash_cols - local.shape[1]) > 0:
                local = nn.functional.pad(local, (0, pad), value=-1)
            indices, dst = local.flatten().sort()
            dst = dst.int()
        # The table dwarfs TLB reach, so a persistent grid near the SM count
        # beats one program per row; halve it to leave SMs for the main stream.
        max_grid = self._num_sms // 2 if background else self._num_sms
        # Host reads are latency bound: once the rows fill the grid with
        # 128-row tiles, each program keeps 8x the reads in flight.
        block_r = 128 if rows >= 128 * max_grid else 16
        packed = out.dtype == torch.uint8
        # Copy packed rows 8 bytes per load where their values, scales and
        # output rows are 8-byte aligned.
        wide = (
            packed
            and self.dim // self.block_size % 8 == 0
            and (out.stride(0) | out.data_ptr()) % 8 == 0
        )
        grid = min(triton.cdiv(rows, block_r), max_grid)
        _engram_lookup_kernel[(grid,)](
            self.peer_ptrs if self.peer_rows else weight,
            scales,
            indices,
            dst,
            out,
            self.vocab_start_idx,
            self.vocab_end_idx,
            rows,
            ids_stride_t,
            ids_stride_h,
            HEAD_START=self.head_start,
            LOCAL_HEADS=self.part_n_hash_cols,
            TOTAL_HEADS=self.n_hash_cols,
            DIM=self.dim,
            QUANT_BLOCK=self.block_size,
            BLOCK_R=block_r,
            GRID=grid,
            SORTED=sort,
            PACKED=packed,
            OUT_STRIDE=out.stride(0),
            PEER_ROWS=self.peer_rows,
            PACKED_FP4=self.packed_fp4,
            WORD=tl.uint64 if wide else tl.uint8,
            num_warps=8 if block_r == 128 else 4,
        )

    def forward(self, indices: torch.Tensor) -> torch.Tensor:
        """indices: [num_tokens, n_hash_cols] -> [num_tokens, n_hash_cols, dim]
        bf16, gathered from all shards for this replica's tokens."""
        num_tokens = indices.shape[0]
        if self.dp_size > 1:
            indices = gather_engram_hashes(indices)
        out = torch.empty(
            (indices.shape[0], self.part_n_hash_cols, self.dim),
            dtype=torch.bfloat16,
            device=indices.device,
        )
        self.lookup(indices, out)
        if self.dp_size > 1:
            out = _gather_engram_rows(out, num_tokens)
        if self.tp_size > 1:
            out = tensor_model_parallel_all_gather(out, dim=1)
        return out[:, : self.n_hash_cols]

_uses_small_host_pages property

Whether the GPU maps this host table with pages too small for its TLB.

collapse_huge_pages()

Best-effort MADV_COLLAPSE (Linux >= 6.1) of pages that faulted small.

Source code in vllm/models/deepseek_v41/common/engram.py
def collapse_huge_pages(self) -> None:
    """Best-effort MADV_COLLAPSE (Linux >= 6.1) of pages that faulted small."""
    if self._packed is None:
        return
    addr, num_bytes = self._packed.data_ptr(), self._packed.nbytes
    libc = ctypes.CDLL(None, use_errno=True)
    libc.madvise.argtypes = (ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int)
    if libc.madvise(addr, num_bytes, 25) != 0:
        logger.warning(
            "Engram MADV_COLLAPSE failed; keeping existing pages: %s",
            os.strerror(ctypes.get_errno()),
        )

forward(indices)

indices: [num_tokens, n_hash_cols] -> [num_tokens, n_hash_cols, dim] bf16, gathered from all shards for this replica's tokens.

Source code in vllm/models/deepseek_v41/common/engram.py
def forward(self, indices: torch.Tensor) -> torch.Tensor:
    """indices: [num_tokens, n_hash_cols] -> [num_tokens, n_hash_cols, dim]
    bf16, gathered from all shards for this replica's tokens."""
    num_tokens = indices.shape[0]
    if self.dp_size > 1:
        indices = gather_engram_hashes(indices)
    out = torch.empty(
        (indices.shape[0], self.part_n_hash_cols, self.dim),
        dtype=torch.bfloat16,
        device=indices.device,
    )
    self.lookup(indices, out)
    if self.dp_size > 1:
        out = _gather_engram_rows(out, num_tokens)
    if self.tp_size > 1:
        out = tensor_model_parallel_all_gather(out, dim=1)
    return out[:, : self.n_hash_cols]

lookup(indices, out, background=False)

Look up local heads into BF16, or packed FP8 bytes followed by scales.

background limits the grid to leave SMs for concurrent work.

Source code in vllm/models/deepseek_v41/common/engram.py
def lookup(
    self, indices: torch.Tensor, out: torch.Tensor, background: bool = False
) -> None:
    """Look up local heads into BF16, or packed FP8 bytes followed by scales.

    `background` limits the grid to leave SMs for concurrent work.
    """
    if self.packed_fp4 and out.dtype == torch.uint8:
        raise ValueError("MXFP4 Engram tables do not support packed FP8 output")
    rows = indices.shape[0] * self.part_n_hash_cols
    if not rows:
        return
    weight, scales = self._storage()
    ids_stride_t, ids_stride_h = indices.stride()
    sort_min_rows, _ = _engram_lookup_thresholds(indices.device)
    sort = (
        self._uses_small_host_pages
        and sort_min_rows is not None
        and rows >= sort_min_rows
    )
    dst = indices
    if sort:
        # Small-page host tables miss the TLB per row; in table order, rows on
        # one page share it.
        head_end = self.head_start + self.part_n_hash_cols
        local = indices[:, self.head_start : head_end] - self.vocab_start_idx
        if (pad := self.part_n_hash_cols - local.shape[1]) > 0:
            local = nn.functional.pad(local, (0, pad), value=-1)
        indices, dst = local.flatten().sort()
        dst = dst.int()
    # The table dwarfs TLB reach, so a persistent grid near the SM count
    # beats one program per row; halve it to leave SMs for the main stream.
    max_grid = self._num_sms // 2 if background else self._num_sms
    # Host reads are latency bound: once the rows fill the grid with
    # 128-row tiles, each program keeps 8x the reads in flight.
    block_r = 128 if rows >= 128 * max_grid else 16
    packed = out.dtype == torch.uint8
    # Copy packed rows 8 bytes per load where their values, scales and
    # output rows are 8-byte aligned.
    wide = (
        packed
        and self.dim // self.block_size % 8 == 0
        and (out.stride(0) | out.data_ptr()) % 8 == 0
    )
    grid = min(triton.cdiv(rows, block_r), max_grid)
    _engram_lookup_kernel[(grid,)](
        self.peer_ptrs if self.peer_rows else weight,
        scales,
        indices,
        dst,
        out,
        self.vocab_start_idx,
        self.vocab_end_idx,
        rows,
        ids_stride_t,
        ids_stride_h,
        HEAD_START=self.head_start,
        LOCAL_HEADS=self.part_n_hash_cols,
        TOTAL_HEADS=self.n_hash_cols,
        DIM=self.dim,
        QUANT_BLOCK=self.block_size,
        BLOCK_R=block_r,
        GRID=grid,
        SORTED=sort,
        PACKED=packed,
        OUT_STRIDE=out.stride(0),
        PEER_ROWS=self.peer_rows,
        PACKED_FP4=self.packed_fp4,
        WORD=tl.uint64 if wide else tl.uint8,
        num_warps=8 if block_r == 128 else 4,
    )

process_weights_after_loading()

Peers read this table's rows directly, so serve only once every GPU of the group has finished loading its own.

Source code in vllm/models/deepseek_v41/common/engram.py
def process_weights_after_loading(self) -> None:
    """Peers read this table's rows directly, so serve only once every GPU
    of the group has finished loading its own."""
    if not self.peer_rows:
        return
    assert self._peer_group is not None
    if self._loaded and isinstance(self._peer_group, EngramPeerGroup):
        # Other engines read these rows at any time, also while they reload.
        raise RuntimeError(
            "Engram tables split across engines (peer_group_size) cannot "
            "reload weights; restart the peer group instead"
        )
    torch.accelerator.synchronize()
    self._peer_group.barrier()
    self._loaded = True

_allocate_huge_page_storage(num_bytes)

Register prefaulted huge pages, or return None for pinned-memory fallback.

Source code in vllm/models/deepseek_v41/common/engram.py
def _allocate_huge_page_storage(num_bytes: int) -> torch.Tensor | None:
    """Register prefaulted huge pages, or return None for pinned-memory fallback."""
    if sys.platform != "linux":
        return None
    try:
        mapping = mmap.mmap(-1, num_bytes, flags=mmap.MAP_PRIVATE | mmap.MAP_ANONYMOUS)
        mapping.madvise(mmap.MADV_HUGEPAGE)
    except OSError as exc:
        logger.warning(
            "Engram huge-page allocation failed (%s); using pinned memory.", exc
        )
        return None

    owner = np.frombuffer(mapping, dtype=np.uint8)
    # Fault the pages in before CUDA pins them, so they can be huge.
    owner[:: mmap.PAGESIZE] = 0
    tensor = torch.from_numpy(owner)
    pointer = tensor.data_ptr()
    result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0)
    if result.value != 0:
        logger.warning(
            "Engram cudaHostRegister failed (%s); using pinned memory.", result
        )
        return None
    # Tensor views retain owner; its finalizer retains mapping until unregister.
    finalizer = weakref.finalize(
        owner, DPSharedEngramStorage._unregister, mapping, pointer
    )
    finalizer.atexit = False  # type: ignore[misc]
    if not tensor.is_pinned():
        logger.warning(
            "CUDA did not recognize Engram registration; using pinned memory."
        )
        return None
    return tensor

_allow_any_numa_node(address, num_bytes)

Let a mapping's pages spill to any NUMA node despite --numa-bind.

mbind(MPOL_LOCAL) overrides the task's membind for this range: pages still go to the faulting CPU's node first, then to any other node.

Source code in vllm/models/deepseek_v41/common/engram.py
def _allow_any_numa_node(address: int, num_bytes: int) -> None:
    """Let a mapping's pages spill to any NUMA node despite --numa-bind.

    mbind(MPOL_LOCAL) overrides the task's membind for this range: pages still
    go to the faulting CPU's node first, then to any other node.
    """
    sys_mbind = {"aarch64": 235, "x86_64": 237}.get(platform.machine())
    libc = ctypes.CDLL(None, use_errno=True)
    libc.syscall.restype = ctypes.c_long
    if sys_mbind is None or libc.syscall(
        ctypes.c_long(sys_mbind),
        ctypes.c_void_p(address),
        ctypes.c_ulong(num_bytes),
        ctypes.c_int(4),  # MPOL_LOCAL
        None,
        ctypes.c_ulong(0),
        ctypes.c_uint(0),
    ):
        logger.warning_once(
            "Engram mbind(MPOL_LOCAL) failed; /dev/shm tables follow the "
            "process memory policy: %s",
            os.strerror(ctypes.get_errno()),
        )

_engram_head_shard_weight_loader(param, loaded_weight)

Load this rank's complete head buckets. ue8m0 scales arrive as float8_e8m0fnu; keep the raw bytes (the param stores uint8).

Source code in vllm/models/deepseek_v41/common/engram.py
def _engram_head_shard_weight_loader(
    param: torch.nn.Parameter, loaded_weight: torch.Tensor
) -> None:
    """Load this rank's complete head buckets. ue8m0 scales arrive as
    float8_e8m0fnu; keep the raw bytes (the param stores uint8)."""
    part_rows = param.shape[0]
    if loaded_weight.dtype == torch.float8_e8m0fnu:
        loaded_weight = loaded_weight.view(torch.uint8)
    shard = loaded_weight.narrow(0, param.engram_vocab_start, part_rows)
    assert shard.shape == param.shape, (
        f"engram shard {tuple(shard.shape)} does not fit param {tuple(param.shape)}"
    )
    param.data.copy_(shard)

_engram_lookup_kernel(weight, scales, ids, sorted_dst, out, vocab_start, vocab_end, num_rows, ids_stride_t, ids_stride_h, HEAD_START, LOCAL_HEADS, TOTAL_HEADS, DIM, QUANT_BLOCK, BLOCK_R, GRID, SORTED, PACKED, OUT_STRIDE, PEER_ROWS=0, WORD=tl.uint8, PACKED_FP4=False)

Gather FP8 or MXFP4 rows as dequantized BF16, or FP8 rows as packed values/scales (copied WORD at a time).

Only this rank's heads are read; padded heads write zeros for all-gather. weight/scales may address pinned host memory through UVA. If SORTED, ids holds rows relative to vocab_start in table order and sorted_dst their output rows.

Source code in vllm/models/deepseek_v41/common/engram.py
@triton.jit(
    do_not_specialize=[
        "vocab_start",
        "vocab_end",
        "num_rows",
        "ids_stride_t",
        "ids_stride_h",
        "GRID",
        *(() if _SORTED_IS_CONSTEXPR else ("SORTED",)),
    ]
)
def _engram_lookup_kernel(
    weight,
    scales,
    ids,
    sorted_dst,
    out,
    vocab_start,
    vocab_end,
    num_rows,
    ids_stride_t,
    ids_stride_h,
    HEAD_START: tl.constexpr,
    LOCAL_HEADS: tl.constexpr,
    TOTAL_HEADS: tl.constexpr,
    DIM: tl.constexpr,
    QUANT_BLOCK: tl.constexpr,
    BLOCK_R: tl.constexpr,
    GRID,
    SORTED: tl.constexpr if _SORTED_IS_CONSTEXPR else None,  # type: ignore[valid-type]
    PACKED: tl.constexpr,
    OUT_STRIDE: tl.constexpr,
    PEER_ROWS: tl.constexpr = 0,
    WORD: tl.constexpr = tl.uint8,
    PACKED_FP4: tl.constexpr = False,
):
    """Gather FP8 or MXFP4 rows as dequantized BF16, or FP8 rows as packed
    values/scales (copied WORD at a time).

    Only this rank's heads are read; padded heads write zeros for all-gather.
    `weight`/`scales` may address pinned host memory through UVA. If SORTED,
    `ids` holds rows relative to vocab_start in table order and `sorted_dst`
    their output rows.
    """
    cols = tl.arange(0, DIM)
    scale_cols = cols // QUANT_BLOCK
    for base in tl.range(tl.program_id(0) * BLOCK_R, num_rows, GRID * BLOCK_R):
        rows = base + tl.arange(0, BLOCK_R)
        valid = rows < num_rows
        if SORTED:
            local = tl.load(ids + rows, mask=valid, other=-1).to(tl.int64)
            dst = tl.load(sorted_dst + rows, mask=valid, other=0).to(tl.int32)
            owned = (local >= 0) & (local < vocab_end - vocab_start)
        else:
            head = HEAD_START + rows % LOCAL_HEADS
            token = (rows // LOCAL_HEADS).to(tl.int64)
            index = tl.load(
                ids + token * ids_stride_t + head * ids_stride_h,
                mask=valid & (head < TOTAL_HEADS),
                other=-1,
            ).to(tl.int64)
            dst = rows
            owned = valid & (head < TOTAL_HEADS)
            owned &= (index >= vocab_start) & (index < vocab_end)
            local = index - vocab_start
        local = tl.where(owned, local, 0)
        ROW_BYTES: tl.constexpr = DIM // 2 if PACKED_FP4 else DIM
        if PEER_ROWS:
            # `weight` holds each rank's buffer: PEER_ROWS rows, then scales.
            owner = local // PEER_ROWS
            peer = tl.load(weight + owner, mask=owned, other=0)
            local -= owner * PEER_ROWS
            if PACKED_FP4:
                row_weight = peer.to(tl.pointer_type(tl.uint8))
            else:
                row_weight = peer.to(tl.pointer_type(tl.float8e4nv))
            row_scales = (peer + PEER_ROWS * ROW_BYTES).to(tl.pointer_type(tl.uint8))
        else:
            row_weight, row_scales = weight, scales
        row_weight += local * ROW_BYTES
        row_scales += local * (DIM // QUANT_BLOCK)
        if PACKED:
            # Copy values and scales WORD at a time: one memory request a word.
            word_bytes: tl.constexpr = WORD.primitive_bitwidth // 8
            value_words = tl.arange(0, DIM // word_bytes)
            scale_words = tl.arange(0, DIM // QUANT_BLOCK // word_bytes)
            width: tl.constexpr = DIM + DIM // QUANT_BLOCK
            dst = dst.to(tl.int64)
            dst = dst // LOCAL_HEADS * OUT_STRIDE + dst % LOCAL_HEADS * width
            out_row = (out + dst).to(tl.pointer_type(WORD))
            # Load both first: a load after a store waits out its round trip.
            values = tl.load(
                row_weight.to(tl.pointer_type(WORD))[:, None] + value_words[None, :],
                mask=owned[:, None],
                other=0,
            )
            scale = tl.load(
                row_scales.to(tl.pointer_type(WORD))[:, None] + scale_words[None, :],
                mask=owned[:, None],
                other=0,
            )
            tl.store(
                out_row[:, None] + value_words[None, :], values, mask=valid[:, None]
            )
            out_row += DIM // word_bytes
            tl.store(
                out_row[:, None] + scale_words[None, :], scale, mask=valid[:, None]
            )
        else:
            if PACKED_FP4:
                # Even column in the low nibble. E2M1 magnitude with e = bits 2:1,
                # m = bit 0: m * 0.5 if e == 0, else (2 + m) * 2**(e - 2).
                byte = tl.load(
                    row_weight[:, None] + (cols // 2)[None, :],
                    mask=owned[:, None],
                    other=0,
                ).to(tl.int32)
                nib = tl.where(cols[None, :] % 2 == 0, byte & 0xF, (byte >> 4) & 0xF)
                e = (nib & 0x7) >> 1
                m = (nib & 0x1).to(tl.float32)
                pow2 = ((e - 2 + 127) << 23).to(tl.float32, bitcast=True)
                mag = tl.where(e == 0, m * 0.5, (2.0 + m) * pow2)
                values = tl.where((nib & 0x8) != 0, -mag, mag)
            else:
                values = tl.load(
                    row_weight[:, None] + cols[None, :], mask=owned[:, None], other=0.0
                )
            scale = tl.load(
                row_scales[:, None] + scale_cols[None, :], mask=owned[:, None], other=0
            )
            # ue8m0 is a power of two, so its byte *is* the fp32 exponent field.
            scale = (scale.to(tl.int32) << 23).to(tl.float32, bitcast=True)
            tl.store(
                out + dst[:, None] * DIM + cols[None, :],
                (values.to(tl.float32) * scale).to(tl.bfloat16),
                mask=valid[:, None],
            )

_engram_lookup_thresholds(device)

Host lookup thresholds on device: the rows from which to sort, and the tokens (max over DP ranks) from which to run inline. None disables that path.

Source code in vllm/models/deepseek_v41/common/engram.py
def _engram_lookup_thresholds(device: torch.device) -> tuple[int | None, int | None]:
    """Host lookup thresholds on `device`: the rows from which to sort, and the
    tokens (max over DP ranks) from which to run inline. None disables that path."""
    if current_platform.is_cuda() and current_platform.is_device_capability_family(
        100, device.index or 0
    ):
        # Measured on GB200.
        return 36864, 4096
    # TODO: Add hand-tuned thresholds for other GPU families.
    return None, None

_engram_select_rows(gathered, output, source_tokens, token_start, local_width)

Copy one token window out of a rank-major gathered buffer.

Both gathers land rank-major ([rank][token][local width]); this walks the window the rank keeps and lays its ranks out side by side as width.

Source code in vllm/models/deepseek_v41/common/engram.py
def _engram_select_rows(
    gathered: torch.Tensor,
    output: torch.Tensor,
    source_tokens: int,
    token_start: int,
    local_width: int,
) -> None:
    """Copy one token window out of a rank-major gathered buffer.

    Both gathers land rank-major (`[rank][token][local width]`); this walks the
    window the rank keeps and lays its ranks out side by side as width.
    """
    if output.numel() == 0:
        return
    _engram_select_rows_kernel[(triton.cdiv(output.numel(), 1024),)](
        gathered,
        output,
        source_tokens,
        token_start,
        output.numel(),
        local_width,
        output.shape[1] * output.shape[2],
        BLOCK_SIZE=1024,
    )

_engram_unpack_fp8_kernel(packed, values, scales, num_tokens, values_stride, scales_stride, HEADS, LOCAL_HEADS, LAYERS, RANK, DIM, BLOCK, PEER)

Split [rank][token][layer][local head] packed rows into MXFP8 values and F8_128x4 scales, one layer per grid column; packed holds per-rank pointers if PEER. Scales past num_tokens zero-pad the 128-row tile.

Source code in vllm/models/deepseek_v41/common/engram.py
@triton.jit(do_not_specialize=["num_tokens"])
def _engram_unpack_fp8_kernel(
    packed,
    values,
    scales,
    num_tokens,
    values_stride,
    scales_stride,
    HEADS: tl.constexpr,
    LOCAL_HEADS: tl.constexpr,
    LAYERS: tl.constexpr,
    RANK: tl.constexpr,
    DIM: tl.constexpr,
    BLOCK: tl.constexpr,
    PEER: tl.constexpr,
):
    """Split `[rank][token][layer][local head]` packed rows into MXFP8 values and
    F8_128x4 scales, one layer per grid column; `packed` holds per-rank
    pointers if PEER. Scales past `num_tokens` zero-pad the 128-row tile.
    """
    row = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
    layer = tl.program_id(1).to(tl.int64)
    token, head = row // HEADS, row % HEADS
    if PEER:
        src = tl.load(packed + head // LOCAL_HEADS)
        src = tl.multiple_of(src, DIM // 32).to(tl.pointer_type(tl.uint8))
        source = RANK * num_tokens + token
    else:
        src = packed
        source = head // LOCAL_HEADS * num_tokens + token
    source = (source * LAYERS + layer) * LOCAL_HEADS + head % LOCAL_HEADS
    # Packed rows are only scale-width aligned.
    src += tl.multiple_of(source * (DIM + DIM // 32), DIM // 32)
    valid = token < num_tokens
    cols = tl.arange(0, DIM)
    data = tl.load(src[:, None] + cols[None, :], mask=valid[:, None], other=0)
    values += layer * values_stride + row * DIM
    tl.store(values[:, None] + cols[None, :], data, mask=valid[:, None])
    scale_cols = tl.arange(0, DIM // 32)
    scale = tl.load(
        src[:, None] + DIM + scale_cols[None, :], mask=valid[:, None], other=0
    )
    group = head[:, None] * (DIM // 32) + scale_cols[None, :]
    offset = mxfp8_scale_offset(token[:, None], group, HEADS * DIM // 32)
    tl.store(scales + layer * scales_stride + offset, scale)

_gather_engram_rows(staged, num_tokens)

Exchange DP tokens for heads, retaining only this replica's tokens.

Source code in vllm/models/deepseek_v41/common/engram.py
def _gather_engram_rows(staged: torch.Tensor, num_tokens: int) -> torch.Tensor:
    """Exchange DP tokens for heads, retaining only this replica's tokens."""
    dp_group = get_engram_dp_group()
    assert dp_group is not None
    slot, remainder = divmod(staged.shape[0], dp_group.world_size)
    assert remainder == 0 and 0 <= num_tokens <= slot
    gathered = dp_group.all_gather(staged, dim=0)
    local_heads, dim = staged.shape[1:]
    rows = staged.new_empty((num_tokens, dp_group.world_size * local_heads, dim))
    _engram_select_rows(
        gathered,
        rows,
        staged.shape[0],
        dp_group.rank_in_group * slot,
        local_heads * dim,
    )
    return rows

_is_prime(n)

Deterministic Miller-Rabin for n < 2**32 (avoids a sympy import).

Source code in vllm/models/deepseek_v41/common/engram.py
def _is_prime(n: int) -> bool:
    """Deterministic Miller-Rabin for n < 2**32 (avoids a sympy import)."""
    if n < 2:
        return False
    for p in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37):
        if n % p == 0:
            return n == p
    d = n - 1
    r = 0
    while d % 2 == 0:
        d //= 2
        r += 1
    for a in (2, 7, 61):
        x = pow(a, d, n)
        if x in (1, n - 1):
            continue
        for _ in range(r - 1):
            x = x * x % n
            if x == n - 1:
                break
        else:
            return False
    return True

_shm_shortfall(num_bytes)

Why /dev/shm cannot hold a table of this size, or None if it can.

Source code in vllm/models/deepseek_v41/common/engram.py
def _shm_shortfall(num_bytes: int) -> str | None:
    """Why /dev/shm cannot hold a table of this size, or None if it can."""
    if not os.path.isdir(SHM_PATH):
        return f"{SHM_PATH} is not mounted"
    try:
        check_shm_free_space(num_bytes, allocation_name="Engram tables")
    except Exception as exc:
        return describe_error(exc)
    # tmpfs pages beyond available host memory are faulted in by the OOM killer.
    available = get_available_memory()
    if num_bytes > available:
        return (
            f"{num_bytes / 2**30:.1f} GiB exceeds the "
            f"{available / 2**30:.1f} GiB of available host memory"
        )
    return None

build_compressed_token_map(tokenizer)

Map every token id onto a smaller id space where tokens that normalize alike collapse together.

N-grams are hashed over these compressed ids, so " The", "the" and "THE" all hash the same way. The compressed size matters beyond bounds checking: every hash multiplier is derived from it.

Source code in vllm/models/deepseek_v41/common/engram.py
def build_compressed_token_map(tokenizer) -> tuple[list[int], int]:
    """Map every token id onto a smaller id space where tokens that normalize
    alike collapse together.

    N-grams are hashed over these compressed ids, so " The", "the" and "THE"
    all hash the same way. The compressed size matters beyond bounds checking:
    every hash multiplier is derived from it.
    """
    from tokenizers import Regex, normalizers

    # A private-use char, so a token that is exactly one space survives
    # Strip() instead of collapsing to the empty string and merging with
    # unrelated tokens.
    sentinel = "\ue000"
    normalizer = normalizers.Sequence(
        [
            normalizers.NFKC(),
            normalizers.NFD(),
            normalizers.StripAccents(),
            normalizers.Lowercase(),
            normalizers.Replace(Regex(r"[ \t\r\n]+"), " "),
            normalizers.Replace(Regex(r"^ $"), sentinel),
            normalizers.Strip(),
            normalizers.Replace(sentinel, " "),
        ]
    )

    # The raw Rust tokenizer, matching what training decodes with
    # (no clean_up_tokenization_spaces).
    backend = tokenizer.backend_tokenizer
    key_to_new: dict[str, int] = {}
    lookup = [0] * len(tokenizer)
    for token_id in range(len(tokenizer)):
        text = backend.decode([token_id], skip_special_tokens=False)
        if "\ufffd" in text:
            # A partial UTF-8 byte token: nothing to normalize, so key it
            # by its raw form.
            key = backend.id_to_token(token_id)
        else:
            normalized = normalizer.normalize_str(text)
            key = normalized if normalized else text

        new_id = key_to_new.get(key)
        if new_id is None:
            new_id = len(key_to_new)
            key_to_new[key] = new_id
        lookup[token_id] = new_id

    return lookup, len(key_to_new)

can_share_engram_tables(layout, block_size=32)

Whether co-located DP replicas exist and can hold the full tables.

Source code in vllm/models/deepseek_v41/common/engram.py
def can_share_engram_tables(layout: EngramLayout, block_size: int = 32) -> bool:
    """Whether co-located DP replicas exist and can hold the full tables."""
    if get_engram_dp_size() == 1:
        logger.warning_once(
            "Engram DP replicas are not co-located on one node; "
            "storing the offloaded tables per rank instead of sharing them."
        )
        return False
    num_bytes = sum(layout.num_embeddings) * (
        layout.row_bytes + layout.head_dim // block_size
    )
    group = get_engram_dp_group()
    assert group is not None
    # Only the leader allocates, so every replica follows its check. /dev/shm
    # must fit even where cuMem will, since it is the fallback.
    error = _shm_shortfall(num_bytes) if group.rank_in_group == 0 else None
    error = group.broadcast_object(error)
    if error is not None:
        logger.warning_once(
            "Sharding the offloaded Engram tables across DP replicas: %s", error
        )
    return error is None

compute_hash_multipliers(layer_ids, max_ngram_size, compressed_vocab_size)

One multiplier per (layer, lookback), from a per-layer RNG so layers hash differently. Kept odd and bounded so token_id * multiplier cannot overflow int64.

Source code in vllm/models/deepseek_v41/common/engram.py
def compute_hash_multipliers(
    layer_ids: tuple[int, ...], max_ngram_size: int, compressed_vocab_size: int
) -> torch.Tensor:
    """One multiplier per (layer, lookback), from a per-layer RNG so layers
    hash differently. Kept odd and bounded so `token_id * multiplier` cannot
    overflow int64.
    """
    max_long = np.iinfo(np.int64).max
    multiplier_bound = max(1, (max_long // compressed_vocab_size) // 2)
    rows = []
    for layer_id in layer_ids:
        generator = np.random.default_rng(10007 * layer_id)
        values = generator.integers(
            low=0,
            high=multiplier_bound,
            size=(max_ngram_size,),
            dtype=np.int64,
        )
        rows.append(torch.tensor(values * 2 + 1))
    return torch.stack(rows)

engram_gathered_num_tokens()

Per-replica token slot for the node-local Engram DP group.

Source code in vllm/models/deepseek_v41/common/engram.py
def engram_gathered_num_tokens() -> int:
    """Per-replica token slot for the node-local Engram DP group."""
    dp_metadata = get_forward_context().dp_metadata
    if dp_metadata is None:
        raise RuntimeError("a DP-shared engram table needs DP token metadata")
    group = get_engram_dp_group()
    assert group is not None
    # Engram groups are contiguous slices of the full DP group.
    start = get_dp_group().rank_in_group - group.rank_in_group
    return int(
        dp_metadata.num_tokens_across_dp_cpu[start : start + group.world_size].max()
    )

engram_head_shard_rank()

This rank's slot among the hash-head shards of one engram table.

TP-major, so the shards a DP gather brings in are contiguous heads and the following TP gather completes the head order.

Source code in vllm/models/deepseek_v41/common/engram.py
def engram_head_shard_rank() -> int:
    """This rank's slot among the hash-head shards of one engram table.

    TP-major, so the shards a DP gather brings in are contiguous heads and
    the following TP gather completes the head order.
    """
    dp_group = get_engram_dp_group()
    dp_size = dp_group.world_size if dp_group is not None else 1
    dp_rank = dp_group.rank_in_group if dp_group is not None else 0
    return get_tensor_model_parallel_rank() * dp_size + dp_rank

find_next_prime(start, seen_primes)

The smallest prime above start that has not been handed out yet.

Source code in vllm/models/deepseek_v41/common/engram.py
def find_next_prime(start: int, seen_primes: set[int]) -> int:
    """The smallest prime above `start` that has not been handed out yet."""
    candidate = start + 1
    while not _is_prime(candidate) or candidate in seen_primes:
        candidate += 1
    return candidate

gather_engram_hashes(hash_ids, *, dp_shared_memory=False)

Collect the n-gram ids of every DP replica sharing one table.

Replicas are padded to a common token slot, so the gathered shape is static under CUDA graph capture (where DP already pads alike).

Source code in vllm/models/deepseek_v41/common/engram.py
def gather_engram_hashes(
    hash_ids: torch.Tensor, *, dp_shared_memory: bool = False
) -> torch.Tensor:
    """Collect the n-gram ids of every DP replica sharing one table.

    Replicas are padded to a common token slot, so the gathered shape is
    static under CUDA graph capture (where DP already pads alike).
    """
    dp_group = get_engram_dp_group()
    if dp_group is None or dp_shared_memory:
        return hash_ids
    slot = engram_gathered_num_tokens()
    if hash_ids.shape[0] > slot:
        raise ValueError("Engram token count exceeds the DP token slot")
    if hash_ids.shape[0] < slot:
        pad = hash_ids.new_full(
            (slot - hash_ids.shape[0], *hash_ids.shape[1:]), DEAD_ID
        )
        hash_ids = torch.cat((hash_ids, pad))
    return dp_group.all_gather(hash_ids, dim=0)

gather_errors(group, error)

Each rank's error, joined; empty if every rank succeeded.

Source code in vllm/models/deepseek_v41/common/engram.py
def gather_errors(group: GroupCoordinator, error: str | None) -> str:
    """Each rank's error, joined; empty if every rank succeeded."""
    errors: list[str | None] = [None] * group.world_size
    torch.distributed.all_gather_object(errors, error, group=group.cpu_group)
    return "; ".join(
        f"EDP rank {rank}: {error}" for rank, error in enumerate(errors) if error
    )

join_engram_peer_group(size, path) cached

This GPU's membership, shared by all of its process's tables.

Source code in vllm/models/deepseek_v41/common/engram.py
@cache
def join_engram_peer_group(size: int, path: str) -> EngramPeerGroup:
    """This GPU's membership, shared by all of its process's tables."""
    return EngramPeerGroup(size, path)