Skip to content

vllm.config.engram

Classes:

  • EngramConfig –

    Configuration for Engram embedding storage and sharding.

EngramConfig

Configuration for Engram embedding storage and sharding.

Methods:

Attributes:

Source code in vllm/config/engram.py 
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
@config
class EngramConfig:
    """Configuration for Engram embedding storage and sharding."""

    cpu_offload: bool = Field(default_factory=_default_cpu_offload)
    """Store embedding weights in pinned CPU memory for UVA lookup.
    Defaults to False, or VLLM_PLE_CPU_OFFLOAD when set for compatibility.
    An explicit value takes precedence over the legacy environment variable."""

    embedding_across_dp: bool = False
    """Shard embeddings across TP and all DP ranks when enabled.
    Otherwise, each DP rank has a separate TP-sharded embedding replica."""

    def verify_model_config(self, model_config: "ModelConfig | None") -> None:
        """Reject Engram configuration for models without n-gram embeddings."""
        from vllm.platforms import current_platform

        field = (
            _NGRAM_LAYER_FIELDS.get(model_config.architecture)
            if model_config is not None
            else None
        )
        if (
            model_config is None
            or field is None
            or not current_platform.is_cuda()
            or not getattr(model_config.hf_text_config, field, None)
        ):
            raise ValueError(
                "EngramConfig requires a model with supported Engram "
                "embeddings. Currently only the CUDA Qwen4Exp and DeepSeek "
                "V4.1 implementations with non-empty n-gram layer ids are "
                "supported."
            )

    def verify_parallel_config(self, parallel_config: "ParallelConfig") -> None:
        """Reject unsupported embedding parallel topologies."""
        if (
            self.embedding_across_dp
            and parallel_config.data_parallel_size > 1
            and parallel_config.enable_elastic_ep
        ):
            raise ValueError(
                "Engram embedding_across_dp is not supported with elastic EP yet."
            )

    def get_parallel_size(self, parallel_config: "ParallelConfig") -> int:
        """Derive the embedding group size from the parallel configuration."""
        size = parallel_config.tensor_parallel_size
        if self.embedding_across_dp and parallel_config.data_parallel_size > 1:
            size *= parallel_config.data_parallel_size
        return size

    def compute_hash(self) -> str:
        """Hash settings that affect embedding execution and graph structure."""
        return hash_factors(get_hash_factors(self, set()))

cpu_offload = Field(default_factory=_default_cpu_offload) class-attribute instance-attribute

Store embedding weights in pinned CPU memory for UVA lookup. Defaults to False, or VLLM_PLE_CPU_OFFLOAD when set for compatibility. An explicit value takes precedence over the legacy environment variable.

embedding_across_dp = False class-attribute instance-attribute

Shard embeddings across TP and all DP ranks when enabled. Otherwise, each DP rank has a separate TP-sharded embedding replica.

compute_hash()

Hash settings that affect embedding execution and graph structure.

Source code in vllm/config/engram.py 
90
91
92
def compute_hash(self) -> str:
    """Hash settings that affect embedding execution and graph structure."""
    return hash_factors(get_hash_factors(self, set()))

get_parallel_size(parallel_config)

Derive the embedding group size from the parallel configuration.

Source code in vllm/config/engram.py 
83
84
85
86
87
88
def get_parallel_size(self, parallel_config: "ParallelConfig") -> int:
    """Derive the embedding group size from the parallel configuration."""
    size = parallel_config.tensor_parallel_size
    if self.embedding_across_dp and parallel_config.data_parallel_size > 1:
        size *= parallel_config.data_parallel_size
    return size

verify_model_config(model_config)

Reject Engram configuration for models without n-gram embeddings.

Source code in vllm/config/engram.py 
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
def verify_model_config(self, model_config: "ModelConfig | None") -> None:
    """Reject Engram configuration for models without n-gram embeddings."""
    from vllm.platforms import current_platform

    field = (
        _NGRAM_LAYER_FIELDS.get(model_config.architecture)
        if model_config is not None
        else None
    )
    if (
        model_config is None
        or field is None
        or not current_platform.is_cuda()
        or not getattr(model_config.hf_text_config, field, None)
    ):
        raise ValueError(
            "EngramConfig requires a model with supported Engram "
            "embeddings. Currently only the CUDA Qwen4Exp and DeepSeek "
            "V4.1 implementations with non-empty n-gram layer ids are "
            "supported."
        )

verify_parallel_config(parallel_config)

Reject unsupported embedding parallel topologies.

Source code in vllm/config/engram.py 
72
73
74
75
76
77
78
79
80
81
def verify_parallel_config(self, parallel_config: "ParallelConfig") -> None:
    """Reject unsupported embedding parallel topologies."""
    if (
        self.embedding_across_dp
        and parallel_config.data_parallel_size > 1
        and parallel_config.enable_elastic_ep
    ):
        raise ValueError(
            "Engram embedding_across_dp is not supported with elastic EP yet."
        )

_default_cpu_offload()

Honor the legacy environment variable only when the field is omitted.

Source code in vllm/config/engram.py 
27
28
29
30
31
32
33
34
def _default_cpu_offload() -> bool:
    """Honor the legacy environment variable only when the field is omitted."""
    if envs.is_set("VLLM_PLE_CPU_OFFLOAD"):
        logger.warning_once(
            "VLLM_PLE_CPU_OFFLOAD is a legacy setting and may be removed in a "
            "future release. Use --engram-config.cpu_offload instead."
        )
    return envs.VLLM_PLE_CPU_OFFLOAD