Skip to content

vllm.entrypoints.generate.base.protocol

Classes:

Functions:

SpeculativeDecodingMetrics

Bases: OpenAIBaseModel

Per-request speculative-decoding acceptance metrics.

Experimental, subject to change. Only populated for single-sequence requests (n == 1); null for n > 1, mirroring the timing metrics.

Source code in vllm/entrypoints/generate/base/protocol.py 
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
class SpeculativeDecodingMetrics(OpenAIBaseModel):
    """Per-request speculative-decoding acceptance metrics.

    Experimental, subject to change. Only populated for single-sequence requests
    (`n == 1`); `null` for `n > 1`, mirroring the timing metrics.
    """

    mean_acceptance_length: float
    draft_acceptance_rate: float
    # Dense histogram: index j holds the number of verify steps that accepted
    # exactly j draft tokens (length num_spec_tokens + 1). Excludes the
    # always-accepted bonus token.
    acceptance_histogram: list[int]
    num_spec_steps: int
    num_accepted_draft_tokens: int
    num_draft_tokens: int
    num_spec_tokens: int
    # Ordered per-verify-step arrays; populated only at the `detailed` level.
    per_step_accepted: list[int] | None = None
    per_step_drafted: list[int] | None = None

structured_outputs_from_response_format(structured_outputs, response_format)

Apply response_format overrides to structured_outputs.

Source code in vllm/entrypoints/generate/base/protocol.py 
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
def structured_outputs_from_response_format(
    structured_outputs: StructuredOutputsParams | None,
    response_format: AnyResponseFormat | None,
) -> StructuredOutputsParams | None:
    """Apply ``response_format`` overrides to ``structured_outputs``."""
    if response_format is None or response_format.type == "text":
        return structured_outputs

    overrides: dict[str, Any]
    if response_format.type == "json_object":
        overrides = {"json_object": True}
    elif response_format.type == "json_schema":
        json_schema = response_format.json_schema
        assert json_schema is not None
        overrides = {"json": json_schema.json_schema}
    else:
        assert isinstance(
            response_format,
            (
                LegacyStructuralTagResponseFormat,
                StructuralTagResponseFormat,
            ),
        )
        overrides = {
            "structural_tag": json.dumps(response_format.model_dump(by_alias=True))
        }

    if structured_outputs is None:
        return StructuredOutputsParams(**overrides)

    return replace(structured_outputs, **overrides)

validate_cache_salt(cache_salt)

Validate cache salts before they reach downstream cache backends.

Source code in vllm/entrypoints/generate/base/protocol.py 
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
def validate_cache_salt(cache_salt: object) -> None:
    """Validate cache salts before they reach downstream cache backends."""
    if cache_salt is None:
        return
    if not isinstance(cache_salt, str) or not cache_salt:
        raise VLLMValidationError(
            "Parameter 'cache_salt' must be a non-empty string if provided.",
            parameter="cache_salt",
        )
    if len(cache_salt) > _MAX_CACHE_SALT_LENGTH or any(
        char in _CACHE_SALT_FORBIDDEN_CHARS for char in cache_salt
    ):
        raise VLLMValidationError(
            "Parameter 'cache_salt' must be at most 128 characters and must "
            "not contain '@', '/', '\\\\', or NUL.",
            parameter="cache_salt",
        )

validate_structural_tag_response_format(response_format)

Validate structural tags before they are sent to the engine.

Engine-side validation reports malformed structural tags as generation failures. OpenAI request parsing should classify them as bad requests.

Source code in vllm/entrypoints/generate/base/protocol.py 
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
def validate_structural_tag_response_format(
    response_format: AnyStructuralTagResponseFormat | dict[str, Any],
) -> None:
    """Validate structural tags before they are sent to the engine.

    Engine-side validation reports malformed structural tags as generation
    failures. OpenAI request parsing should classify them as bad requests.
    """
    from pydantic import TypeAdapter, ValidationError

    if isinstance(response_format, dict):
        try:
            response_format = TypeAdapter(
                AnyStructuralTagResponseFormat
            ).validate_python(response_format)
        except ValidationError as exc:
            raise VLLMValidationError(
                "Invalid response_format structural_tag specification.",
                parameter="response_format",
            ) from exc

    try:
        payload = json.dumps(response_format.model_dump(by_alias=True))
        validate_structural_tag_payload(payload, parameter="response_format")
    except (TypeError, ValueError) as exc:
        raise VLLMValidationError(
            "Invalid response_format structural_tag specification.",
            parameter="response_format",
        ) from exc