Skip to content

vllm.model_executor.kernels.linear.base

Classes:

  • FP8Params –

    FP8 layer parameters with typed fields.

  • LinearKernel –

    Base class of every quantized linear kernel.

  • LinearKernelOracle –

    Shared behaviour of linear kernel selection (dispatch).

  • MMLinearKernel –

    Abstract base class for quantized matrix multiplication kernels.

  • Params –

    Base class for quantized layer parameters.

FP8Params dataclass

Bases: Params

FP8 layer parameters with typed fields.

Methods:

Source code in vllm/model_executor/kernels/linear/base.py
@dataclass(slots=True)
class FP8Params(Params):
    """FP8 layer parameters with typed fields."""

    input_scale_ub: torch.Tensor | None

    INPUT_SCALE_UB: ClassVar[str] = "input_scale_ub"

    @classmethod
    def from_layer(cls, layer: torch.nn.Module) -> "FP8Params":
        """Extract parameters from layer."""
        return cls(
            weight=getattr(layer, cls.WEIGHT),
            weight_scale=getattr(layer, cls.WEIGHT_SCALE),
            input_scale=getattr(layer, cls.INPUT_SCALE, None),
            input_scale_ub=getattr(layer, cls.INPUT_SCALE_UB, None),
        )

from_layer(layer) classmethod

Extract parameters from layer.

Source code in vllm/model_executor/kernels/linear/base.py
@classmethod
def from_layer(cls, layer: torch.nn.Module) -> "FP8Params":
    """Extract parameters from layer."""
    return cls(
        weight=getattr(layer, cls.WEIGHT),
        weight_scale=getattr(layer, cls.WEIGHT_SCALE),
        input_scale=getattr(layer, cls.INPUT_SCALE, None),
        input_scale_ub=getattr(layer, cls.INPUT_SCALE_UB, None),
    )

LinearKernel

Bases: ABC, Generic[_LinearConfigT]

Base class of every quantized linear kernel.

Methods:

Source code in vllm/model_executor/kernels/linear/base.py
class LinearKernel(ABC, Generic[_LinearConfigT]):
    """Base class of every quantized linear kernel."""

    def __init__(self, config: _LinearConfigT) -> None:
        assert self.can_implement(config)[0]
        assert self.is_supported()[0]
        self.config = config

    @classmethod
    @abstractmethod
    def is_supported(
        cls, compute_capability: int | None = None
    ) -> tuple[bool, str | None]:
        """Whether the kernel runs on this device (the current one if
        `compute_capability` is None), and if not, why."""
        raise NotImplementedError

    @classmethod
    @abstractmethod
    def can_implement(cls, config: _LinearConfigT) -> tuple[bool, str | None]:
        """Whether the kernel handles `config`, and if not, why."""
        raise NotImplementedError

    @classmethod
    def supports_batch_invariance(cls) -> bool:
        """Whether a row's output is independent of the other rows in the batch."""
        return False

    @classmethod
    def executed_activation_key(
        cls, activation_quant_key: QuantKey | None
    ) -> QuantKey | None:
        """The activation quantization the kernel runs for a layer requesting
        `activation_quant_key`. Weight-only kernels return None; kernel
        selection then warns about, or under VLLM_STRICT_QUANT_SCHEME forbids,
        the fallback."""
        return activation_quant_key

    def input_quant_key(self) -> QuantKey | None:
        """The activation quant key the kernel accepts pre-quantized (via
        `as_quantized_activation`), or None if it quantizes its own input."""
        return None

    @abstractmethod
    def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
        """Convert the loaded weights on `layer` in place to the kernel's format."""
        raise NotImplementedError

    @abstractmethod
    def apply_weights(
        self,
        layer: torch.nn.Module,
        x: torch.Tensor,
        bias: torch.Tensor | None = None,
    ) -> torch.Tensor:
        """Run the quantized GEMM."""
        raise NotImplementedError

apply_weights(layer, x, bias=None) abstractmethod

Run the quantized GEMM.

Source code in vllm/model_executor/kernels/linear/base.py
@abstractmethod
def apply_weights(
    self,
    layer: torch.nn.Module,
    x: torch.Tensor,
    bias: torch.Tensor | None = None,
) -> torch.Tensor:
    """Run the quantized GEMM."""
    raise NotImplementedError

can_implement(config) abstractmethod classmethod

Whether the kernel handles config, and if not, why.

Source code in vllm/model_executor/kernels/linear/base.py
@classmethod
@abstractmethod
def can_implement(cls, config: _LinearConfigT) -> tuple[bool, str | None]:
    """Whether the kernel handles `config`, and if not, why."""
    raise NotImplementedError

executed_activation_key(activation_quant_key) classmethod

The activation quantization the kernel runs for a layer requesting activation_quant_key. Weight-only kernels return None; kernel selection then warns about, or under VLLM_STRICT_QUANT_SCHEME forbids, the fallback.

Source code in vllm/model_executor/kernels/linear/base.py
@classmethod
def executed_activation_key(
    cls, activation_quant_key: QuantKey | None
) -> QuantKey | None:
    """The activation quantization the kernel runs for a layer requesting
    `activation_quant_key`. Weight-only kernels return None; kernel
    selection then warns about, or under VLLM_STRICT_QUANT_SCHEME forbids,
    the fallback."""
    return activation_quant_key

input_quant_key()

The activation quant key the kernel accepts pre-quantized (via as_quantized_activation), or None if it quantizes its own input.

Source code in vllm/model_executor/kernels/linear/base.py
def input_quant_key(self) -> QuantKey | None:
    """The activation quant key the kernel accepts pre-quantized (via
    `as_quantized_activation`), or None if it quantizes its own input."""
    return None

is_supported(compute_capability=None) abstractmethod classmethod

Whether the kernel runs on this device (the current one if compute_capability is None), and if not, why.

Source code in vllm/model_executor/kernels/linear/base.py
@classmethod
@abstractmethod
def is_supported(
    cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
    """Whether the kernel runs on this device (the current one if
    `compute_capability` is None), and if not, why."""
    raise NotImplementedError

process_weights_after_loading(layer) abstractmethod

Convert the loaded weights on layer in place to the kernel's format.

Source code in vllm/model_executor/kernels/linear/base.py
@abstractmethod
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
    """Convert the loaded weights on `layer` in place to the kernel's format."""
    raise NotImplementedError

supports_batch_invariance() classmethod

Whether a row's output is independent of the other rows in the batch.

Source code in vllm/model_executor/kernels/linear/base.py
@classmethod
def supports_batch_invariance(cls) -> bool:
    """Whether a row's output is independent of the other rows in the batch."""
    return False

LinearKernelOracle

Shared behaviour of linear kernel selection (dispatch).

Linear kernels have no oracle abstraction yet; this only hosts the logic shared by the init_*_linear_kernel selectors, to be extended as they are migrated.

Methods:

Source code in vllm/model_executor/kernels/linear/base.py
class LinearKernelOracle:
    """Shared behaviour of linear kernel selection (dispatch).

    Linear kernels have no oracle abstraction yet; this only hosts the logic
    shared by the `init_*_linear_kernel` selectors, to be extended as they are
    migrated.
    """

    @staticmethod
    def maybe_skip_for_requantization_source() -> bool:
        """Whether to return a placeholder instead of selecting a compute kernel.

        A checkpoint method created as an online requantization source only
        dequantizes its weights, so it needs no linear kernel.
        """
        from vllm.model_executor.layers.quantization.online.requantization import (
            is_requantization_source,
        )

        return is_requantization_source()

maybe_skip_for_requantization_source() staticmethod

Whether to return a placeholder instead of selecting a compute kernel.

A checkpoint method created as an online requantization source only dequantizes its weights, so it needs no linear kernel.

Source code in vllm/model_executor/kernels/linear/base.py
@staticmethod
def maybe_skip_for_requantization_source() -> bool:
    """Whether to return a placeholder instead of selecting a compute kernel.

    A checkpoint method created as an online requantization source only
    dequantizes its weights, so it needs no linear kernel.
    """
    from vllm.model_executor.layers.quantization.online.requantization import (
        is_requantization_source,
    )

    return is_requantization_source()

MMLinearKernel

Bases: LinearKernel[_ConfigT], Generic[_ConfigT, _ParamsT]

Abstract base class for quantized matrix multiplication kernels.

This class provides the interface for implementing custom quantized linear layer kernels in vLLM. Subclasses should implement specific quantization strategies (e.g., FP8, INT8) and their corresponding compute kernels.

Generic Type Parameters

_ConfigT: Configuration type for the kernel (subclass of MMLinearLayerConfig). Contains kernel-specific settings like quantization keys, dtypes, etc. _ParamsT: Parameter type for the kernel (subclass of Params). Defines the quantized weights and scales needed by the kernel.

Typical Usage
  1. Define a config dataclass inheriting from MMLinearLayerConfig
  2. Define a params dataclass inheriting from Params
  3. Subclass MMLinearKernel with your config and params types
  4. Implement all abstract methods
  5. Register the kernel with the quantization method
Example
@dataclass
class MyKernelConfig(MMLinearLayerConfig):
    static: bool
    output_dtype: torch.dtype


@dataclass
class MyKernelParams(FP8Params):
    custom_scale: torch.Tensor
    CUSTOM_SCALE: ClassVar[str] = "custom_scale"


class MyKernel(MMLinearKernel[MyKernelConfig, MyKernelParams]):
    @classmethod
    def is_supported(cls, compute_capability=None):
        if compute_capability and compute_capability < 90:
            return False, "Requires compute capability >= 9.0"
        return True, None

    @classmethod
    def can_implement(cls, config):
        if not config.static:
            return False, "Only static quantization supported"
        return True, None

    def process_weights_after_loading(self, layer):
        # Preprocess weights for the kernel
        params = self._get_layer_params(layer)
        processed = preprocess_weights(params.weight)
        replace_parameter(layer, params.WEIGHT, processed)

    def _get_layer_params(self, layer, **kwargs):
        return MyKernelParams.from_layer(layer)

    def apply_weights(self, layer, x, bias=None, **kwargs):
        params = self._get_layer_params(layer)
        # Call your custom kernel
        output = my_custom_kernel(x, params.weight, params.weight_scale)
        if bias is not None:
            output += bias
        return output

Methods:

  • __init__ –

    Initialize the kernel with the given configuration.

  • get_output_padding –

    Get the number of output tokens to pad for this kernel.

Source code in vllm/model_executor/kernels/linear/base.py
class MMLinearKernel(LinearKernel[_ConfigT], Generic[_ConfigT, _ParamsT]):
    """Abstract base class for quantized matrix multiplication kernels.

    This class provides the interface for implementing custom quantized linear layer
    kernels in vLLM. Subclasses should implement specific quantization strategies
    (e.g., FP8, INT8) and their corresponding compute kernels.

    Generic Type Parameters:
        _ConfigT: Configuration type for the kernel (subclass of MMLinearLayerConfig).
                  Contains kernel-specific settings like quantization keys, dtypes, etc.
        _ParamsT: Parameter type for the kernel (subclass of Params).
                  Defines the quantized weights and scales needed by the kernel.

    Typical Usage:
        1. Define a config dataclass inheriting from MMLinearLayerConfig
        2. Define a params dataclass inheriting from Params
        3. Subclass MMLinearKernel with your config and params types
        4. Implement all abstract methods
        5. Register the kernel with the quantization method

    Example:
        ```python
        @dataclass
        class MyKernelConfig(MMLinearLayerConfig):
            static: bool
            output_dtype: torch.dtype


        @dataclass
        class MyKernelParams(FP8Params):
            custom_scale: torch.Tensor
            CUSTOM_SCALE: ClassVar[str] = "custom_scale"


        class MyKernel(MMLinearKernel[MyKernelConfig, MyKernelParams]):
            @classmethod
            def is_supported(cls, compute_capability=None):
                if compute_capability and compute_capability < 90:
                    return False, "Requires compute capability >= 9.0"
                return True, None

            @classmethod
            def can_implement(cls, config):
                if not config.static:
                    return False, "Only static quantization supported"
                return True, None

            def process_weights_after_loading(self, layer):
                # Preprocess weights for the kernel
                params = self._get_layer_params(layer)
                processed = preprocess_weights(params.weight)
                replace_parameter(layer, params.WEIGHT, processed)

            def _get_layer_params(self, layer, **kwargs):
                return MyKernelParams.from_layer(layer)

            def apply_weights(self, layer, x, bias=None, **kwargs):
                params = self._get_layer_params(layer)
                # Call your custom kernel
                output = my_custom_kernel(x, params.weight, params.weight_scale)
                if bias is not None:
                    output += bias
                return output
        ```

    """

    def __init__(self, config: _ConfigT) -> None:
        """Initialize the kernel with the given configuration.

        Args:
            config: Kernel-specific configuration containing settings like
                   quantization keys, output dtypes, etc.

        """
        self.config = config

    # return a covariant type in the subclass
    @abstractmethod
    def _get_layer_params(self, layer: torch.nn.Module, **kwargs: Any) -> _ParamsT:
        """Extract typed parameters from the layer module.

        This internal method retrieves the quantized weights and scales from
        the layer as a typed parameter object. Subclasses should typically
        delegate to ParamsClass.from_layer().

        Args:
            layer: The layer module containing the parameters
            **kwargs: Additional arguments

        Returns:
            A typed parameter object containing weights, scales, and other
            quantization parameters

        Example:
            ```python
            def _get_layer_params(self, layer, **kwargs):
                return MyKernelParams.from_layer(layer)
            ```

        """
        raise NotImplementedError

    def get_output_padding(self) -> int | None:
        """Get the number of output tokens to pad for this kernel.

        Some kernels require input padding for optimal performance.
        Override this method to specify padding requirements.

        Returns:
            Number of tokens to pad, or None for no padding (default)

        """
        return None

    @abstractmethod
    def apply_weights(
        self,
        layer: torch.nn.Module,
        x: torch.Tensor,
        bias: torch.Tensor | None = None,
        **kwargs: Any,
    ) -> torch.Tensor:
        raise NotImplementedError

__init__(config)

Initialize the kernel with the given configuration.

Parameters:

  • config

    (_ConfigT) –

    Kernel-specific configuration containing settings like quantization keys, output dtypes, etc.

Source code in vllm/model_executor/kernels/linear/base.py
def __init__(self, config: _ConfigT) -> None:
    """Initialize the kernel with the given configuration.

    Args:
        config: Kernel-specific configuration containing settings like
               quantization keys, output dtypes, etc.

    """
    self.config = config

_get_layer_params(layer, **kwargs) abstractmethod

Extract typed parameters from the layer module.

This internal method retrieves the quantized weights and scales from the layer as a typed parameter object. Subclasses should typically delegate to ParamsClass.from_layer().

Parameters:

  • layer

    (Module) –

    The layer module containing the parameters

  • **kwargs

    (Any, default: {} ) –

    Additional arguments

Returns:

  • _ParamsT –

    A typed parameter object containing weights, scales, and other

  • _ParamsT –

    quantization parameters

Example
def _get_layer_params(self, layer, **kwargs):
    return MyKernelParams.from_layer(layer)
Source code in vllm/model_executor/kernels/linear/base.py
@abstractmethod
def _get_layer_params(self, layer: torch.nn.Module, **kwargs: Any) -> _ParamsT:
    """Extract typed parameters from the layer module.

    This internal method retrieves the quantized weights and scales from
    the layer as a typed parameter object. Subclasses should typically
    delegate to ParamsClass.from_layer().

    Args:
        layer: The layer module containing the parameters
        **kwargs: Additional arguments

    Returns:
        A typed parameter object containing weights, scales, and other
        quantization parameters

    Example:
        ```python
        def _get_layer_params(self, layer, **kwargs):
            return MyKernelParams.from_layer(layer)
        ```

    """
    raise NotImplementedError

get_output_padding()

Get the number of output tokens to pad for this kernel.

Some kernels require input padding for optimal performance. Override this method to specify padding requirements.

Returns:

  • int | None –

    Number of tokens to pad, or None for no padding (default)

Source code in vllm/model_executor/kernels/linear/base.py
def get_output_padding(self) -> int | None:
    """Get the number of output tokens to pad for this kernel.

    Some kernels require input padding for optimal performance.
    Override this method to specify padding requirements.

    Returns:
        Number of tokens to pad, or None for no padding (default)

    """
    return None

Params dataclass

Base class for quantized layer parameters.

This class provides a typed interface for accessing quantized weights and scales from layer modules. It serves as a parameter container that can be extracted from layers and passed to kernel implementations.

Attributes:

  • weight (Tensor) –

    The quantized weight tensor

  • weight_scale (Tensor) –

    weight scaling factors

  • input_scale (Tensor | None) –

    Optional input scaling factors

Class Variables

WEIGHT: Attribute name for weight tensor on the layer module WEIGHT_SCALE: Attribute name for weight scale tensor on the layer module INPUT_SCALE: Attribute name for input scale tensor on the layer module

Important

The string values of WEIGHT, WEIGHT_SCALE, and INPUT_SCALE class variables MUST match the attribute names used in the corresponding quantization method's create_weights() implementation. For example, if FP8LinearMethod.create_weights() sets layer.weight and layer.weight_scale, then WEIGHT="weight" and WEIGHT_SCALE="weight_scale" must be used here.

Usage
# Extract parameters from a quantized layer
params = Params.from_layer(layer)

# Access typed parameters
output = func(input, params.weight, params.weight_scale)
Source code in vllm/model_executor/kernels/linear/base.py
@dataclass(slots=True)
class Params:
    """Base class for quantized layer parameters.

    This class provides a typed interface for accessing quantized weights and scales
    from layer modules. It serves as a parameter container that can be extracted from
    layers and passed to kernel implementations.

    Attributes:
        weight: The quantized weight tensor
        weight_scale: weight scaling factors
        input_scale: Optional input scaling factors

    Class Variables:
        WEIGHT: Attribute name for weight tensor on the layer module
        WEIGHT_SCALE: Attribute name for weight scale tensor on the layer module
        INPUT_SCALE: Attribute name for input scale tensor on the layer module

    Important:
        The string values of WEIGHT, WEIGHT_SCALE, and INPUT_SCALE class variables
        MUST match the attribute names used in the corresponding quantization method's
        create_weights() implementation.
        For example, if FP8LinearMethod.create_weights()
        sets layer.weight and layer.weight_scale,
        then WEIGHT="weight" and
        WEIGHT_SCALE="weight_scale" must be used here.

    Usage:
        ```python
        # Extract parameters from a quantized layer
        params = Params.from_layer(layer)

        # Access typed parameters
        output = func(input, params.weight, params.weight_scale)
        ```

    """

    weight: torch.Tensor
    weight_scale: torch.Tensor
    input_scale: torch.Tensor | None

    # Attribute names on the layer
    WEIGHT: ClassVar[str] = "weight"
    WEIGHT_SCALE: ClassVar[str] = "weight_scale"
    INPUT_SCALE: ClassVar[str] = "input_scale"

    @classmethod
    def from_layer(cls, layer: torch.nn.Module) -> Self:
        return cls(
            weight=getattr(layer, cls.WEIGHT),
            weight_scale=getattr(layer, cls.WEIGHT_SCALE),
            input_scale=getattr(layer, cls.INPUT_SCALE, None),
        )