Skip to content

vllm.distributed.engram_pool.verbs

ctypes bindings to libibverbs: just enough for RC QPs and RDMA WRITEs.

Structs and constants mirror rdma-core's verbs.h and mlx5dv.h under their C names (LP64 ABI), as pynccl_wrapper does for NCCL. ibv_post_send, ibv_post_recv, ibv_poll_cq and ibv_req_notify_cq are inline in verbs.h, so they are called through the context's ops table. Only the standard library is imported: the RDMA proxy process loads this file without vllm or torch.

Classes:

  • Device –

    One opened RDMA device and protection domain on port 1.

  • ibv_context –

    Up to the ops called here; the skipped ops are _compat_* stubs or

  • ibv_qp –

    Up to the fields read here.

  • ibv_send_wr –

    With wr.rdma of the wr union. RDMA WRITEs and READs leave the

  • mlx5dv_obj –

    Its qp member's in and out; the other members stay null.

  • mlx5dv_qp_queue –

    The sq and rq members of struct mlx5dv_qp.

Functions:

  • active_devices –

    Names of mlx5 RDMA devices (the WQEs we build are mlx5's) whose port 1

  • local_cpus –

    CPUs on the device's NUMA node.

Device

One opened RDMA device and protection domain on port 1.

Methods:

  • connect –

    Move a QP through INIT, RTR and RTS towards remote_qpn.

  • cq_event –

    Sleep until the next event of channel and consume it.

  • create_cq –

    A CQ, and if wakeable a completion channel (else 0) whose events

  • create_qp –

    An RC QP; it receives only WRITE_WITH_IMM notices. Returns it and

  • mlx5_queue –

    An mlx5 QP's send queue, doorbell record and BlueFlame register, for

  • notify_cq –

    Raise one event on the CQ's channel at its next completion.

  • reg_host –

    Register memory with access NIC_READ, PEER_WRITE or PEER_ORDERED;

Attributes:

Source code in vllm/distributed/engram_pool/verbs.py
class Device:
    """One opened RDMA device and protection domain on port 1."""

    def __init__(self, name: str) -> None:
        lib = _lib()
        count = c_int()
        devices = _check(lib.ibv_get_device_list(ctypes.byref(count)), "list")
        handle = next(
            (
                devices[i]
                for i in range(count.value)
                if lib.ibv_get_device_name(devices[i]).decode() == name
            ),
            None,
        )
        if handle is None:
            raise RuntimeError(f"RDMA device {name} not found")
        self.name = name
        self.context = _check(lib.ibv_open_device(handle), f"open {name}")
        self.pd = _check(lib.ibv_alloc_pd(self.context), "ibv_alloc_pd")
        attr = ibv_device_attr()
        if lib.ibv_query_device(self.context, attr):
            raise RuntimeError(f"ibv_query_device({name}) failed")
        self.max_qp_wr, self.max_sge = attr.max_qp_wr, attr.max_sge
        self.max_cqe = attr.max_cqe
        # READs a QP may have in flight, as initiator and as target.
        self.rd_atomic = min(attr.max_qp_rd_atom, attr.max_qp_init_rd_atom)
        port = ibv_port_attr()
        if lib.ibv_query_port(self.context, 1, port):
            raise RuntimeError(f"ibv_query_port({name}) failed")
        self.mtu, self.lid = port.active_mtu, port.lid
        self.is_ib = port.link_layer == IBV_LINK_LAYER_INFINIBAND
        self.gid_index = _pick_gid_index(name)
        self.gid = ibv_gid()
        if lib.ibv_query_gid(self.context, 1, self.gid_index, self.gid):
            raise RuntimeError(f"ibv_query_gid({name}) failed")
        ops = self.context.contents
        self._post_send = _post_send_t(ops.post_send)
        self._post_recv = _post_recv_t(ops.post_recv)
        self._poll_cq = _poll_cq_t(ops.poll_cq)
        self._req_notify_cq = _req_notify_cq_t(ops.req_notify_cq)
        self._bad = ctypes.byref(c_void_p())
        self._event = c_void_p(), c_void_p()
        self._event_refs = tuple(ctypes.byref(p) for p in self._event)

    @property
    def mtu_bytes(self) -> int:
        """The path MTU (enum ibv_mtu) in bytes."""
        return 128 << self.mtu

    def describe(self) -> str:
        kind = "InfiniBand" if self.is_ib else f"RoCE, GID index {self.gid_index}"
        return f"{self.name} ({kind}, MTU {self.mtu_bytes})"

    def address(self) -> tuple[bytes, int]:
        return bytes(self.gid.raw), self.lid

    def reg_host(self, address: int, num_bytes: int, access: int) -> tuple[int, int]:
        """Register memory with `access` NIC_READ, PEER_WRITE or PEER_ORDERED;
        returns its lkey and rkey."""
        mr = _check(
            _lib().ibv_reg_mr(self.pd, address, num_bytes, access),
            f"ibv_reg_mr({num_bytes} B)",
        ).contents
        return mr.lkey, mr.rkey

    def reg_dmabuf(
        self, fd: int, address: int, num_bytes: int, access: int
    ) -> tuple[int, int]:
        mr = _check(
            _lib().ibv_reg_dmabuf_mr(self.pd, 0, num_bytes, address, fd, access),
            f"ibv_reg_dmabuf_mr({num_bytes} B)",
        ).contents
        return mr.lkey, mr.rkey

    def create_cq(self, depth: int, wakeable: bool = False) -> tuple[int, int]:
        """A CQ, and if `wakeable` a completion channel (else 0) whose events
        (`cq_event`) wake a sleeper."""
        channel = 0
        if wakeable:
            channel = _check(_lib().ibv_create_comp_channel(self.context), "channel")
        cq = _check(
            _lib().ibv_create_cq(self.context, depth, None, channel, 0),
            "ibv_create_cq",
        )
        return cq, channel

    def notify_cq(self, cq: int) -> None:
        """Raise one event on the CQ's channel at its next completion."""
        if error := self._req_notify_cq(cq, 0):
            raise RuntimeError(f"ibv_req_notify_cq failed: {os.strerror(error)}")

    def cq_event(self, channel: int) -> None:
        """Sleep until the next event of `channel` and consume it."""
        lib = _lib()
        _check(lib.ibv_get_cq_event(channel, *self._event_refs) == 0, "get_cq_event")
        lib.ibv_ack_cq_events(self._event[0], 1)

    def post_recv(self, qp: int, wr: ibv_recv_wr) -> None:
        if error := self._post_recv(qp, wr, self._bad):
            raise RuntimeError(f"ibv_post_recv failed: {os.strerror(error)}")

    def create_qp(
        self,
        send_cq: int,
        recv_cq: int,
        max_send_wr: int,
        max_send_sge: int = 1,
        max_inline: int = 0,
        max_recv_wr: int = 1,
    ) -> tuple[int, int]:
        """An RC QP; it receives only WRITE_WITH_IMM notices. Returns it and
        its number."""
        cap = ibv_qp_cap(max_send_wr, max_recv_wr, max_send_sge, 1, max_inline)
        attr = ibv_qp_init_attr(
            send_cq=send_cq, recv_cq=recv_cq, cap=cap, qp_type=IBV_QPT_RC
        )
        qp = _check(_lib().ibv_create_qp(self.pd, attr), "ibv_create_qp").contents
        return ctypes.addressof(qp), qp.qp_num

    def connect(self, qp: int, remote_qpn: int, remote: tuple[bytes, int]) -> None:
        """Move a QP through INIT, RTR and RTS towards `remote_qpn`."""
        init = ibv_qp_attr(
            qp_state=IBV_QPS_INIT,
            qp_access_flags=IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ,
            port_num=1,
        )
        rtr = ibv_qp_attr(
            qp_state=IBV_QPS_RTR,
            path_mtu=self.mtu,
            dest_qp_num=remote_qpn,
            max_dest_rd_atomic=self.rd_atomic,
            min_rnr_timer=12,
        )
        rtr.ah_attr.port_num = 1
        gid, lid = remote
        if self.is_ib:
            rtr.ah_attr.dlid = lid
        else:
            rtr.ah_attr.is_global = 1
            rtr.ah_attr.grh.dgid.raw[:] = gid
            rtr.ah_attr.grh.sgid_index = self.gid_index
            rtr.ah_attr.grh.hop_limit = 64
        rts = ibv_qp_attr(
            qp_state=IBV_QPS_RTS,
            max_rd_atomic=self.rd_atomic,
            timeout=14,
            retry_cnt=7,
            rnr_retry=7,
        )
        for name, attr, mask in (
            ("INIT", init, IBV_QP_ACCESS_FLAGS | IBV_QP_PKEY_INDEX | IBV_QP_PORT),
            (
                "RTR",
                rtr,
                IBV_QP_AV
                | IBV_QP_PATH_MTU
                | IBV_QP_DEST_QPN
                | IBV_QP_RQ_PSN
                | IBV_QP_MAX_DEST_RD_ATOMIC
                | IBV_QP_MIN_RNR_TIMER,
            ),
            (
                "RTS",
                rts,
                IBV_QP_TIMEOUT
                | IBV_QP_RETRY_CNT
                | IBV_QP_RNR_RETRY
                | IBV_QP_SQ_PSN
                | IBV_QP_MAX_QP_RD_ATOMIC,
            ),
        ):
            if _lib().ibv_modify_qp(qp, attr, IBV_QP_STATE | mask):
                err = os.strerror(ctypes.get_errno())
                raise RuntimeError(f"QP {name} failed: {err}")

    @staticmethod
    def mlx5_queue(qp: int) -> mlx5dv_qp:
        """An mlx5 QP's send queue, doorbell record and BlueFlame register, for
        posting without libibverbs."""
        mlx5 = ctypes.CDLL("libmlx5.so.1")
        mlx5.mlx5dv_init_obj.argtypes = [POINTER(mlx5dv_obj), c_uint64]
        out = mlx5dv_qp()
        if mlx5.mlx5dv_init_obj(mlx5dv_obj(qp, ctypes.pointer(out)), MLX5DV_OBJ_QP):
            raise RuntimeError("mlx5dv_init_obj failed: not an mlx5 device?")
        return out

    def post_send(self, qp: int, wr: int) -> None:
        if error := self._post_send(qp, wr, self._bad):
            raise RuntimeError(f"ibv_post_send failed: {os.strerror(error)}")

    def poll_cq(self, cq: int, wc: ctypes.Array, num_entries: int) -> int:
        count = self._poll_cq(cq, num_entries, wc)
        if count < 0:
            raise RuntimeError("ibv_poll_cq failed")
        return count

mtu_bytes property

The path MTU (enum ibv_mtu) in bytes.

connect(qp, remote_qpn, remote)

Move a QP through INIT, RTR and RTS towards remote_qpn.

Source code in vllm/distributed/engram_pool/verbs.py
def connect(self, qp: int, remote_qpn: int, remote: tuple[bytes, int]) -> None:
    """Move a QP through INIT, RTR and RTS towards `remote_qpn`."""
    init = ibv_qp_attr(
        qp_state=IBV_QPS_INIT,
        qp_access_flags=IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ,
        port_num=1,
    )
    rtr = ibv_qp_attr(
        qp_state=IBV_QPS_RTR,
        path_mtu=self.mtu,
        dest_qp_num=remote_qpn,
        max_dest_rd_atomic=self.rd_atomic,
        min_rnr_timer=12,
    )
    rtr.ah_attr.port_num = 1
    gid, lid = remote
    if self.is_ib:
        rtr.ah_attr.dlid = lid
    else:
        rtr.ah_attr.is_global = 1
        rtr.ah_attr.grh.dgid.raw[:] = gid
        rtr.ah_attr.grh.sgid_index = self.gid_index
        rtr.ah_attr.grh.hop_limit = 64
    rts = ibv_qp_attr(
        qp_state=IBV_QPS_RTS,
        max_rd_atomic=self.rd_atomic,
        timeout=14,
        retry_cnt=7,
        rnr_retry=7,
    )
    for name, attr, mask in (
        ("INIT", init, IBV_QP_ACCESS_FLAGS | IBV_QP_PKEY_INDEX | IBV_QP_PORT),
        (
            "RTR",
            rtr,
            IBV_QP_AV
            | IBV_QP_PATH_MTU
            | IBV_QP_DEST_QPN
            | IBV_QP_RQ_PSN
            | IBV_QP_MAX_DEST_RD_ATOMIC
            | IBV_QP_MIN_RNR_TIMER,
        ),
        (
            "RTS",
            rts,
            IBV_QP_TIMEOUT
            | IBV_QP_RETRY_CNT
            | IBV_QP_RNR_RETRY
            | IBV_QP_SQ_PSN
            | IBV_QP_MAX_QP_RD_ATOMIC,
        ),
    ):
        if _lib().ibv_modify_qp(qp, attr, IBV_QP_STATE | mask):
            err = os.strerror(ctypes.get_errno())
            raise RuntimeError(f"QP {name} failed: {err}")

cq_event(channel)

Sleep until the next event of channel and consume it.

Source code in vllm/distributed/engram_pool/verbs.py
def cq_event(self, channel: int) -> None:
    """Sleep until the next event of `channel` and consume it."""
    lib = _lib()
    _check(lib.ibv_get_cq_event(channel, *self._event_refs) == 0, "get_cq_event")
    lib.ibv_ack_cq_events(self._event[0], 1)

create_cq(depth, wakeable=False)

A CQ, and if wakeable a completion channel (else 0) whose events (cq_event) wake a sleeper.

Source code in vllm/distributed/engram_pool/verbs.py
def create_cq(self, depth: int, wakeable: bool = False) -> tuple[int, int]:
    """A CQ, and if `wakeable` a completion channel (else 0) whose events
    (`cq_event`) wake a sleeper."""
    channel = 0
    if wakeable:
        channel = _check(_lib().ibv_create_comp_channel(self.context), "channel")
    cq = _check(
        _lib().ibv_create_cq(self.context, depth, None, channel, 0),
        "ibv_create_cq",
    )
    return cq, channel

create_qp(send_cq, recv_cq, max_send_wr, max_send_sge=1, max_inline=0, max_recv_wr=1)

An RC QP; it receives only WRITE_WITH_IMM notices. Returns it and its number.

Source code in vllm/distributed/engram_pool/verbs.py
def create_qp(
    self,
    send_cq: int,
    recv_cq: int,
    max_send_wr: int,
    max_send_sge: int = 1,
    max_inline: int = 0,
    max_recv_wr: int = 1,
) -> tuple[int, int]:
    """An RC QP; it receives only WRITE_WITH_IMM notices. Returns it and
    its number."""
    cap = ibv_qp_cap(max_send_wr, max_recv_wr, max_send_sge, 1, max_inline)
    attr = ibv_qp_init_attr(
        send_cq=send_cq, recv_cq=recv_cq, cap=cap, qp_type=IBV_QPT_RC
    )
    qp = _check(_lib().ibv_create_qp(self.pd, attr), "ibv_create_qp").contents
    return ctypes.addressof(qp), qp.qp_num

mlx5_queue(qp) staticmethod

An mlx5 QP's send queue, doorbell record and BlueFlame register, for posting without libibverbs.

Source code in vllm/distributed/engram_pool/verbs.py
@staticmethod
def mlx5_queue(qp: int) -> mlx5dv_qp:
    """An mlx5 QP's send queue, doorbell record and BlueFlame register, for
    posting without libibverbs."""
    mlx5 = ctypes.CDLL("libmlx5.so.1")
    mlx5.mlx5dv_init_obj.argtypes = [POINTER(mlx5dv_obj), c_uint64]
    out = mlx5dv_qp()
    if mlx5.mlx5dv_init_obj(mlx5dv_obj(qp, ctypes.pointer(out)), MLX5DV_OBJ_QP):
        raise RuntimeError("mlx5dv_init_obj failed: not an mlx5 device?")
    return out

notify_cq(cq)

Raise one event on the CQ's channel at its next completion.

Source code in vllm/distributed/engram_pool/verbs.py
def notify_cq(self, cq: int) -> None:
    """Raise one event on the CQ's channel at its next completion."""
    if error := self._req_notify_cq(cq, 0):
        raise RuntimeError(f"ibv_req_notify_cq failed: {os.strerror(error)}")

reg_host(address, num_bytes, access)

Register memory with access NIC_READ, PEER_WRITE or PEER_ORDERED; returns its lkey and rkey.

Source code in vllm/distributed/engram_pool/verbs.py
def reg_host(self, address: int, num_bytes: int, access: int) -> tuple[int, int]:
    """Register memory with `access` NIC_READ, PEER_WRITE or PEER_ORDERED;
    returns its lkey and rkey."""
    mr = _check(
        _lib().ibv_reg_mr(self.pd, address, num_bytes, access),
        f"ibv_reg_mr({num_bytes} B)",
    ).contents
    return mr.lkey, mr.rkey

ibv_context

Bases: Structure

Up to the ops called here; the skipped ops are _compat_* stubs or unused.

Source code in vllm/distributed/engram_pool/verbs.py
class ibv_context(Structure):
    """Up to the ops called here; the skipped ops are `_compat_*` stubs or
    unused."""

    _fields_ = [
        ("device", c_void_p),
        ("_ops_0", c_void_p * 11),
        *_fields(c_void_p, "poll_cq req_notify_cq"),
        ("_ops_13", c_void_p * 12),
        *_fields(c_void_p, "post_send post_recv"),
    ]

ibv_qp

Bases: Structure

Up to the fields read here.

Source code in vllm/distributed/engram_pool/verbs.py
class ibv_qp(Structure):
    """Up to the fields read here."""

    _fields_ = [
        *_fields(c_void_p, "context qp_context pd send_cq recv_cq srq"),
        *_fields(c_uint32, "handle qp_num"),
    ]

ibv_send_wr

Bases: Structure

With wr.rdma of the wr union. RDMA WRITEs and READs leave the trailing bind_mw/tso union (spare) unused.

Source code in vllm/distributed/engram_pool/verbs.py
class ibv_send_wr(Structure):
    """With `wr.rdma` of the `wr` union. RDMA WRITEs and READs leave the
    trailing `bind_mw`/`tso` union (`spare`) unused."""

    _fields_ = [
        ("wr_id", c_uint64),
        *_fields(c_void_p, "next sg_list"),
        *_fields(c_int, "num_sge opcode send_flags imm_data"),
        ("remote_addr", c_uint64),
        ("rkey", c_uint32),
        ("_wr", c_uint8 * 20),
        ("remote_srqn", c_uint32),
        ("spare", c_uint64 * 6),
    ]

mlx5dv_obj

Bases: Structure

Its qp member's in and out; the other members stay null.

Source code in vllm/distributed/engram_pool/verbs.py
class mlx5dv_obj(Structure):
    """Its `qp` member's `in` and `out`; the other members stay null."""

    _fields_ = [
        ("qp_in", c_void_p),
        ("qp_out", POINTER(mlx5dv_qp)),
        ("_others", c_void_p * 12),
    ]

mlx5dv_qp_queue

Bases: Structure

The sq and rq members of struct mlx5dv_qp.

Source code in vllm/distributed/engram_pool/verbs.py
class mlx5dv_qp_queue(Structure):
    """The `sq` and `rq` members of struct mlx5dv_qp."""

    _fields_ = [("buf", c_void_p), *_fields(c_uint32, "wqe_cnt stride")]

_pick_gid_index(device)

RoCE v2 with an IPv4-mapped address if present, else index 0 (IB).

Source code in vllm/distributed/engram_pool/verbs.py
def _pick_gid_index(device: str) -> int:
    """RoCE v2 with an IPv4-mapped address if present, else index 0 (IB)."""
    best = 0
    for index in range(16):
        try:
            kind = _sysfs(device, "ports", "1", "gid_attrs", "types", str(index))
            gid = _sysfs(device, "ports", "1", "gids", str(index))
        except OSError:
            continue
        if kind == "RoCE v2" and gid.startswith("0000:0000:0000:0000:0000:ffff"):
            return index
        if kind == "RoCE v2" and int(gid.replace(":", ""), 16):
            best = index
    return best

active_devices()

Names of mlx5 RDMA devices (the WQEs we build are mlx5's) whose port 1 is active. By driver, since udev may rename devices.

Source code in vllm/distributed/engram_pool/verbs.py
def active_devices() -> list[str]:
    """Names of mlx5 RDMA devices (the WQEs we build are mlx5's) whose port 1
    is active. By driver, since udev may rename devices."""
    root = "/sys/class/infiniband"
    if not os.path.isdir(root):
        return []
    return sorted(
        name
        for name in os.listdir(root)
        if os.path.basename(os.path.realpath(f"{root}/{name}/device/driver"))
        == "mlx5_core"
        and _sysfs(name, "ports", "1", "state").endswith("ACTIVE")
    )

local_cpus(device)

CPUs on the device's NUMA node.

Source code in vllm/distributed/engram_pool/verbs.py
def local_cpus(device: str) -> set[int]:
    """CPUs on the device's NUMA node."""
    cpus: set[int] = set()
    for part in _sysfs(device, "device", "local_cpulist").split(","):
        first, _, last = part.partition("-")
        cpus.update(range(int(first), int(last or first) + 1))
    return cpus