GPUs of several engines that split each table over their memory. They
meet only at startup (a file store), making the same exchanges in order.
Methods:
-
all_gather
–
Every GPU's value, in rank order. Collective.
Source code in vllm/distributed/engram_pool/group.py
| class PeerGroup:
"""GPUs of several engines that split each table over their memory. They
meet only at startup (a file store), making the same exchanges in order."""
def __init__(self, size: int, path: str) -> None:
self.world_size = size
os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
self.store = torch.distributed.FileStore(path, size)
# Peers load weights at their own pace; their engines wait as long.
self.store.set_timeout(timedelta(seconds=envs.VLLM_ENGINE_READY_TIMEOUT_S))
# Any assignment works: a rank only decides which rows a GPU holds.
self.rank_in_group = self.store.add("next_rank", 1) - 1
if self.rank_in_group >= size:
raise RuntimeError(
f"More than {size} GPUs joined the Engram peer group at {path}; "
"check peer_group_size and use a fresh store path per launch"
)
self._step = 0
def all_gather(self, value: bytes) -> list[bytes]:
"""Every GPU's `value`, in rank order. Collective."""
self._step += 1
keys = [f"{self._step}/{rank}" for rank in range(self.world_size)]
self.store.set(keys[self.rank_in_group], value)
try:
self.store.wait(keys)
except RuntimeError as e:
missing = [r for r, key in enumerate(keys) if not self.store.check([key])]
raise TimeoutError(
f"Engram peer GPUs {missing} did not reach startup step "
f"{self._step}; every engine of the group must start (see "
"VLLM_ENGINE_READY_TIMEOUT_S), and none may restart alone"
) from e
return self.store.multi_get(keys)
def barrier(self) -> None:
self.all_gather(b"")
|
all_gather(value)
Every GPU's value, in rank order. Collective.
Source code in vllm/distributed/engram_pool/group.py
| def all_gather(self, value: bytes) -> list[bytes]:
"""Every GPU's `value`, in rank order. Collective."""
self._step += 1
keys = [f"{self._step}/{rank}" for rank in range(self.world_size)]
self.store.set(keys[self.rank_in_group], value)
try:
self.store.wait(keys)
except RuntimeError as e:
missing = [r for r, key in enumerate(keys) if not self.store.check([key])]
raise TimeoutError(
f"Engram peer GPUs {missing} did not reach startup step "
f"{self._step}; every engine of the group must start (see "
"VLLM_ENGINE_READY_TIMEOUT_S), and none may restart alone"
) from e
return self.store.multi_get(keys)
|