Direct weight reload (VLLM_RELOAD_MODE=direct).
Cold start records each tensor's checkpoint layout before post-load processing
(PWAL). begin views re-laid-out tensors back to that layout on their
storage, the loaders write the checkpoint in place, and finish reruns PWAL,
which must keep addresses (captured CUDA graphs) and refill derived tensors.
Between begin and a successful finish the model must not run; there is
no rollback, the next full update repairs it.
Delivery contract: an update loads every parameter cold start loaded (as
model.load_weights reports it; after a dummy start, the first update sets
the record and is not checked itself), except those level 2 sleep preserves
(--sleep-preserve-parameter-names: the update does not rewrite them).
finish checks it before rerunning PWAL, which would otherwise reprocess the
kept runtime bytes of a parameter the update left out.
A parameter flagged fill_on_reload must also be delivered whole: begin
zero-fills each such tensor this model owns whole, as its checkpoint-layout
view (padding the loaders skip or fill, and a load plan skips loader fills);
a tensor shared with another model (draft and target) is zeroed only by its
owner's begin.
Functions:
_check_params_kept(model)
Raise if post-load processing registered a new parameter under a
checkpoint name: begin cannot view it back, and the loader would write
checkpoint bytes into its runtime layout.
Source code in vllm/model_executor/model_loader/reload/direct.py
| def _check_params_kept(model: torch.nn.Module) -> None:
"""Raise if post-load processing registered a new parameter under a
checkpoint name: `begin` cannot view it back, and the loader would write
checkpoint bytes into its runtime layout."""
replaced = [
name
for name, p in model.named_parameters(remove_duplicate=False)
if name in model._ckpt_param_names and not hasattr(p, "_ckpt")
]
if replaced:
raise RuntimeError(
f"direct reload: post-load processing replaced the parameter object "
f"of {len(replaced)} checkpoint tensors (e.g. {replaced[:3]}); keep "
"it with `replace_parameter` / `retire_parameter` instead"
)
|
_check_update_complete(model, model_config)
Raise unless the update loaded every parameter cold start loaded, except
those level 2 sleep preserves because the update does not rewrite them.
load_weights reports parameter names, whatever checkpoint names fed
them (fused or per-expert); a tied parameter counts under either name.
Source code in vllm/model_executor/model_loader/reload/direct.py
| def _check_update_complete(model: torch.nn.Module, model_config: ModelConfig) -> None:
"""Raise unless the update loaded every parameter cold start loaded, except
those level 2 sleep preserves because the update does not rewrite them.
`load_weights` reports parameter names, whatever checkpoint names fed
them (fused or per-expert); a tied parameter counts under either name."""
loaded = model.__dict__.pop("_loaded_params", None)
required = model.__dict__.get("_required_params")
if loaded is None or required is None: # the model does not report loads
return
if not required: # dummy start: the first update sets what is required
model._required_params = loaded
return
preserved = getattr(model_config, "sleep_preserve_parameter_names", ())
params = dict(model.named_parameters(remove_duplicate=False))
delivered = {params.get(n, n) for n in loaded}
missing = sorted(
n
for n in required
if params.get(n, n) not in delivered
and not any(fnmatchcase(n, p) for p in preserved)
)
if missing:
raise RuntimeError(
f"direct reload: the update did not load {len(missing)} parameters "
f"cold start loaded (e.g. {missing[:8]}); every update must resend "
"them, start a new full update"
)
|
_checkpoint_views(model)
(tensor, view of its storage in its checkpoint layout) for every
tensor post-load processing re-laid out; raises if a storage cannot
hold its checkpoint layout.
Source code in vllm/model_executor/model_loader/reload/direct.py
| def _checkpoint_views(
model: torch.nn.Module,
) -> list[tuple[torch.Tensor, torch.Tensor]]:
"""(tensor, view of its storage in its checkpoint layout) for every
tensor post-load processing re-laid out; raises if a storage cannot
hold its checkpoint layout."""
views: list[tuple[torch.Tensor, torch.Tensor]] = []
for name, t in loadable_tensors(model):
ckpt = getattr(t, "_ckpt", None)
if ckpt is None:
continue # no layout: created by post-load processing, derived
if _layout(t) == ckpt[1]:
continue
if ckpt[0] is not model._ckpt_owner:
# shared from another model (a target lm_head in a draft): this
# finish would reprocess bytes a draft update does not resend
raise RuntimeError(
f"direct reload: {name} is shared from another model and its "
"post-load processing changed its layout"
)
shape, stride, dtype, offset = ckpt[1]
storage = t.untyped_storage()
if storage.data_ptr() != ckpt[3]: # moved, e.g. `set_` into a fused buffer
offset = t.storage_offset()
span = 0 if 0 in shape else 1 + sum((n - 1) * s for n, s in zip(shape, stride))
need = (offset + span) * dtype.itemsize
if need > storage.nbytes():
raise RuntimeError(
f"direct reload: {name} storage has {storage.nbytes()} bytes, "
f"its checkpoint layout needs {need}"
)
views.append((t, storage_view(storage, offset, shape, stride, dtype)))
return views
|
begin_direct_reload(model)
Make every tensor writable by its checkpoint weight loader.
Everything that can refuse the reload is checked first and raises with
the model untouched. Once this returns, the model holds checkpoint-layout
bytes until a finish succeeds.
Source code in vllm/model_executor/model_loader/reload/direct.py
| @torch.no_grad()
def begin_direct_reload(model: torch.nn.Module) -> None:
"""Make every tensor writable by its checkpoint weight loader.
Everything that can refuse the reload is checked first and raises with
the model untouched. Once this returns, the model holds checkpoint-layout
bytes until a finish succeeds."""
if not hasattr(model, "_ckpt_param_names"):
raise RuntimeError("direct reload: no checkpoint layout was recorded")
_check_params_kept(model)
views = _checkpoint_views(model)
# nothing above modified the model
model._direct_reload_begun = True # one finish per begin
track_loaded_params(model)
for t, view in views:
t.data = view # no zeroing: the full sync rewrites the checkpoint region
for _, t in loadable_tensors(model):
if not _owned(model, t):
continue
for k in _LOADER_ATTRS:
t.__dict__.pop(k, None)
t.__dict__.update(t._ckpt[2])
if getattr(t, "fill_on_reload", False):
# Reset padding: loaders skip it or fill it, and a load plan skips
# loader fills. Erases the checkpoint region too.
t.data.zero_()
logger.info(
"direct reload: %d tensors viewed back to checkpoint layout", len(views)
)
|
direct_reload_active(model)
Whether model is between begin_direct_reload and its finish.
Source code in vllm/model_executor/model_loader/reload/direct.py
| def direct_reload_active(model: torch.nn.Module) -> bool:
"""Whether `model` is between `begin_direct_reload` and its finish."""
return getattr(model, "_direct_reload_begun", False)
|
direct_reload_enabled()
Whether weight updates reload in place (VLLM_RELOAD_MODE=direct).
Source code in vllm/model_executor/model_loader/reload/direct.py
| def direct_reload_enabled() -> bool:
"""Whether weight updates reload in place (`VLLM_RELOAD_MODE=direct`)."""
return envs.VLLM_RELOAD_MODE == "direct"
|
finish_direct_reload(model, model_config, device)
Rerun cold-start post-load processing.
Source code in vllm/model_executor/model_loader/reload/direct.py
| @torch.no_grad()
def finish_direct_reload(
model: torch.nn.Module, model_config: ModelConfig, device: torch.device
) -> None:
"""Rerun cold-start post-load processing."""
from vllm.model_executor.model_loader.utils import process_weights_after_loading
if not model.__dict__.pop("_direct_reload_begun", False):
# e.g. a retry after a failed finish, which may have processed some
# modules already: only a new full update repairs the model
raise RuntimeError(
"direct reload: finish without a begin; start a new full update"
)
_check_update_complete(model, model_config)
if device.type != "cpu":
torch.accelerator.synchronize() # engines may write on other streams
# One-shot "already processed" guards would make the rerun below a no-op
# and leave the freshly written checkpoint layout in place. Only quant
# methods' guards: other modules set it for other reasons.
for module in model.modules():
if isinstance(getattr(module, "quant_method", None), QuantizeMethodBase):
module.__dict__.pop("_already_called_process_weights_after_loading", None)
process_weights_after_loading(model, model_config, device)
|
loadable_tensors(model)
Parameters and persistent buffers, each object once.
Source code in vllm/model_executor/model_loader/reload/direct.py
| def loadable_tensors(model: torch.nn.Module) -> Iterator[tuple[str, torch.Tensor]]:
"""Parameters and persistent buffers, each object once."""
seen: set[int] = set()
for module_name, module in model.named_modules():
persistent = (
(n, b)
for n, b in module._buffers.items()
if n not in module._non_persistent_buffers_set
)
for name, tensor in [*module._parameters.items(), *persistent]:
if tensor is None or id(tensor) in seen:
continue
seen.add(id(tensor))
yield (f"{module_name}.{name}" if module_name else name), tensor
|
mark_loaded(model, name)
Report parameter name loaded without load_weights (written in
place by a weight transfer engine).
Source code in vllm/model_executor/model_loader/reload/direct.py
| def mark_loaded(model: torch.nn.Module, name: str) -> None:
"""Report parameter `name` loaded without `load_weights` (written in
place by a weight transfer engine)."""
names = model.__dict__.get("_loaded_params")
if names is not None:
names.add(name)
|
record_checkpoint_layout(model)
Cold start, before post-load processing: remember each tensor's
checkpoint layout and loader metadata, which tensors this model owns (those
no other model recorded first), and which parameter names hold checkpoint
tensors.
Runs once; reloads reuse the cold-start record.
Source code in vllm/model_executor/model_loader/reload/direct.py
| def record_checkpoint_layout(model: torch.nn.Module) -> None:
"""Cold start, before post-load processing: remember each tensor's
checkpoint layout and loader metadata, which tensors this model owns (those
no other model recorded first), and which parameter names hold checkpoint
tensors.
Runs once; reloads reuse the cold-start record."""
if hasattr(model, "_ckpt_param_names") or is_weights_pre_processed():
return # pre-processed weights are already in runtime layout
_check_supported_modules(model)
# what cold start loaded, every update must load again; after a dummy
# start (nothing loaded) the first update sets it
model._required_params = model.__dict__.pop("_loaded_params", None)
# a model owns the tensors it records first; `begin` refuses a shared one
# (a target lm_head in a draft) whose post-load processing re-laid it out
model._ckpt_owner = object() # tags owned tensors (ids of freed ones recur)
for _, t in loadable_tensors(model):
if not t.is_meta and not hasattr(t, "_ckpt"):
attrs = {k: t.__dict__[k] for k in _LOADER_ATTRS if k in t.__dict__}
storage = t.untyped_storage().data_ptr()
t._ckpt = (model._ckpt_owner, _layout(t), attrs, storage)
# every name, aliases included: post-load processing may replace either
model._ckpt_param_names = frozenset(
name
for name, p in model.named_parameters(remove_duplicate=False)
if hasattr(p, "_ckpt")
)
# reruns skip modules added after cold start (LoRA wrappers)
model._ckpt_modules = list(model.named_modules())
|
track_loaded_params(model)
Collect the parameter names model.load_weights returns until the
cold-start record or the reload's finish; None once a call returns
None (the model does not report them: nothing is checked).
Source code in vllm/model_executor/model_loader/reload/direct.py
| def track_loaded_params(model: torch.nn.Module) -> None:
"""Collect the parameter names `model.load_weights` returns until the
cold-start record or the reload's finish; `None` once a call returns
`None` (the model does not report them: nothing is checked)."""
load_weights = getattr(model, "load_weights", None)
if load_weights is None: # a bare module: nothing to check
return
if not getattr(load_weights, "_tracks_loads", False):
def tracked(*args, **kwargs):
loaded = load_weights(*args, **kwargs)
names = model.__dict__.get("_loaded_params")
if names is not None:
if loaded is None:
model._loaded_params = None
else:
names.update(loaded)
return loaded
tracked._tracks_loads = True # type: ignore[attr-defined]
model.load_weights = tracked
model._loaded_params = set()
|