class GteNewMLP(nn.Module):
"""GTE's GeGLU MLP in the checkpoint's fused `[up; gate]` layout (no
`up_gate_proj` bias), so LoRA adapters on `up_gate_proj` apply as trained."""
def __init__(
self,
hidden_size: int,
intermediate_size: int,
bias: bool = True,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
self.act_fn = get_act_fn("gelu")
self.up_gate_proj = MergedColumnParallelLinear(
hidden_size,
[intermediate_size] * 2,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.up_gate_proj",
)
self.down_proj = RowParallelLinear(
input_size=intermediate_size,
output_size=hidden_size,
bias=bias,
quant_config=quant_config,
prefix=f"{prefix}.down_proj",
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
up_gate, _ = self.up_gate_proj(hidden_states)
up, gate = up_gate.chunk(2, dim=-1)
hidden_states, _ = self.down_proj(self.act_fn(gate) * up)
return hidden_states