apply fix from adapter-hub#770

julian-fong · Dec 24, 2024 · 20ddb5c · 20ddb5c
1 parent 1229fc5
commit 20ddb5c
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/src/adapters/methods/lora.py b/src/adapters/methods/lora.py
@@ -259,6 +259,7 @@ def forward(self, hidden_states: Optional[torch.Tensor], layer_input: torch.Tens
             hidden_states = hidden_states * gate
         else:
             gate = None
+            hidden_states = hidden_states * self.scaling
 
         return hidden_states, gate