[Minor] Remove ortho init for DAS

jasonlim131 · Jun 3, 2024 · 48024f8 · 48024f8
1 parent f92a379
commit 48024f8
Showing 1 changed file with 4 additions and 4 deletions.
diff --git a/pyvene/models/layers.py b/pyvene/models/layers.py
@@ -22,8 +22,8 @@ def __init__(self, n, init_orth=True):
         # we don't need init if the saved checkpoint has a nice
         # starting point already.
         # you can also study this if you want, but it is our focus.
-        if init_orth:
-            torch.nn.init.orthogonal_(weight)
+        # if init_orth:
+        #     torch.nn.init.orthogonal_(weight)
         self.weight = torch.nn.Parameter(weight, requires_grad=True)
 
     def forward(self, x):
@@ -37,7 +37,7 @@ def __init__(self, n, m):
         super().__init__()
         # n > m
         self.weight = torch.nn.Parameter(torch.empty(n, m), requires_grad=True)
-        torch.nn.init.orthogonal_(self.weight)
+        # torch.nn.init.orthogonal_(self.weight)
 
     def forward(self, x):
         return torch.matmul(x.to(self.weight.dtype), self.weight)
@@ -50,7 +50,7 @@ def __init__(self, n, m):
         super().__init__()
         # n > m
         self.weight = torch.nn.Parameter(torch.empty(n, m), requires_grad=True)
-        torch.nn.init.orthogonal_(self.weight)
+        # torch.nn.init.orthogonal_(self.weight)
 
     def forward(self, x, l, r):
         return torch.matmul(x.to(self.weight.dtype), self.weight[:, l:r])