Skip to content

Commit 3aed07a

Browse files
committed
More fixes following review comments, test passed
Signed-off-by: Hua Huang <huah@nvidia.com>
1 parent 56879c2 commit 3aed07a

2 files changed

Lines changed: 9 additions & 33 deletions

File tree

emerging_optimizers/orthogonalized_optimizers/muon_utils.py

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -207,14 +207,11 @@ def newton_schulz(
207207
else:
208208
raise ValueError(f"Invalid coefficient type: {coefficient_type}")
209209

210-
if coefficient_type == "cubic5" and steps > len(coefficient_sets):
211-
logging.warning(
212-
"cubic5 is a fixed %d-step schedule, but steps=%d was requested. "
213-
"Skipping the extra Newton-Schulz steps.",
214-
len(coefficient_sets),
215-
steps,
210+
if coefficient_type == "cubic5" and steps != len(coefficient_sets):
211+
raise ValueError(
212+
f"cubic5 is a fixed {len(coefficient_sets)}-step schedule; got steps={steps}. "
213+
"Use steps=5, or pass explicit custom coefficients."
216214
)
217-
steps = len(coefficient_sets)
218215

219216
repeat_last_types = ("polar_express", "cans", "deepseekv4")
220217
iter_mode: CoeffIterMode = "repeat_last" if coefficient_type in repeat_last_types else "cycle"

tests/test_muon_utils.py

Lines changed: 5 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -163,32 +163,11 @@ def test_cubic5_close_to_reference(self, dim1, dim2):
163163
rtol=1e-7,
164164
)
165165

166-
@parameterized.product(
167-
size=[(511, 513), (511, 257), (257, 513)],
168-
steps=[1, 2, 3, 4],
169-
)
170-
def test_cubic5_prefix_close_to_reference(self, size, steps):
171-
dim1, dim2 = size
172-
x = torch.randn(dim1, dim2, device=self.device, dtype=torch.float32)
173-
out_cubic5_test = muon_utils.newton_schulz(x, steps=steps, coefficient_type="cubic5")
174-
out_cubic5_ref = newton_schulz_ref(
175-
x,
176-
coefficient_sets=muon_utils._COEFFICIENT_SETS["cubic5"][:steps],
177-
)
178-
179-
torch.testing.assert_close(
180-
out_cubic5_test,
181-
out_cubic5_ref,
182-
atol=1e-6,
183-
rtol=1e-7,
184-
)
185-
186-
def test_cubic5_too_many_steps_matches_five_step_schedule(self) -> None:
166+
@parameterized.parameters(1, 4, 6)
167+
def test_cubic5_wrong_step_count_raises_value_error(self, steps) -> None:
187168
x = torch.randn(5, 7, device=self.device, dtype=torch.float32)
188-
out_cubic5_6 = muon_utils.newton_schulz(x, steps=6, coefficient_type="cubic5")
189-
out_cubic5_5 = muon_utils.newton_schulz(x, steps=5, coefficient_type="cubic5")
190-
191-
torch.testing.assert_close(out_cubic5_6, out_cubic5_5, atol=0, rtol=0)
169+
with self.assertRaisesRegex(ValueError, "cubic5.*fixed.*5-step schedule.*steps=5"):
170+
muon_utils.newton_schulz(x, steps=steps, coefficient_type="cubic5")
192171

193172
@parameterized.parameters(
194173
(511, 513),
@@ -430,7 +409,7 @@ def test_batched_newton_schulz_step_close_to_unbatched(self, batch, dim1, dim2):
430409
(4, 16, 32),
431410
(3, 32, 16),
432411
)
433-
def test_batched_cubic_newton_schulz_step_matches_formula(self, batch, dim1, dim2):
412+
def test_batched_cubic_newton_schulz_step_close_to_formula(self, batch, dim1, dim2):
434413
x = torch.randint(-3, 4, (batch, dim1, dim2), device=self.device, dtype=torch.float32)
435414
x = x / x.norm(dim=(-2, -1), keepdim=True).clamp_min_(1e-7)
436415

0 commit comments

Comments
 (0)