Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ def __init__(

def raw_orthogonalize_fn(grad: torch.Tensor) -> torch.Tensor:
# Adaptive variants normalize the raw Newton-Schulz output first; Muon scale is applied after moment2.
logging.debug(f"Orthogonalizing grad with {num_ns_steps} steps, {coefficient_type} coefficient")
logging.debug("Orthogonalizing grad with %s steps, %s coefficient", num_ns_steps, coefficient_type)
return muon_utils.newton_schulz(
grad,
steps=num_ns_steps,
Expand All @@ -120,7 +120,7 @@ def raw_orthogonalize_fn(grad: torch.Tensor) -> torch.Tensor:

def _apply_muon_scale(self, update: torch.Tensor, size_out: int, size_in: int) -> torch.Tensor:
scale_factor = muon.get_muon_scale_factor(size_out, size_in, mode=self.scale_mode)
logging.debug(f"Applying Muon scale factor {scale_factor}, extra_scale_factor={self.extra_scale_factor}")
logging.debug("Applying Muon scale factor %s, extra_scale_factor=%s", scale_factor, self.extra_scale_factor)
return update * scale_factor * self.extra_scale_factor

def _match_frobenius_norm(
Expand Down
10 changes: 7 additions & 3 deletions emerging_optimizers/orthogonalized_optimizers/muon.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,14 +100,18 @@ def __init__(
use_syrk = False
elif sm_version not in ((8, 0), (9, 0), (10, 0), (10, 3)):
logging.error(
f"Correctness of Triton kernel on SM {sm_version} cannot be guaranteed. Setting use_syrk to False."
"Correctness of Triton kernel on SM %s cannot be guaranteed. Setting use_syrk to False.",
sm_version,
)
use_syrk = False

def scaled_orthogonalize_fn(grad: torch.Tensor) -> torch.Tensor:
logging.debug(
f"Orthogonalizing grad with {num_ns_steps} steps, {coefficient_type} coefficient, "
f"{scale_mode} scale mode, extra_scale_factor={extra_scale_factor}"
"Orthogonalizing grad with %s steps, %s coefficient, %s scale mode, extra_scale_factor=%s",
num_ns_steps,
coefficient_type,
scale_mode,
extra_scale_factor,
)
orth_grad = muon_utils.newton_schulz(
grad,
Expand Down
4 changes: 2 additions & 2 deletions emerging_optimizers/orthogonalized_optimizers/muon_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,8 @@ def get_coefficient_iterator(
ValueError: If coefficient_sets is empty.
ValueError: If an invalid mode is provided.
"""
logging.debug(f"Iterating through {steps} steps with {mode} mode.")
logging.debug(f"Coefficient sets: {coefficient_sets}")
logging.debug("Iterating through %s steps with %s mode.", steps, mode)
logging.debug("Coefficient sets: %s", coefficient_sets)

if not coefficient_sets:
raise ValueError("coefficient_sets must be non-empty.")
Expand Down
7 changes: 5 additions & 2 deletions emerging_optimizers/orthogonalized_optimizers/polargrad.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,11 @@ def __init__(

def scaled_orthogonalize_fn(grad: torch.Tensor) -> torch.Tensor:
logging.debug(
f"Orthogonalizing grad with {num_ns_steps} steps, {coefficient_type} coefficient, "
f"multiplied with the nuclear norm of grad, extra_scale_factor={extra_scale_factor}"
"Orthogonalizing grad with %s steps, %s coefficient, "
"multiplied with the nuclear norm of grad, extra_scale_factor=%s",
num_ns_steps,
coefficient_type,
extra_scale_factor,
)
orth_grad = muon_utils.newton_schulz(
grad,
Expand Down
5 changes: 4 additions & 1 deletion emerging_optimizers/orthogonalized_optimizers/scion.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,10 @@ def __init__(

def scaled_orthogonalize_fn(grad: torch.Tensor) -> torch.Tensor:
logging.debug(
f"Orthogonalizing grad with {num_ns_steps} steps, {coefficient_type} coefficient, spectral_radius={spectral_radius}"
"Orthogonalizing grad with %s steps, %s coefficient, spectral_radius=%s",
num_ns_steps,
coefficient_type,
spectral_radius,
)
orth_grad = muon_utils.newton_schulz(
grad, steps=num_ns_steps, coefficient_type=coefficient_type, use_syrk=False
Expand Down
2 changes: 1 addition & 1 deletion emerging_optimizers/orthogonalized_optimizers/spel.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ def __init__(
raise ValueError(f"num_ns_steps must be at least 1, got {num_ns_steps}")

def scaled_orthogonalize_fn(X: torch.Tensor) -> torch.Tensor:
logging.debug(f"Orthogonalizing with {num_ns_steps} steps, {coefficient_type} coefficient")
logging.debug("Orthogonalizing with %s steps, %s coefficient", num_ns_steps, coefficient_type)
return muon_utils.newton_schulz(
X,
steps=num_ns_steps,
Expand Down
2 changes: 1 addition & 1 deletion emerging_optimizers/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ def get_optimizer_cls(name: str) -> type[optim.Optimizer]:
>>> opt_cls
<class 'emerging_optimizers.orthogonalized_optimizers.muon.Muon'>
"""
logging.debug(f"Available optimizers: {list(_OPTIMIZERS.keys())}")
logging.debug("Available optimizers: %s", list(_OPTIMIZERS.keys()))
optimizer = _OPTIMIZERS.get(name.lower())
if optimizer is None:
raise ValueError(f"Optimizer {name} not found in the registry.")
Expand Down
2 changes: 1 addition & 1 deletion emerging_optimizers/soap/soap.py
Original file line number Diff line number Diff line change
Expand Up @@ -425,7 +425,7 @@ def update_kronecker_factors_kl_shampoo(
for idx, (eigenbasis, approx_eigvals) in enumerate(zip(eigenbasis_list, eigvals_list, strict=True)):
scale_factor = 1 / grad.shape[idx] * approx_eigvals.clamp_min(eps) ** eigval_exp

logging.debug(f"scale_factor[{idx}]: {scale_factor}")
logging.debug("scale_factor[%s]: %s", idx, scale_factor)
Comment thread
skyw marked this conversation as resolved.

correction = (eigenbasis * scale_factor[None, :]) @ eigenbasis.T

Expand Down
2 changes: 1 addition & 1 deletion emerging_optimizers/utils/eig.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ def eigh_with_fallback(
eigenvalues, eigenvectors = torch.linalg.eigh(x)
except (torch.linalg.LinAlgError, RuntimeError) as e:
if not force_double:
logging.warning(f"Falling back to double precision: {e}")
logging.warning("Falling back to double precision: %s", e)
# Fallback to double precision if the default precision fails
x = x.to(torch.float64)
eigenvalues, eigenvectors = torch.linalg.eigh(x)
Expand Down
8 changes: 6 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -136,14 +136,18 @@ select = [
"F821", # undefined name
"E266", # too many leading '#' for block comment
"I", # isort
"D101", # docstring
"D103",
"D101", # missing docstring in public class
"D103", # missing docstring in public function
"G004", # logging statement uses f-string
]

# Additional rules can be added here
ignore = [
"E501", # Line too long - handled by formatter
]

logger-objects = ["absl.logging"]
Comment thread
skyw marked this conversation as resolved.

[tool.ruff.lint.isort]
known-first-party = ["emerging_optimizers"]
known-third-party = ["examples", "scripts"]
Expand Down
4 changes: 2 additions & 2 deletions tests/test_muon_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -240,8 +240,8 @@ def test_polar_express_and_deepseekv4_10steps_better_than_quintic(self, size, co
l2_norm_diff_polar = torch.norm(out_polar_express.float() - out_svd.float(), p=2)
l2_norm_diff_quintic = torch.norm(out_quintic.float() - out_svd.float(), p=2)

logging.info(f"{coefficient_type} norm difference: {l2_norm_diff_polar:.6f}")
logging.info(f"Quintic norm difference: {l2_norm_diff_quintic:.6f}")
logging.info("%s norm difference: %.6f", coefficient_type, l2_norm_diff_polar)
logging.info("Quintic norm difference: %.6f", l2_norm_diff_quintic)

self.assertLess(
l2_norm_diff_polar,
Expand Down
2 changes: 1 addition & 1 deletion tests/test_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ def test_get_configured_optimizer_smoke(self):

def test_get_optimizer_name_list_all_names_registered(self):
epot_name_list = registry.get_optimizer_name_list()
logging.debug(f"Available optimizers: {epot_name_list}")
logging.debug("Available optimizers: %s", epot_name_list)

self.assertNotEmpty(epot_name_list)

Expand Down
24 changes: 12 additions & 12 deletions tests/test_spectral_clipping_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ def setUp(self):
self.prev_precision = torch.get_float32_matmul_precision()
torch.set_float32_matmul_precision("highest")
self.device = FLAGS.device
logging.info(f"Using device: {self.device}")
logging.info("Using device: %s", self.device)

def tearDown(self):
torch.set_float32_matmul_precision(self.prev_precision)
Expand All @@ -64,11 +64,11 @@ def test_spectral_clipping_clips_singular_values_to_range(self, dims, sigma_rang
min_sv = singular_values.min().item()
max_sv = singular_values.max().item()

logging.debug(f"Original matrix shape: {x.shape}")
logging.debug(f"Original singular values range: [{original_min_sv:.6f}, {original_max_sv:.6f}]")
logging.debug(f"Clipped singular values range: [{min_sv:.6f}, {max_sv:.6f}]")
logging.debug(f"Target range: [{sigma_min:.6f}, {sigma_max:.6f}]")
logging.debug(f"Shape preservation: input {x.shape} -> output {clipped_x.shape}")
logging.debug("Original matrix shape: %s", x.shape)
logging.debug("Original singular values range: [%.6f, %.6f]", original_min_sv, original_max_sv)
logging.debug("Clipped singular values range: [%.6f, %.6f]", min_sv, max_sv)
logging.debug("Target range: [%.6f, %.6f]", sigma_min, sigma_max)
logging.debug("Shape preservation: input %s -> output %s", x.shape, clipped_x.shape)

# use higher tolerance for lower singular values
# typically, this algorithm introduces more error for lower singular values
Expand Down Expand Up @@ -96,8 +96,8 @@ def test_spectral_hardcap(self, dims, beta):
U_orig, original_singular_values, Vt_orig = torch.linalg.svd(x.double(), full_matrices=False)
original_min_sv = original_singular_values.min().item()
original_max_sv = original_singular_values.max().item()
logging.debug(f"Original matrix shape: {x.shape}")
logging.debug(f"Original singular values range: [{original_min_sv:.6f}, {original_max_sv:.6f}]")
logging.debug("Original matrix shape: %s", x.shape)
logging.debug("Original singular values range: [%.6f, %.6f]", original_min_sv, original_max_sv)

hardcapped_x = orthogonalized_optimizers.spectral_hardcap(x, beta=beta)

Expand All @@ -107,9 +107,9 @@ def test_spectral_hardcap(self, dims, beta):

max_sv = singular_values.max().item()

logging.debug(f"Hardcapped max singular value: {max_sv:.6f}")
logging.debug(f"Beta (upper bound): {beta:.6f}")
logging.debug(f"Shape preservation: input {x.shape} -> output {hardcapped_x.shape}")
logging.debug("Hardcapped max singular value: %.6f", max_sv)
logging.debug("Beta (upper bound): %.6f", beta)
logging.debug("Shape preservation: input %s -> output %s", x.shape, hardcapped_x.shape)

self.assertLessEqual(
max_sv - tolerance_upper,
Expand All @@ -126,7 +126,7 @@ def test_spectral_hardcap(self, dims, beta):
relative_polar_frobenius_diff = torch.norm(polar_orig - polar_hard, "fro") / torch.norm(polar_orig, "fro")
polar_tolerance = 1e-4

logging.debug(f"Polar factor Frobenius norm difference: {relative_polar_frobenius_diff:.6f}")
logging.debug("Polar factor Frobenius norm difference: %.6f", relative_polar_frobenius_diff)

self.assertLessEqual(
relative_polar_frobenius_diff,
Expand Down
Loading