2222from emerging_optimizers .orthogonalized_optimizers .orthogonalized_optimizer import OrthogonalizedOptimizer
2323
2424
25- _args_doc = """params: Iterable of parameters to optimize or dicts defining parameter groups
26- lr: The learning rate used by the internal SGD.
27- momentum_beta: The momentum used by the internal SGD.
28- spectral_radius: The spectral radius to use for the update, we are scaling the LMO by this spectral radius.
29- """
30-
31-
3225class Scion (OrthogonalizedOptimizer ):
3326 """Scion: Stochastic CondItional descent with Operator Norms
3427
@@ -50,8 +43,12 @@ class Scion(OrthogonalizedOptimizer):
5043 parameters; those should all be optimized by the appropriate LMO for that layer. For example,
5144 for 1d params, it is scaled by the `ell_inf` radius.
5245
46+
5347 Args:
54- {_args_doc}
48+ params: Iterable of parameters to optimize or dicts defining parameter groups
49+ lr: The learning rate used by the internal SGD.
50+ momentum_beta: The momentum used by the internal SGD.
51+ fp32_matmul_prec: Precision of the matmul operations in optimizer states GEMM operations.
5552 coefficient_type: The type of coefficient set to use for the Newton-Schulz iteration. Can be one of
5653 ["simple", "quintic", "polar_express"].
5754 num_ns_steps: The number of iteration steps to use in the Newton-Schulz iteration.
@@ -81,7 +78,6 @@ def __init__(
8178 logging .info ("Scion does not use weight decay. Setting use_independent_wd to False to allow Franke-Wolfe." )
8279 use_independent_wd = False
8380
84- # Scion does not use Nesterov momentum.
8581 logging .info ("Scion does not use Nesterov momentum. Setting use_nesterov to False." )
8682 use_nesterov = False
8783
@@ -104,6 +100,3 @@ def scaled_orthogonalize_fn(grad: torch.Tensor) -> torch.Tensor:
104100 fp32_matmul_prec ,
105101 scaled_orthogonalize_fn ,
106102 )
107-
108-
109- Scion .__doc__ = Scion .__doc__ .format (_args_doc = _args_doc ) # type: ignore[union-attr]
0 commit comments