Normalize step size to the batch size used

HISTORY.md

@@ -1,5 +1,10 @@
 ### ensmallen ?.??.?: "???"
 ###### ????-??-??
+ * SGD-like optimizers now all divide the step size by the batch size so that
+   step sizes don't need to be tuned in addition to batch sizes.  If you require
+   behavior from ensmallen 2, define the `ENS_OLD_SEPARABLE_STEP_BEHAVIOR` macro
+   before including `ensmallen.hpp`
+   ([#431](https://github.com/mlpack/ensmallen/pull/431)).
 
 ### ensmallen 2.22.2: "E-Bike Excitement"
 ###### 2025-04-30

include/ensmallen_bits/cmaes/active_cmaes_impl.hpp

@@ -105,9 +105,9 @@ typename MatType::elem_type ActiveCMAES<SelectionPolicyType,
 
   // Step size control parameters.
   BaseMatType sigma(2, 1); // sigma is vector-shaped.
-  if (stepSize == 0) 
+  if (stepSize == 0)
     sigma(0) = transformationPolicy.InitialStepSize();
-  else 
+  else
     sigma(0) = stepSize;
 
   const ElemType cs = 4.0 / (iterate.n_elem + 4);

include/ensmallen_bits/cmaes/cmaes_impl.hpp

@@ -104,9 +104,9 @@ typename MatType::elem_type CMAES<SelectionPolicyType,
 
   // Step size control parameters.
   BaseMatType sigma(2, 1); // sigma is vector-shaped.
-  if (stepSize == 0) 
+  if (stepSize == 0)
     sigma(0) = transformationPolicy.InitialStepSize();
-  else 
+  else
     sigma(0) = stepSize;
 
   const double cs = (muEffective + 2) / (iterate.n_elem + muEffective + 5);

include/ensmallen_bits/eve/eve_impl.hpp

@@ -147,8 +147,14 @@ Eve::Optimize(SeparableFunctionType& function,
 
     lastObjective = objective;
 
+    // TODO: remove in ensmallen 4.0.0.
+    #if defined(ENS_OLD_SEPARABLE_STEP_BEHAVIOR)
     iterate -= stepSize / dt * (m / biasCorrection1) /
         (arma::sqrt(v / biasCorrection2) + epsilon);
+    #else
+    iterate -= (stepSize / (dt * effectiveBatchSize)) * (m / biasCorrection1) /
+        (arma::sqrt(v / biasCorrection2) + epsilon);
+    #endif
 
     terminate |= Callback::StepTaken(*this, f, iterate, callbacks...);
 

include/ensmallen_bits/lookahead/lookahead_impl.hpp

@@ -188,7 +188,7 @@ Lookahead<BaseOptimizerType, DecayPolicyType>::Optimize(
     size_t batchSize = 1;
     // Check if the optimizer implements the BatchSize() method and use the
     // parameter for the objective calculation.
-    if (traits::HasBatchSizeSignature<BaseOptimizerType>::value)
+    if constexpr (traits::HasBatchSizeSignature<BaseOptimizerType>::value)
       batchSize = baseOptimizer.BatchSize();
 
     overallObjective = 0;

include/ensmallen_bits/parallel_sgd/parallel_sgd_impl.hpp

@@ -132,7 +132,7 @@ typename MatType::elem_type>::type ParallelSGD<DecayPolicyType>::Optimize(
       return overallObjective;
     }
 
-    // Get the stepsize for this iteration
+    // Get the stepsize for this iteration.
     double stepSize = decayPolicy.StepSize(i);
 
     // Shuffle for uniform sampling of functions by each thread.
@@ -180,6 +180,8 @@ typename MatType::elem_type>::type ParallelSGD<DecayPolicyType>::Optimize(
 
             // Call out to utility function to use the right type of OpenMP
             // lock.
+            // TODO: if batch size support > 1 is added, `stepSize` will need to
+            // be updated here.
             UpdateLocation(iterate, row, i, stepSize * value);
           }
         }

include/ensmallen_bits/sgd/sgd_impl.hpp

@@ -150,8 +150,14 @@ SGD<UpdatePolicyType, DecayPolicyType>::Optimize(
         gradient, callbacks...);
 
     // Use the update policy to take a step.
+    // TODO: remove old behavior in ensmallen 4.0.0.
+    #if defined(ENS_OLD_SEPARABLE_STEP_BEHAVIOR)
     instUpdatePolicy.As<InstUpdatePolicyType>().Update(iterate, stepSize,
         gradient);
+    #else
+    instUpdatePolicy.As<InstUpdatePolicyType>().Update(iterate,
+        (stepSize / effectiveBatchSize), gradient);
+    #endif
 
     terminate |= Callback::StepTaken(*this, f, iterate, callbacks...);