Normalize step size to the batch size used

HISTORY.md

@@ -1,5 +1,10 @@
 ### ensmallen ?.??.?: "???"
 ###### ????-??-??
+ * SGD-like optimizers now all divide the step size by the batch size so that
+   step sizes don't need to be tuned in addition to batch sizes.  If you require
+   behavior from ensmallen 2, define the `ENS_OLD_SEPARABLE_STEP_BEHAVIOR` macro
+   before including `ensmallen.hpp`
+   ([#431](https://github.com/mlpack/ensmallen/pull/431)).
 
 ### ensmallen 2.22.2: "E-Bike Excitement"
 ###### 2025-04-30

include/ensmallen_bits/cmaes/active_cmaes_impl.hpp

@@ -108,9 +108,9 @@ typename MatType::elem_type ActiveCMAES<SelectionPolicyType,
 
   // Step size control parameters.
   BaseMatType sigma(2, 1); // sigma is vector-shaped.
-  if (stepSize == 0) 
+  if (stepSize == 0)
     sigma(0) = transformationPolicy.InitialStepSize();
-  else 
+  else
     sigma(0) = stepSize;
 
   const ElemType cs = 4.0 / (iterate.n_elem + 4);

include/ensmallen_bits/eve/eve_impl.hpp

@@ -147,8 +147,14 @@ Eve::Optimize(SeparableFunctionType& function,
 
     lastObjective = objective;
 
+    // TODO: remove in ensmallen 4.0.0.
+    #if defined(ENS_OLD_SEPARABLE_STEP_BEHAVIOR)
     iterate -= stepSize / dt * (m / biasCorrection1) /
         (sqrt(v / biasCorrection2) + epsilon);
+    #else
+    iterate -= (stepSize / (dt * effectiveBatchSize)) * (m / biasCorrection1) /
+        (sqrt(v / biasCorrection2) + epsilon);
+    #endif
 
     terminate |= Callback::StepTaken(*this, f, iterate, callbacks...);
 

include/ensmallen_bits/lookahead/lookahead_impl.hpp

@@ -66,6 +66,24 @@ inline Lookahead<BaseOptimizerType, DecayPolicyType>::~Lookahead()
   instDecayPolicy.Clean();
 }
 
+template<typename BaseOptimizerType>
+size_t GetBatchSize(
+    const BaseOptimizerType& baseOptimizer,
+    const typename std::enable_if_t<traits::HasBatchSizeSignature<
+        BaseOptimizerType>::value>* = 0)
+{
+  return baseOptimizer.BatchSize();
+}
+
+template<typename BaseOptimizerType>
+size_t GetBatchSize(
+    const BaseOptimizerType& baseOptimizer,
+    const typename std::enable_if_t<!traits::HasBatchSizeSignature<
+        BaseOptimizerType>::value>* = 0)
+{
+  return 1;
+}
+
 //! Optimize the function (minimize).
 template<typename BaseOptimizerType, typename DecayPolicyType>
 template<typename SeparableFunctionType,
@@ -185,11 +203,9 @@ Lookahead<BaseOptimizerType, DecayPolicyType>::Optimize(
     // Find the number of functions to use.
     const size_t numFunctions = f.NumFunctions();
 
-    size_t batchSize = 1;
     // Check if the optimizer implements the BatchSize() method and use the
     // parameter for the objective calculation.
-    if (traits::HasBatchSizeSignature<BaseOptimizerType>::value)
-      batchSize = baseOptimizer.BatchSize();
+    size_t batchSize = GetBatchSize(baseOptimizer);
 
     overallObjective = 0;
     for (size_t i = 0; i < numFunctions; i += batchSize)

include/ensmallen_bits/parallel_sgd/parallel_sgd_impl.hpp

@@ -132,7 +132,7 @@ typename MatType::elem_type>::type ParallelSGD<DecayPolicyType>::Optimize(
       return overallObjective;
     }
 
-    // Get the stepsize for this iteration
+    // Get the stepsize for this iteration.
     double stepSize = decayPolicy.StepSize(i);
 
     // Shuffle for uniform sampling of functions by each thread.
@@ -180,6 +180,8 @@ typename MatType::elem_type>::type ParallelSGD<DecayPolicyType>::Optimize(
 
             // Call out to utility function to use the right type of OpenMP
             // lock.
+            // TODO: if batch size support > 1 is added, `stepSize` will need to
+            // be updated here.
             UpdateLocation(iterate, row, i, stepSize * value);
           }
         }

include/ensmallen_bits/sgd/sgd_impl.hpp

@@ -150,8 +150,14 @@ SGD<UpdatePolicyType, DecayPolicyType>::Optimize(
         gradient, callbacks...);
 
     // Use the update policy to take a step.
+    // TODO: remove old behavior in ensmallen 4.0.0.
+    #if defined(ENS_OLD_SEPARABLE_STEP_BEHAVIOR)
     instUpdatePolicy.As<InstUpdatePolicyType>().Update(iterate, stepSize,
         gradient);
+    #else
+    instUpdatePolicy.As<InstUpdatePolicyType>().Update(iterate,
+        (stepSize / effectiveBatchSize), gradient);
+    #endif
 
     terminate |= Callback::StepTaken(*this, f, iterate, callbacks...);