Apply suggestions from code review

Co-authored-by: Ryan Curtin <ryan@ratml.org>

Author: ranjodhsingh1729

doc/optimizers.md

@@ -2470,11 +2470,11 @@ for all Lagrange multipliers and 10 is used as the initial penalty parameter.
 
 *An optimizer for [differentiable functions](#differentiable-functions).*
 
-A DeltaBarDelta variant that incorporates the following modifications:
- - In the original DeltaBarDelta, the momentum term (delta_bar) is used
+A [DeltaBarDelta](#deltabardelta) variant that incorporates the following modifications:
+ - In the original DeltaBarDelta, the momentum term (`delta_bar`) is used
    solely for sign comparison with the current gradient and does not
    participate in the parameter update. In this modified variant, the
-   momentum term (velocity) is directly used to update the parameters.
+   momentum term (`velocity`) is directly used to update the parameters.
  - Instead of adjusting the step size directly, each parameter maintains
    a gain value initialized to 1.0. Updates apply additive increases or
    multiplicative decreases to this gain. The effective step size for a
@@ -2503,7 +2503,7 @@ Note: This variant originates from optimization of the t-SNE cost function.
 | `bool` | **`resetPolicy`** | If true, parameters are reset before every `Optimize()` call. | `true` |
 
 Attributes of the optimizer may also be modified via the member methods
-`StepSize()`, `MaxIterations()`, `Tolerance()`, `Kappa()`, `Phi()`, `Momentum()`, `MinGain`, and `ResetPolicy()`.
+`StepSize()`, `MaxIterations()`, `Tolerance()`, `Kappa()`, `Phi()`, `Momentum()`, `MinGain()`, and `ResetPolicy()`.
 
 #### Examples:
 

include/ensmallen_bits/delta_bar_delta/update_policies/delta_bar_delta_update.hpp

@@ -134,7 +134,7 @@ class DeltaBarDeltaUpdate
     {
       deltaBar.zeros(rows, cols);
       epsilon.set_size(rows, cols);
-      epsilon.fill(parent.InitialStepSize());
+      epsilon.fill(ElemType(parent.InitialStepSize()));
     }
 
     /**
@@ -145,15 +145,14 @@ class DeltaBarDeltaUpdate
      * @param delta The gradient matrix.
      */
     void Update(MatType& iterate,
-                const double stepSize,
+                const double /* stepSize */,
                 const GradType& delta)
     {
       const MatType signMatrix = sign(delta % deltaBar);
 
-      epsilon += (signMatrix == +1) * kappa -
-          (signMatrix == -1) * phi % epsilon;
-      epsilon.clamp(minStepSize,
-          arma::Datum<typename MatType::elem_type>::inf);
+      epsilon += conv_to<MatType>((signMatrix == +1) * kappa -
+          (signMatrix == -1) * phi % epsilon);
+      epsilon.clamp(minStepSize, arma::Datum<ElemType>::inf);
 
       deltaBar = theta * deltaBar + (1 - theta) * delta;
       iterate -= epsilon % delta;

include/ensmallen_bits/delta_bar_delta/update_policies/momentum_delta_bar_delta_update.hpp

@@ -133,11 +133,12 @@ class MomentumDeltaBarDeltaUpdate
                 const double stepSize,
                 const GradType& gradient)
     {
-      gains += (sign(gradient) != sign(velocity)) * kappa -
-          (sign(gradient) == sign(velocity)) * (1 - phi) % gains;
-      gains.clamp(minGain, arma::Datum<typename MatType::elem_type>::inf);
+      gains += conv_to<MatType>::from(
+          (sign(gradient) != sign(velocity)) * kappa -
+          (sign(gradient) == sign(velocity)) * (1 - phi) % gains);
+      gains.clamp(minGain, arma::Datum<ElemType>::inf);
 
-      velocity = momentum * velocity - (stepSize * gains) % gradient;
+      velocity = momentum * velocity - (ElemType(stepSize) * gains) % gradient;
       iterate += velocity;
     }
 

tests/delta_bar_delta_test.cpp

@@ -29,9 +29,9 @@ TEMPLATE_TEST_CASE("DeltaBarDelta_GDTestFunction", "[DeltaBarDelta]",
 }
 
 TEMPLATE_TEST_CASE("DeltaBarDelta_RosenbrockFunction", "[DeltaBarDelta]",
-    ENS_ALL_TEST_TYPES)
+    ENS_ALL_CPU_TEST_TYPES)
 {
-  DeltaBarDelta s(0.001, 0, Tolerances<TestType>::Obj / 100,
+  DeltaBarDelta s(0.001, 100000, Tolerances<TestType>::Obj / 100,
       0.0001, 0.2, 0.5);
   FunctionTest<RosenbrockFunction, TestType>(s,
       10 * Tolerances<TestType>::LargeObj,
@@ -41,7 +41,7 @@ TEMPLATE_TEST_CASE("DeltaBarDelta_RosenbrockFunction", "[DeltaBarDelta]",
 TEMPLATE_TEST_CASE("DeltaBarDelta_LogisticRegressionFunction",
     "[DeltaBarDelta]", ENS_ALL_TEST_TYPES)
 {
-  DeltaBarDelta s(0.0008, 20, Tolerances<TestType>::Obj,
+  DeltaBarDelta s(0.0001, 500, Tolerances<TestType>::Obj,
       0.00008, 0.2, 0.5);
   LogisticRegressionFunctionTest<TestType>(s);
 }

tests/momentum_delta_bar_delta_test.cpp

@@ -22,17 +22,17 @@ using namespace ens::test;
 TEMPLATE_TEST_CASE("MomentumDeltaBarDelta_GDTestFunction",
     "[MomentumDeltaBarDelta]", ENS_ALL_TEST_TYPES)
 {
-  MomentumDeltaBarDelta s(0.9, 50, 1e-9, 0.2, 0.8, 0.5);
+  MomentumDeltaBarDelta s(0.1, 1000, 1e-9, 0.2, 0.8, 0.5);
   FunctionTest<GDTestFunction, TestType>(s,
       Tolerances<TestType>::LargeObj,
       Tolerances<TestType>::LargeCoord);
 }
 
 TEMPLATE_TEST_CASE("MomentumDeltaBarDelta_RosenbrockFunction",
-    "[MomentumDeltaBarDelta]", ENS_ALL_TEST_TYPES)
+    "[MomentumDeltaBarDelta]", ENS_ALL_CPU_TEST_TYPES)
 {
-  MomentumDeltaBarDelta s(0.001, 0, Tolerances<TestType>::Obj / 100,
-    0.2, 0.8, 0.5);
+  MomentumDeltaBarDelta s(0.001, 100000, Tolerances<TestType>::Obj / 100, 0.2,
+      0.8, 0.5);
   FunctionTest<RosenbrockFunction, TestType>(s,
       10 * Tolerances<TestType>::LargeObj,
       10 * Tolerances<TestType>::LargeCoord);
@@ -41,7 +41,7 @@ TEMPLATE_TEST_CASE("MomentumDeltaBarDelta_RosenbrockFunction",
 TEMPLATE_TEST_CASE("MomentumDeltaBarDelta_LogisticRegressionFunction",
     "[MomentumDeltaBarDelta]", ENS_ALL_TEST_TYPES)
 {
-  MomentumDeltaBarDelta s(0.0004, 20, Tolerances<TestType>::Obj,
+  MomentumDeltaBarDelta s(0.00005, 2000, Tolerances<TestType>::Obj,
       0.2, 0.8, 0.5);
   LogisticRegressionFunctionTest<TestType>(s);
 }