Test FP16 support when available with Armadillo

CMakeLists.txt

@@ -14,9 +14,13 @@ option(USE_BANDICOOT "If available, build against Bandicoot for GPU support." ON
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/CMake")
 
-# Set required C++ standard to C++14.
-set(CMAKE_CXX_STANDARD 14)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
+# Set minimum required C++ standard to C++14.
+if (NOT CMAKE_CXX_STANDARD)
+  set(CMAKE_CXX_STANDARD 14)
+  set(CMAKE_CXX_STANDARD_REQUIRED ON)
+elseif (${CMAKE_CXX_STANDARD} LESS 14)
+  message(FATAL_ERROR "ensmallen requires C++14 or newer!")
+endif ()
 
 # Extract version from sources.
 set(ENSMALLEN_VERSION_FILE_NAME "${PROJECT_SOURCE_DIR}/include/ensmallen_bits/ens_version.hpp")

doc/optimizers.md

@@ -293,7 +293,7 @@ optimizer.Optimize(f, coordinates);
  * [AdaGrad](#adagrad)
  * [Differentiable separable functions](#differentiable-separable-functions)
 
-## Adagrad
+## AdaGrad
 
 *An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
 
@@ -1297,7 +1297,7 @@ total contribution of a gradient to all future updates.
  * `DemonAdam()`
  * `DemonAdam(`_`stepSize, batchSize`_`)`
  * `DemonAdam(`_`stepSize, batchSize, momentum, beta1, beta2, eps, maxIterations, tolerance, shuffle`_`)`
- * `DemonAdam(`_`stepSize, batchSize, momentum, beta1, beta2, eps, maxIterations, tolerance, shuffle, resetPolicy`_`)`
+ * `DemonAdam(`_`stepSize, batchSize, momentum, beta1, beta2, eps, maxIterations, tolerance, shuffle, resetPolicy, exactObjective`_`)`
 
 Note that the `DemonAdam` class is based on
 the `DemonAdamType<`_`UpdateRule`_`>` class with _`UpdateRule`_` = AdamUpdate`.
@@ -1329,12 +1329,20 @@ For convenience the following typedefs have been defined:
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
 | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective at the end of optimization.  (This could be computationally expensive!) | `false` |
 
 The attributes of the optimizer may also be modified via the member methods
 `StepSize()`, `BatchSize()`, `Momentum()`, `MomentumIterations()`, `Beta1()`,
 `Beta2()`, `Eps()`, `MaxIterations()`, `Tolerance()`, `Shuffle()`, and
 `ResetPolicy()`.
 
+***Note:*** if `exactObjective` is `false`, then `Optimize(f, coordinates)` will
+return an estimate of the objective function.  This estimate is the sum of the
+objectives obtained on the last pass of the separable functions.  The estimate
+will not include contributions from any separable functions not visited in the
+last pass (e.g., if `maxIterations` is not an integer multiple of
+`f.NumFunctions()`).
+
 #### Examples
 
 <details open>
@@ -1386,7 +1394,7 @@ optimizer:
  * `DemonSGD()`
  * `DemonSGD(`_`stepSize, batchSize`_`)`
  * `DemonSGD(`_`stepSize, batchSize, momentum, maxIterations, tolerance, shuffle`_`)`
- * `DemonSGD(`_`stepSize, batchSize, momentum, maxIterations, tolerance, shuffle, resetPolicy`_`)`
+ * `DemonSGD(`_`stepSize, batchSize, momentum, maxIterations, tolerance, shuffle, resetPolicy, exactObjective`_`)`
 
 #### Attributes
 
@@ -1399,11 +1407,19 @@ optimizer:
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
 | `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
+| `bool` | **`exactObjective`** | Calculate the exact objective at the end of optimization.  (This could be computationally expensive!) | `false` |
 
 The attributes of the optimizer may also be modified via the member methods
 `StepSize()`, `BatchSize()`, `Momentum()`, `MomentumIterations()`,
 `MaxIterations()`, `Tolerance()`, `Shuffle()`, and `ResetPolicy()`.
 
+***Note:*** if `exactObjective` is `false`, then `Optimize(f, coordinates)` will
+return an estimate of the objective function.  This estimate is the sum of the
+objectives obtained on the last pass of the separable functions.  The estimate
+will not include contributions from any separable functions not visited in the
+last pass (e.g., if `maxIterations` is not an integer multiple of
+`f.NumFunctions()`).
+
 #### Examples
 
 <details open>

include/ensmallen_bits/ada_belief/ada_belief_update.hpp

@@ -79,6 +79,8 @@ class AdaBeliefUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the SGD Optimize() method before the start
      * of the iteration update process.
@@ -89,10 +91,16 @@ class AdaBeliefUpdate
      */
     Policy(AdaBeliefUpdate& parent, const size_t rows, const size_t cols) :
         parent(parent),
+        beta1(ElemType(parent.beta1)),
+        beta2(ElemType(parent.beta2)),
+        epsilon(ElemType(parent.epsilon)),
         iteration(0)
     {
       m.zeros(rows, cols);
       s.zeros(rows, cols);
+      // Prevent underflow.
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -109,18 +117,18 @@ class AdaBeliefUpdate
       // Increment the iteration counter variable.
       ++iteration;
 
-      m *= parent.beta1;
-      m += (1 - parent.beta1) * gradient;
+      m *= beta1;
+      m += (1 - beta1) * gradient;
 
-      s *= parent.beta2;
-      s += (1 - parent.beta2) * pow(gradient - m, 2.0) + parent.epsilon;
+      s *= beta2;
+      s += (1 - beta2) * pow(gradient - m, 2) + epsilon;
 
-      const double biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
-      const double biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);
+      const ElemType biasCorrection1 = 1 - std::pow(beta1, ElemType(iteration));
+      const ElemType biasCorrection2 = 1 - std::pow(beta2, ElemType(iteration));
 
       // And update the iterate.
-      iterate -= ((m / biasCorrection1) * stepSize) / (sqrt(s /
-          biasCorrection2) + parent.epsilon);
+      iterate -= ((m / biasCorrection1) * ElemType(stepSize)) /
+          (sqrt(s / biasCorrection2) + epsilon);
     }
 
    private:
@@ -133,6 +141,11 @@ class AdaBeliefUpdate
     // The exponential moving average of squared gradient values.
     GradType s;
 
+    // Parent parameters converted to the element type of the matrix.
+    ElemType beta1;
+    ElemType beta2;
+    ElemType epsilon;
+
     // The number of iterations.
     size_t iteration;
   };

include/ensmallen_bits/ada_bound/ada_bound_update.hpp

@@ -96,6 +96,8 @@ class AdaBoundUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the SGD Optimize() method before the start
      * of the iteration update process.
@@ -105,10 +107,24 @@ class AdaBoundUpdate
      * @param cols Number of columns in the gradient matrix.
      */
     Policy(AdaBoundUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent), first(true), initialStepSize(0), iteration(0)
+        parent(parent),
+        finalLr(ElemType(parent.finalLr)),
+        gamma(ElemType(parent.gamma)),
+        epsilon(ElemType(parent.epsilon)),
+        beta1(ElemType(parent.beta1)),
+        beta2(ElemType(parent.beta2)),
+        first(true),
+        initialStepSize(0),
+        iteration(0)
     {
       m.zeros(rows, cols);
       v.zeros(rows, cols);
+
+      // Check for underflows in conversions.
+      if (gamma == ElemType(0) && parent.gamma != 0.0)
+        gamma = 10 * std::numeric_limits<ElemType>::epsilon();
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -129,30 +145,30 @@ class AdaBoundUpdate
       if (first)
       {
         first = false;
-        initialStepSize = stepSize;
+        initialStepSize = ElemType(stepSize);
       }
 
       // Increment the iteration counter variable.
       ++iteration;
 
       // Decay the first and second moment running average coefficient.
-      m *= parent.beta1;
-      m += (1 - parent.beta1) * gradient;
+      m *= beta1;
+      m += (1 - beta1) * gradient;
 
-      v *= parent.beta2;
-      v += (1 - parent.beta2) * (gradient % gradient);
+      v *= beta2;
+      v += (1 - beta2) * (gradient % gradient);
 
-      const ElemType biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
-      const ElemType biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);
+      const ElemType biasCorrection1 = 1 - std::pow(beta1, ElemType(iteration));
+      const ElemType biasCorrection2 = 1 - std::pow(beta2, ElemType(iteration));
 
-      const ElemType fl = parent.finalLr * stepSize / initialStepSize;
-      const ElemType lower = fl * (1.0 - 1.0 / (parent.gamma * iteration + 1));
-      const ElemType upper = fl * (1.0 + 1.0 / (parent.gamma * iteration));
+      const ElemType fl = finalLr * ElemType(stepSize) / initialStepSize;
+      const ElemType lower = fl * (1 - 1 / (gamma * iteration + 1));
+      const ElemType upper = fl * (1 + 1 / (gamma * iteration));
 
       // Applies bounds on actual learning rate.
-      iterate -= clamp((stepSize *
-          std::sqrt(biasCorrection2) / biasCorrection1) / (sqrt(v) +
-          parent.epsilon), lower, upper) % m;
+      iterate -= clamp((ElemType(stepSize) *
+          std::sqrt(biasCorrection2) / biasCorrection1) / (sqrt(v) + epsilon),
+          lower, upper) % m;
     }
 
    private:
@@ -165,11 +181,18 @@ class AdaBoundUpdate
     // The exponential moving average of squared gradient values.
     GradType v;
 
+    // Parameters of the parent, casted to the element type of the problem.
+    ElemType finalLr;
+    ElemType gamma;
+    ElemType epsilon;
+    ElemType beta1;
+    ElemType beta2;
+
     // Whether this is the first call of the Update method.
     bool first;
 
     // The initial (Adam) learning rate.
-    double initialStepSize;
+    ElemType initialStepSize;
 
     // The number of iterations.
     size_t iteration;

include/ensmallen_bits/ada_bound/ams_bound_update.hpp

@@ -96,6 +96,8 @@ class AMSBoundUpdate
   class Policy
   {
    public:
+    typedef typename MatType::elem_type ElemType;
+
     /**
      * This constructor is called by the SGD Optimize() method before the start
      * of the iteration update process.
@@ -105,11 +107,25 @@ class AMSBoundUpdate
      * @param cols Number of columns in the gradient matrix.
      */
     Policy(AMSBoundUpdate& parent, const size_t rows, const size_t cols) :
-        parent(parent), first(true), initialStepSize(0), iteration(0)
+        parent(parent),
+        finalLr(ElemType(parent.finalLr)),
+        gamma(ElemType(parent.gamma)),
+        epsilon(ElemType(parent.epsilon)),
+        beta1(ElemType(parent.beta1)),
+        beta2(ElemType(parent.beta2)),
+        first(true),
+        initialStepSize(0),
+        iteration(0)
     {
       m.zeros(rows, cols);
       v.zeros(rows, cols);
       vImproved.zeros(rows, cols);
+
+      // Check for underflows in conversions.
+      if (gamma == ElemType(0) && parent.gamma != 0.0)
+        gamma = 10 * std::numeric_limits<ElemType>::epsilon();
+      if (epsilon == ElemType(0) && parent.epsilon != 0.0)
+        epsilon = 10 * std::numeric_limits<ElemType>::epsilon();
     }
 
     /**
@@ -123,40 +139,36 @@ class AMSBoundUpdate
                 const double stepSize,
                 const GradType& gradient)
     {
-      // Convenience typedefs.
-      typedef typename MatType::elem_type ElemType;
-
       // Save the initial step size.
       if (first)
       {
         first = false;
-        initialStepSize = stepSize;
+        initialStepSize = ElemType(stepSize);
       }
 
       // Increment the iteration counter variable.
       ++iteration;
 
       // Decay the first and second moment running average coefficient.
-      m *= parent.beta1;
-      m += (1 - parent.beta1) * gradient;
+      m *= beta1;
+      m += (1 - beta1) * gradient;
 
-      v *= parent.beta2;
-      v += (1 - parent.beta2) * (gradient % gradient);
+      v *= beta2;
+      v += (1 - beta2) * (gradient % gradient);
 
-      const ElemType biasCorrection1 = 1.0 - std::pow(parent.beta1, iteration);
-      const ElemType biasCorrection2 = 1.0 - std::pow(parent.beta2, iteration);
+      const ElemType biasCorrection1 = 1 - std::pow(beta1, ElemType(iteration));
+      const ElemType biasCorrection2 = 1 - std::pow(beta2, ElemType(iteration));
 
-      const ElemType fl = parent.finalLr * stepSize / initialStepSize;
-      const ElemType lower = fl * (1.0 - 1.0 / (parent.gamma * iteration + 1));
-      const ElemType upper = fl * (1.0 + 1.0 / (parent.gamma * iteration));
+      const ElemType fl = finalLr * ElemType(stepSize) / initialStepSize;
+      const ElemType lower = fl * (1 - 1 / (gamma * iteration + 1));
+      const ElemType upper = fl * (1 + 1 / (gamma * iteration));
 
       // Element wise maximum of past and present squared gradients.
       vImproved = max(vImproved, v);
 
       // Applies bounds on actual learning rate.
-      iterate -= clamp((stepSize * std::sqrt(biasCorrection2) /
-          biasCorrection1) / (sqrt(vImproved) + parent.epsilon),
-          lower, upper) % m;
+      iterate -= clamp((ElemType(stepSize) * std::sqrt(biasCorrection2) /
+          biasCorrection1) / (sqrt(vImproved) + epsilon), lower, upper) % m;
     }
 
    private:
@@ -169,11 +181,18 @@ class AMSBoundUpdate
     // The exponential moving average of squared gradient values.
     GradType v;
 
+    // Parameters of the parent, casted to the element type of the problem.
+    ElemType finalLr;
+    ElemType gamma;
+    ElemType epsilon;
+    ElemType beta1;
+    ElemType beta2;
+
     // Whether this is the first call of the Update method.
     bool first;
 
     // The initial (Adam) learning rate.
-    double initialStepSize;
+    ElemType initialStepSize;
 
     // The optimal squared gradient value.
     GradType vImproved;