smorms3_update.hpp

Go to the documentation of this file.

#ifndef MLPACK_CORE_OPTIMIZERS_SMORMS3_SMORMS3_UPDATE_HPP
#define MLPACK_CORE_OPTIMIZERS_SMORMS3_SMORMS3_UPDATE_HPP

#include <mlpack/prereqs.hpp>

namespace mlpack {
namespace optimization {

class SMORMS3Update
{
 public:
  SMORMS3Update(const double epsilon = 1e-16) : epsilon(epsilon)
  { /* Do nothing. */ }

  void Initialize(const size_t rows, const size_t cols)
  {
    // Initialise the parameters mem, g and g2.
    mem = arma::ones<arma::mat>(rows, cols);
    g = arma::zeros<arma::mat>(rows, cols);
    g2 = arma::zeros<arma::mat>(rows, cols);
  }

  void Update(arma::mat& iterate,
              const double stepSize,
              const arma::mat& gradient)
  {
    // Update the iterate.
    arma::mat r = 1 / (mem + 1);

    g = (1 - r) % g;
    g += r % gradient;

    g2 = (1 - r) % g2;
    g2 += r % (gradient % gradient);

    arma::mat x = (g % g) / (g2 + epsilon);

    x.transform( [stepSize](double &v) { return std::min(v, stepSize); } );

    iterate -= gradient % x / (arma::sqrt(g2) + epsilon);

    mem %= (1 - x);
    mem += 1;
  }

  double Epsilon() const { return epsilon; }
  double& Epsilon() { return epsilon; }

 private:
  double epsilon;

  // The parameters mem, g and g2.
  arma::mat mem, g, g2;
};

} // namespace optimization
} // namespace mlpack

#endif