rmsprop_update.hpp

Go to the documentation of this file.

#ifndef MLPACK_CORE_OPTIMIZERS_RMSPROP_RMSPROP_UPDATE_HPP
#define MLPACK_CORE_OPTIMIZERS_RMSPROP_RMSPROP_UPDATE_HPP

#include <mlpack/prereqs.hpp>

namespace mlpack {
namespace optimization {

class RMSPropUpdate
{
 public:
  RMSPropUpdate(const double epsilon = 1e-8,
                const double alpha = 0.99) :
    epsilon(epsilon),
    alpha(alpha)
  {
    // Nothing to do.
  }

  void Initialize(const size_t rows, const size_t cols)
  {
    // Leaky sum of squares of parameter gradient.
    meanSquaredGradient = arma::zeros<arma::mat>(rows, cols);
  }

  void Update(arma::mat& iterate,
              const double stepSize,
              const arma::mat& gradient)
  {
    meanSquaredGradient *= alpha;
    meanSquaredGradient += (1 - alpha) * (gradient % gradient);
    iterate -= stepSize * gradient / (arma::sqrt(meanSquaredGradient) +
        epsilon);
  }

  double Epsilon() const { return epsilon; }
  double& Epsilon() { return epsilon; }

  double Alpha() const { return alpha; }
  double& Alpha() { return alpha; }

 private:
  // The epsilon value used to initialise the squared gradient parameter.
  double epsilon;

  // The smoothing parameter.
  double alpha;

  // Leaky sum of squares of parameter gradient.
  arma::mat meanSquaredGradient;
};

} // namespace optimization
} // namespace mlpack

#endif