barzilai_borwein_decay.hpp

Go to the documentation of this file.

#ifndef MLPACK_CORE_OPTIMIZERS_SVRG_BARZILIA_BORWEIN_DECAY_HPP
#define MLPACK_CORE_OPTIMIZERS_SVRG_BARZILIA_BORWEIN_DECAY_HPP

#include <mlpack/prereqs.hpp>

namespace mlpack {
namespace optimization {

class BarzilaiBorweinDecay
{
 public:
  /*
   * Construct the Barzilai-Borwein decay policy.
   *
   * @param maxStepSize The maximum step size.
   * @param eps The eps coefficient to avoid division by zero (numerical
   *    stability).
   */
  BarzilaiBorweinDecay(const double maxStepSize = DBL_MAX,
                       const double eps = 1e-7) :
      eps(eps),
      maxStepSize(maxStepSize)
  { /* Nothing to do. */}

  void Initialize(const size_t /* rows */, const size_t /* cols */)
  { /* Do nothing. */ }

  void Update(const arma::mat& iterate,
              const arma::mat& iterate0,
              const arma::mat& /* gradient */,
              const arma::mat& fullGradient,
              const size_t numBatches,
              double& stepSize)
  {
    if (!fullGradient0.is_empty())
    {
      // Step size selection based on Barzilai-Borwein (BB).
      stepSize = std::pow(arma::norm(iterate - iterate0), 2.0) /
          (arma::dot(iterate - iterate0, fullGradient - fullGradient0) + eps) /
          (double) numBatches;

      stepSize = std::min(stepSize, maxStepSize);
    }

    fullGradient0 = std::move(fullGradient);
  }

 private:
  arma::mat fullGradient0;

  double eps;

  double maxStepSize;
};

} // namespace optimization
} // namespace mlpack

#endif // MLPACK_CORE_OPTIMIZERS_SVRG_BARZILIA_BORWEIN_DECAY_HPP