adaptive_stepsize.hpp

Go to the documentation of this file.

#ifndef MLPACK_CORE_OPTIMIZERS_BIGBATCH_SGD_ADAPTIVE_STEPSIZE_HPP
#define MLPACK_CORE_OPTIMIZERS_BIGBATCH_SGD_ADAPTIVE_STEPSIZE_HPP

namespace mlpack {
namespace optimization {

class AdaptiveStepsize
{
 public:
  AdaptiveStepsize(const double backtrackStepSize = 0.1,
                   const double searchParameter = 0.1) :
      backtrackStepSize(backtrackStepSize),
      searchParameter(searchParameter)
  { /* Nothing to do here. */ }

  template<typename DecomposableFunctionType>
  void Update(DecomposableFunctionType& function,
              double& stepSize,
              arma::mat& iterate,
              const arma::mat& gradient,
              const double gradientNorm,
              const double sampleVariance,
              const size_t offset,
              const size_t batchSize,
              const size_t backtrackingBatchSize,
              const bool /* reset */)
  {
    Backtracking(function, stepSize, iterate, gradient, gradientNorm, offset,
        backtrackingBatchSize);

    // Update the iterate.
    iterate -= stepSize * gradient;

    double stepSizeDecay = 0;
    if (batchSize < function.NumFunctions())
    {
      stepSizeDecay = (1 - (1 / ((double) batchSize - 1) * sampleVariance) /
          (batchSize * gradientNorm)) / batchSize;
    }
    else
    {
      stepSizeDecay = 1 / function.NumFunctions();
    }

    // Stepsize smoothing.
    stepSize *= (1 - ((double) batchSize / function.NumFunctions()));
    stepSize += stepSizeDecay * ((double) batchSize / function.NumFunctions());

    Backtracking(function, stepSize, iterate, gradient, gradientNorm, offset,
        backtrackingBatchSize);
  }

  double BacktrackStepSize() const { return backtrackStepSize; }
  double& BacktrackStepSize() { return backtrackStepSize; }

  double SearchParameter() const { return searchParameter; }
  double& SearchParameter() { return searchParameter; }

 private:
  template<typename DecomposableFunctionType>
  void Backtracking(DecomposableFunctionType& function,
                    double& stepSize,
                    const arma::mat& iterate,
                    const arma::mat& gradient,
                    const double gradientNorm,
                    const size_t offset,
                    const size_t backtrackingBatchSize)
  {
    double overallObjective = function.Evaluate(iterate, offset,
        backtrackingBatchSize);

    arma::mat iterateUpdate = iterate - (stepSize * gradient);
    double overallObjectiveUpdate = function.Evaluate(iterateUpdate, offset,
        backtrackingBatchSize);

    while (overallObjectiveUpdate >
        (overallObjective + searchParameter * stepSize * gradientNorm))
    {
      stepSize *= backtrackStepSize;

      iterateUpdate = iterate - (stepSize * gradient);
      overallObjectiveUpdate = function.Evaluate(iterateUpdate, offset,
          backtrackingBatchSize);
    }
  }

  double backtrackStepSize;

  double searchParameter;
};

} // namespace optimization
} // namespace mlpack

#endif // MLPACK_CORE_OPTIMIZERS_BIGBATCH_SGD_ADAPTIVE_STEPSIZE_HPP