backtracking_line_search.hpp

Go to the documentation of this file.

#ifndef MLPACK_CORE_OPTIMIZERS_BIGBATCH_SGD_BACKTRACKING_LINE_SEARCH_HPP
#define MLPACK_CORE_OPTIMIZERS_BIGBATCH_SGD_BACKTRACKING_LINE_SEARCH_HPP

namespace mlpack {
namespace optimization {

class BacktrackingLineSearch
{
 public:
  BacktrackingLineSearch(const double searchParameter = 0.1) :
      searchParameter(searchParameter)
  { /* Nothing to do here. */ }

  template<typename DecomposableFunctionType>
  void Update(DecomposableFunctionType& function,
              double& stepSize,
              arma::mat& iterate,
              const arma::mat& gradient,
              const double gradientNorm,
              const double /* sampleVariance */,
              const size_t offset,
              const size_t /* batchSize */,
              const size_t backtrackingBatchSize,
              const bool reset)
  {
    if (reset)
      stepSize *= 2;

    double overallObjective = function.Evaluate(iterate, offset,
        backtrackingBatchSize);

    arma::mat iterateUpdate = iterate - (stepSize * gradient);
    double overallObjectiveUpdate = function.Evaluate(iterateUpdate,
        offset, backtrackingBatchSize);

    while (overallObjectiveUpdate >
        (overallObjective + searchParameter * stepSize * gradientNorm))
    {
      stepSize /= 2;

      iterateUpdate = iterate - (stepSize * gradient);
      overallObjectiveUpdate = function.Evaluate(iterateUpdate,
        offset, backtrackingBatchSize);
    }
  }

 private:
  double searchParameter;
};

} // namespace optimization
} // namespace mlpack

#endif // MLPACK_CORE_OPTIMIZERS_BIGBATCH_SGD_BACKTRACKING_LINE_SEARCH_HPP