svrg.hpp

Go to the documentation of this file.

#ifndef MLPACK_CORE_OPTIMIZERS_SVRG_SVRG_HPP
#define MLPACK_CORE_OPTIMIZERS_SVRG_SVRG_HPP

#include <mlpack/prereqs.hpp>
#include <mlpack/core/optimizers/sgd/decay_policies/no_decay.hpp>

#include "svrg_update.hpp"
#include "barzilai_borwein_decay.hpp"

namespace mlpack {
namespace optimization {

template<typename UpdatePolicyType = SVRGUpdate,
         typename DecayPolicyType = NoDecay>
class SVRGType
{
 public:
  SVRGType(const double stepSize = 0.01,
           const size_t batchSize = 32,
           const size_t maxIterations = 1000,
           const size_t innerIterations = 0,
           const double tolerance = 1e-5,
           const bool shuffle = true,
           const UpdatePolicyType& updatePolicy = UpdatePolicyType(),
           const DecayPolicyType& decayPolicy = DecayPolicyType(),
           const bool resetPolicy = true);

  template<typename DecomposableFunctionType>
  double Optimize(DecomposableFunctionType& function, arma::mat& iterate);

  double StepSize() const { return stepSize; }
  double& StepSize() { return stepSize; }

  size_t BatchSize() const { return batchSize; }
  size_t& BatchSize() { return batchSize; }

  size_t MaxIterations() const { return maxIterations; }
  size_t& MaxIterations() { return maxIterations; }

  size_t InnerIterations() const { return innerIterations; }
  size_t& InnerIterations() { return innerIterations; }

  double Tolerance() const { return tolerance; }
  double& Tolerance() { return tolerance; }

  bool Shuffle() const { return shuffle; }
  bool& Shuffle() { return shuffle; }

  bool ResetPolicy() const { return resetPolicy; }
  bool& ResetPolicy() { return resetPolicy; }

  const UpdatePolicyType& UpdatePolicy() const { return updatePolicy; }
  UpdatePolicyType& UpdatePolicy() { return updatePolicy; }

  const DecayPolicyType& DecayPolicy() const { return decayPolicy; }
  DecayPolicyType& DecayPolicy() { return decayPolicy; }

 private:
  double stepSize;

  size_t batchSize;

  size_t maxIterations;

  size_t innerIterations;

  double tolerance;

  bool shuffle;

  UpdatePolicyType updatePolicy;

  DecayPolicyType decayPolicy;

  bool resetPolicy;
};

// Convenience typedefs.

using SVRG = SVRGType<SVRGUpdate, NoDecay>;

using SVRG_BB = SVRGType<SVRGUpdate, BarzilaiBorweinDecay>;

} // namespace optimization
} // namespace mlpack

// Include implementation.
#include "svrg_impl.hpp"

#endif