cyclical_decay.hpp

Go to the documentation of this file.

#ifndef MLPACK_CORE_OPTIMIZERS_SGDR_CYCLICAL_DECAY_HPP
#define MLPACK_CORE_OPTIMIZERS_SGDR_CYCLICAL_DECAY_HPP

namespace mlpack {
namespace optimization {

class CyclicalDecay
{
 public:
  CyclicalDecay(const size_t epochRestart,
                const double multFactor,
                const double stepSize) :
      epochRestart(epochRestart),
      multFactor(multFactor),
      constStepSize(stepSize),
      nextRestart(epochRestart),
      batchRestart(0),
      epochBatches(0),
      epoch(0)
  { /* Nothing to do here */ }

  void Update(arma::mat& /* iterate */,
              double& stepSize,
              const arma::mat& /* gradient */)
  {
    // Time to adjust the step size.
    if (epoch >= epochRestart)
    {
      // n_t = n_min^i + 0.5(n_max^i - n_min^i)(1 + cos(T_cur/T_i * pi)).
      stepSize = 0.5 * constStepSize * (1 + cos((batchRestart / epochBatches)
          * M_PI));

      // Keep track of the number of batches since the last restart.
      batchRestart++;
    }

    // Time to restart.
    if (epoch > nextRestart)
    {
      batchRestart = 0;

      // Adjust the period of restarts.
      epochRestart *= multFactor;

      // Update the time for the next restart.
      nextRestart += epochRestart;
    }

    epoch++;
  }

  double StepSize() const { return constStepSize; }
  double& StepSize() { return constStepSize; }

  double EpochBatches() const { return epochBatches; }
  double& EpochBatches() { return epochBatches; }

 private:
  size_t epochRestart;

  double multFactor;

  double constStepSize;

  size_t nextRestart;

  size_t batchRestart;

  double epochBatches;

  size_t epoch;
};

} // namespace optimization
} // namespace mlpack

#endif // MLPACK_CORE_OPTIMIZERS_SGDR_CYCLICAL_DECAY_HPP