snapshot_ensembles.hpp

Go to the documentation of this file.

#ifndef MLPACK_CORE_OPTIMIZERS_SGDR_SNAPSHOT_ENSEMBLES_HPP
#define MLPACK_CORE_OPTIMIZERS_SGDR_SNAPSHOT_ENSEMBLES_HPP

namespace mlpack {
namespace optimization {

class SnapshotEnsembles
{
 public:
  SnapshotEnsembles(const size_t epochRestart,
                    const double multFactor,
                    const double stepSize,
                    const size_t maxIterations,
                    const size_t snapshots) :
    epochRestart(epochRestart),
    multFactor(multFactor),
    constStepSize(stepSize),
    nextRestart(epochRestart),
    batchRestart(0),
    epoch(0)
  {
    snapshotEpochs = 0;
    for (size_t i = 0, er = epochRestart, nr = nextRestart;
        i < maxIterations; ++i)
    {
      if (i > nr)
      {
        er *= multFactor;
        nr += er;
        snapshotEpochs++;
      }
    }

    snapshotEpochs = epochRestart * std::pow(multFactor,
        snapshotEpochs - snapshots + 1);
  }

  void Update(arma::mat& iterate,
              double& stepSize,
              const arma::mat& /* gradient */)
  {
    // Time to adjust the step size.
    if (epoch >= epochRestart)
    {
      // n_t = n_min^i + 0.5(n_max^i - n_min^i)(1 + cos(T_cur/T_i * pi)).
      stepSize = 0.5 * constStepSize * (1 + cos((batchRestart / epochBatches)
          * M_PI));

      // Keep track of the number of batches since the last restart.
      batchRestart++;
    }

    // Time to restart.
    if (epoch > nextRestart)
    {
      batchRestart = 0;

      // Adjust the period of restarts.
      epochRestart *= multFactor;

      // Create a new snapshot.
      if (epochRestart >= snapshotEpochs)
      {
        snapshots.push_back(iterate);
      }

      // Update the time for the next restart.
      nextRestart += epochRestart;
    }

    epoch++;
  }

  double StepSize() const { return constStepSize; }
  double& StepSize() { return constStepSize; }

  double EpochBatches() const { return epochBatches; }
  double& EpochBatches() { return epochBatches; }

  std::vector<arma::mat> Snapshots() const { return snapshots; }
  std::vector<arma::mat>& Snapshots() { return snapshots; }

 private:
  size_t epochRestart;

  double multFactor;

  double constStepSize;

  size_t nextRestart;

  size_t batchRestart;

  double epochBatches;

  size_t epoch;

  size_t snapshotEpochs;

  std::vector<arma::mat> snapshots;
};

} // namespace optimization
} // namespace mlpack

#endif // MLPACK_CORE_OPTIMIZERS_SGDR_CYCLICAL_DECAY_HPP