spalera_stepsize.hpp

Go to the documentation of this file.

#ifndef MLPACK_CORE_OPTIMIZERS_SPALERA_SGD_SPALERA_STEPSIZE_HPP
#define MLPACK_CORE_OPTIMIZERS_SPALERA_SGD_SPALERA_STEPSIZE_HPP

namespace mlpack {
namespace optimization {

class SPALeRAStepsize
{
 public:
  SPALeRAStepsize(const double alpha = 0.001,
                  const double epsilon = 1e-6,
                  const double adaptRate = 3.10e-8) :
      alpha(alpha),
      epsilon(epsilon),
      adaptRate(adaptRate)
  {
    /* Nothing to do here. */
  }

  void Initialize(const size_t rows, const size_t cols, const double lambda)
  {
    learningRates = arma::ones(rows, cols);
    relaxedSums = arma::zeros(rows, cols);

    this->lambda = lambda;

    mu0 = un = mn = relaxedObjective = phCounter = eveCounter = 0;
  }

  bool Update(const double stepSize,
              const double objective,
              const size_t batchSize,
              const size_t numFunctions,
              arma::mat& iterate,
              const arma::mat& gradient)
  {
    // The ratio of mini-batch size to training set size; needed for the
    // Page-Hinkley relaxed objective computations.
    const double mbRatio = batchSize / (double) numFunctions;

    // Page-Hinkley iteration, check if we have to reset the parameter and
    // adjust the step size.
    if (phCounter > (1 / mbRatio))
    {
      relaxedObjective = (1 - mbRatio) * relaxedObjective + mbRatio * objective;
    }
    else
    {
      relaxedObjective = phCounter * relaxedObjective + objective;
      relaxedObjective /= (phCounter + 1);
    }

    // Update the mu0 parameter.
    mu0 = phCounter * mu0 + relaxedObjective;
    mu0 = mu0 / (phCounter + 1);

    // Update the un parameter.
    un += relaxedObjective - mu0;

    // Updating the mn parameter.
    if (un < mn)
      mn = un;

    // If the condition is true we reset the parameter and update parameter.
    if ((un - mn) > lambda)
    {
      // Backtracking, reset the parameter.
      iterate = previousIterate;

      // Dividing learning rates by 2 as proposed in:
      // Stochastic Gradient Descent: Going As Fast As Possible But Not Faster.
      learningRates /= 2;

      if (arma::any(arma::vectorise(learningRates) <= 1e-15))
      {
        // Stop because learning rate too low.
        return false;
      }

      // Reset evaluation and Page-Hinkley counter parameter.
      mu0 = un = mn = relaxedObjective = phCounter = eveCounter = 0;
    }
    else
    {
      const double paramMean = (alpha / (2 - alpha) *
          (1 - std::pow(1 - alpha, 2 * (eveCounter + 1)))) / iterate.n_elem;

      const double paramStd = (alpha / std::sqrt(iterate.n_elem)) /
          std::sqrt(iterate.n_elem);

      const double normGradient = std::sqrt(arma::accu(arma::pow(gradient, 2)));

      relaxedSums *= (1 - alpha);
      if (normGradient > epsilon)
        relaxedSums += gradient * (alpha / normGradient);

      learningRates %= arma::exp((arma::pow(relaxedSums, 2) - paramMean) *
          (adaptRate / paramStd));

      previousIterate = iterate;

      iterate -= stepSize * (learningRates % gradient);

      // Keep track of the the number of evaluations and Page-Hinkley steps.
      eveCounter++;
      phCounter++;
    }

    return true;
  }

  double Alpha() const { return alpha; }
  double& Alpha() { return alpha; }

  double AdaptRate() const { return adaptRate; }
  double& AdaptRate() { return adaptRate; }

 private:
  double mu0;

  double un;

  double mn;

  double relaxedObjective;

  size_t phCounter;

  size_t eveCounter;

  double alpha;

  double epsilon;

  double adaptRate;

  arma::mat learningRates;

  arma::mat relaxedSums;

  double lambda;

  arma::mat previousIterate;
};

} // namespace optimization
} // namespace mlpack

#endif // MLPACK_CORE_OPTIMIZERS_SPALERA_SGD_SPALERA_STEPSIZE_HPP