gradient_clipping.hpp

Go to the documentation of this file.

#ifndef MLPACK_CORE_OPTIMIZERS_SGD_GRADIENT_CLIPPING_HPP
#define MLPACK_CORE_OPTIMIZERS_SGD_GRADIENT_CLIPPING_HPP

#include <mlpack/prereqs.hpp>

namespace mlpack {
namespace optimization {

template<typename UpdatePolicyType>
class GradientClipping
{
 public:
  GradientClipping(const double minGradient,
                   const double maxGradient,
                   UpdatePolicyType& updatePolicy) :
    minGradient(minGradient),
    maxGradient(maxGradient),
    updatePolicy(updatePolicy)
  {
    // Nothing to do here
  }

  void Initialize(const size_t rows, const size_t cols)
  {
    updatePolicy.Initialize(rows, cols);
  }

  void Update(arma::mat& iterate,
              const double stepSize,
              const arma::mat& gradient)
  {
    // First, clip the gradient.
    arma::mat clippedGradient = arma::clamp(gradient, minGradient, maxGradient);
    // And only then do the update.
    updatePolicy.Update(iterate, stepSize, clippedGradient);
  }

  UpdatePolicyType& UpdatePolicy() const { return updatePolicy; }
  UpdatePolicyType& UpdatePolicy() { return updatePolicy; }

  double MinGradient() const { return minGradient; }
  double& MinGradient() { return minGradient; }

  double MaxGradient() const { return maxGradient; }
  double& MaxGradient() { return maxGradient; }

 private:
  double minGradient;

  double maxGradient;

  UpdatePolicyType updatePolicy;
};

} // namespace optimization
} // namespace mlpack

#endif