pendulum.hpp

Go to the documentation of this file.

#ifndef MLPACK_METHODS_RL_ENVIRONMENT_PENDULUM_HPP
#define MLPACK_METHODS_RL_ENVIRONMENT_PENDULUM_HPP

#include <mlpack/prereqs.hpp>

namespace mlpack {
namespace rl {

class Pendulum
{
 public:
  class State
  {
   public:
    State() : data(dimension, arma::fill::zeros)
    { /* Nothing to do here. */ }

    State(const arma::colvec& data): data(data)
    { /* Nothing to do here. */ }

    arma::colvec& Data() { return data; }

    double Theta() const { return data[0]; }
    double& Theta() { return data[0]; }

    double AngularVelocity() const { return data[1]; }
    double& AngularVelocity() { return data[1]; }

    const arma::colvec& Encode() const { return data; }

    static constexpr size_t dimension = 2;

   private:
    arma::colvec data;
  };

  struct Action
  {
    double action[1];
    // Storing degree of freedom
    const int size = 1;
  };

  Pendulum(const double maxAngularVelocity = 8,
           const double maxTorque = 2.0,
           const double dt = 0.05) :
      maxAngularVelocity(maxAngularVelocity),
      maxTorque(maxTorque),
      dt(dt)
  { /* Nothing to do here */ }

  double Sample(const State& state,
                const Action& action,
                State& nextState) const
  {
    // Get current state.
    double theta = state.Theta();
    double angularVelocity = state.AngularVelocity();

    // Define constants which specify our pendulum.
    const double gravity = 10.0;
    const double mass = 1.0;
    const double length = 1.0;

    // Get action and clip the values between max and min limits.
    double torque = std::min(
        std::max(action.action[0], -maxTorque), maxTorque);

    // Calculate costs of taking this action in the current state.
    double costs = std::pow(AngleNormalize(theta), 2) + 0.1 *
        std::pow(angularVelocity, 2) + 0.001 * std::pow(torque, 2);

    // Calculate new state values and assign to the next state.
    double newAngularVelocity = angularVelocity + (-3.0 * gravity / (2 *
        length) * std::sin(theta + M_PI) + 3.0 / std::pow(mass * length, 2) *
        torque) * dt;
    nextState.AngularVelocity() = std::min(std::max(newAngularVelocity,
        -maxAngularVelocity), maxAngularVelocity);
    nextState.Theta() = theta + newAngularVelocity * dt;

    // Return the reward of taking the action in current state.
    // The reward is simply the negative of cost incurred for the action.
    return -costs;
  }

  double Sample(const State& state, const Action& action) const
  {
    State nextState;
    return Sample(state, action, nextState);
  }

  State InitialSample() const
  {
    State state;
    state.Theta() = math::Random(-M_PI, M_PI);
    state.AngularVelocity() = math::Random(-1.0, 1.0);
    return state;
  }

  double AngleNormalize(double theta) const
  {
    // Scale angle within [-pi, pi).
    return double(fmod(theta + M_PI, 2 * M_PI) - M_PI);
  }

 private:
  double maxAngularVelocity;

  double maxTorque;

  double dt;
};

} // namespace rl
} // namespace mlpack

#endif