pendulum.hpp

Go to the documentation of this file.

#ifndef MLPACK_METHODS_RL_ENVIRONMENT_PENDULUM_HPP
#define MLPACK_METHODS_RL_ENVIRONMENT_PENDULUM_HPP

#include <mlpack/prereqs.hpp>

namespace mlpack {
namespace rl {

class Pendulum
{
 public:
  class State
  {
   public:
    State() : data(dimension, arma::fill::zeros)
    { /* Nothing to do here. */ }

    State(const arma::colvec& data): data(data)
    { /* Nothing to do here. */ }

    arma::colvec& Data() { return data; }

    double Theta() const { return data[0]; }
    double& Theta() { return data[0]; }

    double AngularVelocity() const { return data[1]; }
    double& AngularVelocity() { return data[1]; }

    const arma::colvec& Encode() const { return data; }

    static constexpr size_t dimension = 2;

   private:
    arma::colvec data;
  };

  struct Action
  {
    double action[1];
    // Storing degree of freedom
    const int size = 1;
  };

  Pendulum(const double maxAngularVelocity = 8,
           const double maxTorque = 2.0,
           const double dt = 0.05,
           const double angleThreshold = M_PI / 12,
           const double doneReward = 0.0,
           const size_t maxSteps = 0) :
      maxAngularVelocity(maxAngularVelocity),
      maxTorque(maxTorque),
      dt(dt),
      angleThreshold(angleThreshold),
      doneReward(doneReward),
      maxSteps(maxSteps),
      stepsPerformed(0)
  { /* Nothing to do here */ }

  double Sample(const State& state,
                const Action& action,
                State& nextState)
  {
    // Update the number of steps performed.
    stepsPerformed++;

    // Get current state.
    double theta = state.Theta();
    double angularVelocity = state.AngularVelocity();

    // Define constants which specify our pendulum.
    const double gravity = 10.0;
    const double mass = 1.0;
    const double length = 1.0;

    // Get action and clip the values between max and min limits.
    double torque = std::min(
        std::max(action.action[0], -maxTorque), maxTorque);

    // Calculate costs of taking this action in the current state.
    double costs = std::pow(AngleNormalize(theta), 2) + 0.1 *
        std::pow(angularVelocity, 2) + 0.001 * std::pow(torque, 2);

    // Calculate new state values and assign to the next state.
    double newAngularVelocity = angularVelocity + (-3.0 * gravity / (2 *
        length) * std::sin(theta + M_PI) + 3.0 / std::pow(mass * length, 2) *
        torque) * dt;
    nextState.AngularVelocity() = std::min(std::max(newAngularVelocity,
        -maxAngularVelocity), maxAngularVelocity);
    nextState.Theta() = theta + newAngularVelocity * dt;

    // Check if the episode has terminated
    bool done = IsTerminal(nextState);

    // Do not reward the agent if time ran out.
    if (done && maxSteps != 0 && stepsPerformed >= maxSteps)
      return 0;
    else if (done)
      return doneReward;

    // Return the reward of taking the action in current state.
    // The reward is simply the negative of cost incurred for the action.
    return -costs;
  }

  double Sample(const State& state, const Action& action)
  {
    State nextState;
    return Sample(state, action, nextState);
  }

  State InitialSample()
  {
    State state;
    state.Theta() = math::Random(-M_PI + angleThreshold, M_PI - angleThreshold);
    state.AngularVelocity() = math::Random(-1.0, 1.0);
    stepsPerformed = 0;
    return state;
  }

  double AngleNormalize(double theta) const
  {
    // Scale angle within [-pi, pi).
    return double(fmod(theta + M_PI, 2 * M_PI) - M_PI);
  }

  bool IsTerminal(const State& state) const
  {
    if (maxSteps != 0 && stepsPerformed >= maxSteps)
    {
      Log::Info << "Episode terminated due to the maximum number of steps"
          "being taken.";
      return true;
    }
    else if (state.Theta() > M_PI - angleThreshold ||
        state.Theta() < -M_PI + angleThreshold)
    {
      Log::Info << "Episode terminated due to agent succeeding.";
      return true;
    }
    return false;
  }

  size_t StepsPerformed() const { return stepsPerformed; }

  size_t MaxSteps() const { return maxSteps; }
  size_t& MaxSteps() { return maxSteps; }

 private:
  double maxAngularVelocity;

  double maxTorque;

  double dt;

  double angleThreshold;

  double doneReward;

  size_t maxSteps;

  size_t stepsPerformed;
};

} // namespace rl
} // namespace mlpack

#endif