continuous_mountain_car.hpp

Go to the documentation of this file.

#ifndef MLPACK_METHODS_RL_ENVIRONMENT_CONTINUOUS_MOUNTAIN_CAR_HPP
#define MLPACK_METHODS_RL_ENVIRONMENT_CONTINUOUS_MOUNTAIN_CAR_HPP

#include <mlpack/prereqs.hpp>

namespace mlpack {
namespace rl {

class ContinuousMountainCar
{
 public:
  class State
  {
   public:
    State() : data(dimension, arma::fill::zeros)
    { /* Nothing to do here. */ }

    State(const arma::colvec& data): data(data)
    { /* Nothing to do here. */ }

    arma::colvec& Data() { return data; }

    double Velocity() const { return data[0]; }
    double& Velocity() { return data[0]; }

    double Position() const { return data[1]; }
    double& Position() { return data[1]; }

    const arma::colvec& Encode() const { return data; }

    static constexpr size_t dimension = 2;

   private:
    arma::colvec data;
  };

  struct Action
  {
    double action[1];
    // Storing degree of freedom
    const int size = 1;
  };

  ContinuousMountainCar(const double positionMin = -1.2,
                        const double positionMax = 0.6,
                        const double positionGoal = 0.45,
                        const double velocityMin = -0.07,
                        const double velocityMax = 0.07,
                        const double power = 0.0015) :
      positionMin(positionMin),
      positionMax(positionMax),
      positionGoal(positionGoal),
      velocityMin(velocityMin),
      velocityMax(velocityMax),
      power(power)
  { /* Nothing to do here */ }

  double Sample(const State& state,
                const Action& action,
                State& nextState) const
  {
    // Calculate acceleration.
    double force = std::min(std::max(action.action[0], -1.0), 1.0);

    // Update states.
    nextState.Velocity() = state.Velocity() + force * power - 0.0025 *
        std::cos(3 * state.Position());
    nextState.Velocity() = std::min(
        std::max(nextState.Velocity(), velocityMin), velocityMax);
    nextState.Position() = state.Position() + nextState.Velocity();
    nextState.Position() = std::min(
        std::max(nextState.Position(), positionMin), positionMax);
    if (nextState.Position() == positionMin && nextState.Velocity() < 0)
      nextState.Velocity() = 0.0;

    // Calculate reward
    double reward = 0.0;
    // If it is a terminal state, add a reward of 100.0
    if (IsTerminal(nextState))
      reward = 100.0;
    reward -= std::pow(action.action[0], 2) * 0.1;
    return reward;
  }

  double Sample(const State& state, const Action& action) const
  {
    State nextState;
    return Sample(state, action, nextState);
  }

  State InitialSample() const
  {
    State state;
    state.Velocity() = 0.0;
    state.Position() = math::Random(-0.6, -0.4);
    return state;
  }

  bool IsTerminal(const State& state) const
  {
    return bool(state.Position() >= positionGoal);
  }

 private:
  double positionMin;

  double positionMax;

  double positionGoal;

  double velocityMin;

  double velocityMax;

  double power;
};

} // namespace rl
} // namespace mlpack

#endif