char_extract.hpp

Go to the documentation of this file.

#ifndef MLPACK_CORE_DATA_TOKENIZERS_CHAR_EXTRACT_HPP
#define MLPACK_CORE_DATA_TOKENIZERS_CHAR_EXTRACT_HPP

#include <mlpack/prereqs.hpp>

namespace mlpack {
namespace data {

class CharExtract
{
 public:
  using TokenType = int;

  int operator()(boost::string_view& str) const
  {
    if (str.empty())
      return EOF;

    const int retval = static_cast<unsigned char>(str[0]);

    str.remove_prefix(1);

    return retval;
  }

  static bool IsTokenEmpty(const int token)
  {
    return token == EOF;
  }
};

} // namespace data
} // namespace mlpack

#endif