// This file is part of The New Aspell
// Copyright (C) 2001 by Kevin Atkinson under the GNU LGPL license
// version 2.0 or 2.1.  You should have received a copy of the LGPL
// license along with this library if you did not you can find
// it at http://www.gnu.org/.

#ifndef ACOMMON_TOKENIZER__HPP
#define ACOMMON_TOKENIZER__HPP

#include "char_vector.hpp"
#include "filter_char.hpp"
#include "filter_char_vector.hpp"

namespace acommon {

  class Convert;
  class Speller;
  class Config;

  class Tokenizer {

  public:
    Tokenizer();
    virtual ~Tokenizer();

    FilterChar * word_begin;
    FilterChar * word_end;
    FilterChar * end;
    
    CharVector word; // this word is in the final encoded form
    unsigned int begin_pos; // pointers back to the original word
    unsigned int end_pos;
    
    // The string passed in _must_ have a null character
    // at stop - 1. (ie stop must be one past the end)
    void reset (FilterChar * in, FilterChar * stop);
    bool at_end() const {return word_begin == word_end;}
    
    virtual bool advance() = 0; // returns false if there is nothing left

    bool is_begin(unsigned char c) const
      {return char_type_[c].begin;}
    bool is_middle(unsigned char c) const
      {return char_type_[c].middle;}
    bool is_end(unsigned char c) const
      {return char_type_[c].end;}
    bool is_word(unsigned char c) const
      {return char_type_[c].word;}

  public: // but don't use
    // The speller class is expected to fill these members in
    struct CharType {
      bool begin;
      bool middle;
      bool end;
      bool word;
      CharType() : begin(false), middle(false), end(false), word(false) {}
    };
    
    CharType char_type_[256];
    Convert * conv_;
    FilterCharVector buf_;
  };

  // returns a new tokenizer and sets it up with the given speller
  // class

  PosibErr<Tokenizer *> new_tokenizer(Speller *);

}

#endif