#ifndef TOKEN_HH
#define TOKEN_HH
#include "landmark.hh"
#include "operator.hh"
#include <stdio.h>
class OperatorGroup;
class Logic;
class Writer;

class Token {
  
  Operator _kind;
  short _cat;
  PermString _text;
  
  Landmark _landmark;
  unsigned _landmark_length;
  
  union {
    OperatorGroup *oper;
    PermString::Capsule string_capsule;
    Logic *expr;
  } _v;
  
  enum Categories {
    catOperator = 1,
    catString,
    catExpr,
  };
  
 public:
  
  Token()				: _kind(0), _cat(0) { }
  Token(const Token &, const Landmark &, unsigned);
  Token(Operator, PermString, const Landmark &, unsigned);
  Token(Logic *, const Landmark &, unsigned);
  Token(Operator, PermString);
  Token(Operator, PermString, OperatorGroup *);
  
  operator bool() const			{ return _kind != 0; }
  operator const Landmark &() const	{ return _landmark; }
  const Landmark &landmark() const	{ return _landmark; }
  Landmark end_landmark() const		{ return _landmark+_landmark_length; }
  
  Operator kind() const			{ return _kind; }
  bool is(int k) const			{ return _kind == k; }
  bool is_operator() const		{ return _cat == catOperator; }
  
  PermString text() const		{ return _text; }
  operator PermString() const		{ return _text; }
  const char *cc() const		{ return _text.cc(); }
  
  bool starts_expr(bool no_infix) const;
  
  PermString vstring() const;
  Operator voperator() const		{ return Operator(_kind); }
  OperatorGroup *vopgroup() const;
  Logic *vexpr() const			{ return _v.expr; }
  
  void print() const;
  
};


class TokenBuffer {
  
  void (*_free_buffer)(void *);
  Landmark _landmark;
  
  static void delete_buffer(void *);
  
 public:
  
  const unsigned char *buffer;
  const unsigned char *end_buffer;
  
  const unsigned char *pos;
  const unsigned char *end;

  TokenBuffer();
  ~TokenBuffer();
  
  bool empty() const			{ return pos >= end; }
  unsigned length() const		{ return end - buffer; }
  unsigned offset() const		{ return pos - buffer; }
  
  bool null() const			{ return buffer == 0; }
  unsigned buffer_size() const		{ return end_buffer - buffer; }
  operator Landmark() const		{ return _landmark + offset(); }
  Landmark landmark() const		{ return _landmark + offset(); }
  
  void reset(unsigned, const Landmark &);
  void reset(const char *, unsigned, const Landmark &);
  
  void release();
  void make_empty(unsigned);

  friend class Tokenizer;
  
};


class TokenBufferFiller {

 public:
  
  virtual ~TokenBufferFiller()				{ }
  
  virtual bool analyze_lines() const			{ return false; }
  virtual void fill_token_buffer(TokenBuffer *) = 0;
  
};

class FileTokenBufferFiller: public TokenBufferFiller {
  
  FILE *_f;
  Landmark _landmark;
  
 public:
  
  FileTokenBufferFiller(const char *, FILE *);
  
  bool analyze_lines() const				{ return true; }
  void fill_token_buffer(TokenBuffer *);
  
};

class StringTokenBufferFiller: public TokenBufferFiller {

  const char *_data;
  Landmark _landmark;
  bool _done;
  
 public:
  
  StringTokenBufferFiller(const char *landmark_name, const char *data);
  
  bool analyze_lines() const				{ return true; }
  void fill_token_buffer(TokenBuffer *);
  
};

class Tokenizer {
  
  static int *char_class;
  static int char_class_storage[257];
  
  enum {
    Whitespace = 1,
    Literalstart = 2,
    Wordstart = 4,
    Word = 8,
    Numberstart = 16,
    Number = 32,
    Punct = 64,
  };
  
  static bool tis_whitespace(int);
  static bool tis_word_start(int);
  static bool tis_word(int);
  static bool tis_number_start(int);
  static bool tis_number(int);
  static bool tis_literal_start(int);
  static bool tis_punct(int);
  
  static void static_initialize();
  static void add_operator(Operator, PermString, int, int, Operator);
  
  //
  
  TokenBuffer _real_tb;
  TokenBuffer _unread_tb;
  TokenBuffer *_tb;
  TokenBufferFiller *_tb_filler;
  
  bool _analyze_lines;
  
  static const int UNREAD_TB_SIZE = 30;
  
  char *_s;
  int _slen;
  int _scap;
  
  unsigned _column;
  unsigned _line_non_ws;
  
  void increase_s();
  void append(int);
  void append0(int);
  void clear();

  void analyze_lines(TokenBuffer *);
  void change_future_lines(PermString, unsigned);
  int fill_tb();
  int read();
  void unread(int);
  unsigned current_cp() const;
  
  Operator read_word(int);
  Operator read_number(int);
  Operator read_punct(int);
  Operator read_literal(int);
  
  bool handle_hash();
  void check_s_for_hash();
  Token handle_number(const Landmark &, unsigned);
  
 public:
  
  Tokenizer(TokenBufferFiller *);
  virtual ~Tokenizer();
  
  Landmark landmark() const		{ return _tb->landmark(); }
  operator Landmark() const		{ return landmark(); }
  
  Token get_token();
  
};


inline PermString
Token::vstring() const
{
  assert(_cat == catString);
  return PermString::decapsule(_v.string_capsule);
}

inline OperatorGroup *
Token::vopgroup() const
{
  assert(_cat == catOperator);
  return _v.oper;
}

#endif