lexer.h in rbs-1.7.0.beta.3

- old
+ new

@@ -27,39 +27,39 @@
   pBANG,            /* ! */
   pQUESTION,        /* ? */
   pLT,              /* < */
   pEQ,              /* = */
 
+  kALIAS,           /* alias */
+  kATTRACCESSOR,    /* attr_accessor */
+  kATTRREADER,      /* attr_reader */
+  kATTRWRITER,      /* attr_writer */
   kBOOL,            /* bool */
   kBOT,             /* bot */
   kCLASS,           /* class */
+  kDEF,             /* def */
+  kEND,             /* end */
+  kEXTEND,          /* extend */
   kFALSE,           /* false */
+  kIN,              /* in */
+  kINCLUDE,         /* include */
   kINSTANCE,        /* instance */
   kINTERFACE,       /* interface */
+  kMODULE,          /* module */
   kNIL,             /* nil */
+  kOUT,             /* out */
+  kPREPEND,         /* prepend */
+  kPRIVATE,         /* private */
+  kPUBLIC,          /* public */
   kSELF,            /* self */
   kSINGLETON,       /* singleton */
   kTOP,             /* top */
   kTRUE,            /* true */
-  kVOID,            /* void */
   kTYPE,            /* type */
   kUNCHECKED,       /* unchecked */
-  kIN,              /* in */
-  kOUT,             /* out */
-  kEND,             /* end */
-  kDEF,             /* def */
-  kINCLUDE,         /* include */
-  kEXTEND,          /* extend */
-  kPREPEND,         /* prepend */
-  kALIAS,           /* alias */
-  kMODULE,          /* module */
-  kATTRREADER,      /* attr_reader */
-  kATTRWRITER,      /* attr_writer */
-  kATTRACCESSOR,    /* attr_accessor */
-  kPUBLIC,          /* public */
-  kPRIVATE,         /* private */
   kUNTYPED,         /* untyped */
+  kVOID,            /* void */
 
   tLIDENT,          /* Identifiers starting with lower case */
   tUIDENT,          /* Identifiers starting with upper case */
   tULIDENT,         /* Identifiers starting with `_` followed by upper case */
   tULLIDENT,        /* Identifiers starting with `_` followed by lower case */
@@ -119,27 +119,43 @@
 typedef struct {
   VALUE string;
   position current;               /* The current position */
   position start;                 /* The start position of the current token */
   bool first_token_of_line;       /* This flag is used for tLINECOMMENT */
+  unsigned int last_char;         /* Last peeked character */
 } lexstate;
 
 extern token NullToken;
 extern position NullPosition;
 extern range NULL_RANGE;
 
-token rbsparser_next_token(lexstate *state);
-
 char *peek_token(lexstate *state, token tok);
 int token_chars(token tok);
 int token_bytes(token tok);
 
 #define null_position_p(pos) (pos.byte_pos == -1)
 #define null_range_p(range) (range.start.byte_pos == -1)
 #define nonnull_pos_or(pos1, pos2) (null_position_p(pos1) ? pos2 : pos1)
 #define RANGE_BYTES(range) (range.end.byte_pos - range.start.byte_pos)
 
 const char *token_type_str(enum TokenType type);
+
+/**
+ * Read next character.
+ * */
+unsigned int peek(lexstate *state);
+
+/**
+ * Skip one character.
+ * */
+void skip(lexstate *state);
+
+/**
+ * Return new token with given type.
+ * */
+token next_token(lexstate *state, enum TokenType type);
+
+token rbsparser_next_token(lexstate *state);
 
 void print_token(token tok);
 
 #endif