require "e2mmap"
require "irb/slex"

module YARD
  module Parser::Ruby::Legacy
    module RubyToken
      EXPR_BEG   = :EXPR_BEG
      EXPR_MID   = :EXPR_MID
      EXPR_END   = :EXPR_END
      EXPR_ARG   = :EXPR_ARG
      EXPR_FNAME = :EXPR_FNAME
      EXPR_DOT   = :EXPR_DOT
      EXPR_CLASS = :EXPR_CLASS
      
      class Token
        NO_TEXT = "??".freeze

        def initialize(line_no, char_no)
          @line_no = line_no
          @char_no = char_no
          @text    = NO_TEXT
        end

        # Because we're used in contexts that expect to return a token,
        # we set the text string and then return ourselves
        def set_text(text)
          @text = text
          self
        end

        attr_reader :line_no, :char_no, :text
        attr_accessor :lex_state
      end
      
      class TkBlockContents < Token
        def text; '...' end
      end
      
      class TkStatementEnd < Token
        def text; '' end
      end

      class TkNode < Token
        attr :node
      end
      
      class TkWhitespace < Token
      end

      class TkId < Token
        def initialize(line_no, char_no, name)
          super(line_no, char_no)
          @name = name
        end
        attr :name
      end

      class TkKW < TkId
      end

      class TkVal < Token
        def initialize(line_no, char_no, value = nil)
          super(line_no, char_no)
          set_text(value)
        end
      end

      class TkOp < Token
        def name
          self.class.op_name
        end
      end

      class TkOPASGN < TkOp
        def initialize(line_no, char_no, op)
          super(line_no, char_no)
          op = TkReading2Token[op] unless op.kind_of?(Symbol)
          @op = op
        end
        attr :op
      end

      class TkUnknownChar < Token
        def initialize(line_no, char_no, id)
          super(line_no, char_no)
          @name = char_no > 255 ? '?' : char_no.chr
        end
        attr :name
      end

      class TkError < Token
      end

      def set_token_position(line, char)
        @prev_line_no = line
        @prev_char_no = char
      end

      def Token(token, value = nil)
        tk = nil
        case token
        when String, Symbol
          source = token.kind_of?(String) ? TkReading2Token : TkSymbol2Token
          if (tk = source[token]).nil?
            IRB.fail TkReading2TokenNoKey, token
          end
          tk = Token(tk[0], value) 
        else 
          if token
            tk = if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty?
                   token.new(@prev_line_no, @prev_char_no)
                 else
                   token.new(@prev_line_no, @prev_char_no, value)
                 end
          end
        end
        tk
      end

      TokenDefinitions = [
        [:TkCLASS,      TkKW,  "class",  EXPR_CLASS],
        [:TkMODULE,     TkKW,  "module", EXPR_BEG],
        [:TkDEF,        TkKW,  "def",    EXPR_FNAME],
        [:TkUNDEF,      TkKW,  "undef",  EXPR_FNAME],
        [:TkBEGIN,      TkKW,  "begin",  EXPR_BEG],
        [:TkRESCUE,     TkKW,  "rescue", EXPR_MID],
        [:TkENSURE,     TkKW,  "ensure", EXPR_BEG],
        [:TkEND,        TkKW,  "end",    EXPR_END],
        [:TkIF,         TkKW,  "if",     EXPR_BEG, :TkIF_MOD],
        [:TkUNLESS,     TkKW,  "unless", EXPR_BEG, :TkUNLESS_MOD],
        [:TkTHEN,       TkKW,  "then",   EXPR_BEG],
        [:TkELSIF,      TkKW,  "elsif",  EXPR_BEG],
        [:TkELSE,       TkKW,  "else",   EXPR_BEG],
        [:TkCASE,       TkKW,  "case",   EXPR_BEG],
        [:TkWHEN,       TkKW,  "when",   EXPR_BEG],
        [:TkWHILE,      TkKW,  "while",  EXPR_BEG, :TkWHILE_MOD],
        [:TkUNTIL,      TkKW,  "until",  EXPR_BEG, :TkUNTIL_MOD],
        [:TkFOR,        TkKW,  "for",    EXPR_BEG],
        [:TkBREAK,      TkKW,  "break",  EXPR_END],
        [:TkNEXT,       TkKW,  "next",   EXPR_END],
        [:TkREDO,       TkKW,  "redo",   EXPR_END],
        [:TkRETRY,      TkKW,  "retry",  EXPR_END],
        [:TkIN,         TkKW,  "in",     EXPR_BEG],
        [:TkDO,         TkKW,  "do",     EXPR_BEG],
        [:TkRETURN,     TkKW,  "return", EXPR_MID],
        [:TkYIELD,      TkKW,  "yield",  EXPR_END],
        [:TkSUPER,      TkKW,  "super",  EXPR_END],
        [:TkSELF,       TkKW,  "self",   EXPR_END],
        [:TkNIL,        TkKW,  "nil",    EXPR_END],
        [:TkTRUE,       TkKW,  "true",   EXPR_END],
        [:TkFALSE,      TkKW,  "false",  EXPR_END],
        [:TkAND,        TkKW,  "and",    EXPR_BEG],
        [:TkOR,         TkKW,  "or",     EXPR_BEG],
        [:TkNOT,        TkKW,  "not",    EXPR_BEG],
        [:TkIF_MOD,     TkKW],
        [:TkUNLESS_MOD, TkKW],
        [:TkWHILE_MOD,  TkKW],
        [:TkUNTIL_MOD,  TkKW],
        [:TkALIAS,      TkKW,  "alias",    EXPR_FNAME],
        [:TkDEFINED,    TkKW,  "defined?", EXPR_END],
        [:TklBEGIN,     TkKW,  "BEGIN",    EXPR_END],
        [:TklEND,       TkKW,  "END",      EXPR_END],
        [:Tk__LINE__,   TkKW,  "__LINE__", EXPR_END],
        [:Tk__FILE__,   TkKW,  "__FILE__", EXPR_END],

        [:TkIDENTIFIER, TkId],
        [:TkFID,        TkId],
        [:TkGVAR,       TkId],
        [:TkIVAR,       TkId],
        [:TkCONSTANT,   TkId],

        [:TkINTEGER,    TkVal],
        [:TkFLOAT,      TkVal],
        [:TkSYMBOL,     TkVal],
        [:TkSTRING,     TkVal],
        [:TkXSTRING,    TkVal],
        [:TkREGEXP,     TkVal],
        [:TkCOMMENT,    TkVal],

        [:TkDSTRING,    TkNode],
        [:TkDXSTRING,   TkNode],
        [:TkDREGEXP,    TkNode],
        [:TkNTH_REF,    TkId],
        [:TkBACK_REF,   TkId],

        [:TkUPLUS,      TkOp,   "+@"],
        [:TkUMINUS,     TkOp,   "-@"],
        [:TkPOW,        TkOp,   "**"],
        [:TkCMP,        TkOp,   "<=>"],
        [:TkEQ,         TkOp,   "=="],
        [:TkEQQ,        TkOp,   "==="],
        [:TkNEQ,        TkOp,   "!="],
        [:TkGEQ,        TkOp,   ">="],
        [:TkLEQ,        TkOp,   "<="],
        [:TkANDOP,      TkOp,   "&&"],
        [:TkOROP,       TkOp,   "||"],
        [:TkMATCH,      TkOp,   "=~"],
        [:TkNMATCH,     TkOp,   "!~"],
        [:TkDOT2,       TkOp,   ".."],
        [:TkDOT3,       TkOp,   "..."],
        [:TkAREF,       TkOp,   "[]"],
        [:TkASET,       TkOp,   "[]="],
        [:TkLSHFT,      TkOp,   "<<"],
        [:TkRSHFT,      TkOp,   ">>"],
        [:TkCOLON2,     TkOp],
        [:TkCOLON3,     TkOp],
        [:OPASGN,       TkOp],               # +=, -=  etc. #
        [:TkASSOC,      TkOp,   "=>"],
        [:TkQUESTION,   TkOp,   "?"],  #?
        [:TkCOLON,      TkOp,   ":"],        #:
    
#        [:TkfLPAREN],         # func( #
#        [:TkfLBRACK],         # func[ #
#        [:TkfLBRACE],         # func{ #
        [:TkSTAR],            # *arg
        [:TkAMPER],           # &arg #
#        [:TkSYMBOL,     TkId],          # :SYMBOL
        [:TkSYMBEG,     TkId], 
        [:TkGT,         TkOp,   ">"],
        [:TkLT,         TkOp,   "<"],
        [:TkPLUS,       TkOp,   "+"],
        [:TkMINUS,      TkOp,   "-"],
        [:TkMULT,       TkOp,   "*"],
        [:TkDIV,        TkOp,   "/"],
        [:TkMOD,        TkOp,   "%"],
        [:TkBITOR,      TkOp,   "|"],
        [:TkBITXOR,     TkOp,   "^"],
        [:TkBITAND,     TkOp,   "&"],
        [:TkBITNOT,     TkOp,   "~"],
        [:TkNOTOP,      TkOp,   "!"],

        [:TkBACKQUOTE,  TkOp,   "`"],

        [:TkASSIGN,     Token,  "="],
        [:TkDOT,        Token,  "."],
        [:TkLPAREN,     Token,  "("],  #(exp)
        [:TkLBRACK,     Token,  "["],  #[arry]
        [:TkLBRACE,     Token,  "{"],  #{hash}
        [:TkRPAREN,     Token,  ")"],
        [:TkRBRACK,     Token,  "]"],
        [:TkRBRACE,     Token,  "}"],
        [:TkCOMMA,      Token,  ","],
        [:TkSEMICOLON,  Token,  ";"],

#        [:TkRD_COMMENT,     TkVal],
        [:TkSPACE,          TkWhitespace],
        [:TkNL,             TkWhitespace],
        [:TkEND_OF_SCRIPT,  TkWhitespace],

        [:TkBACKSLASH,  TkUnknownChar,  "\\"],
        [:TkAT,         TkUnknownChar,  "@"],
        [:TkDOLLAR,     TkUnknownChar,  "\$"], #"
      ]

      # { reading => token_class }
      # { reading => [token_class, *opt] }
      TkReading2Token = {}
      TkSymbol2Token = {}

      def RubyToken.def_token(token_n, super_token = Token, reading = nil, *opts)
        token_n = token_n.id2name unless token_n.kind_of?(String)
        if RubyToken.const_defined?(token_n)
          # IRB.fail AlreadyDefinedToken, token_n
        end

        token_c =  Class.new super_token
        RubyToken.const_set token_n, token_c
        # token_c.inspect
 
        if reading
          if TkReading2Token[reading]
            IRB.fail TkReading2TokenDuplicateError, token_n, reading
          end
          if opts.empty?
            TkReading2Token[reading] = [token_c]
          else
            TkReading2Token[reading] = [token_c].concat(opts)
          end
        end
        TkSymbol2Token[token_n.intern] = token_c

        if token_c <= TkOp
          token_c.class_eval %{
            def self.op_name; "#{reading}"; end
          }
        end
      end

      for defs in TokenDefinitions
        def_token(*defs)
      end

      NEWLINE_TOKEN = TkNL.new(0,0)
      NEWLINE_TOKEN.set_text("\n")

    end



    # Lexical analyzer for Ruby source

    class RubyLex

      ######################################################################
      #
      # Read an input stream character by character. We allow for unlimited
      # ungetting of characters just read.
      #
      # We simplify the implementation greatly by reading the entire input
      # into a buffer initially, and then simply traversing it using
      # pointers.
      #
      # We also have to allow for the <i>here document diversion</i>. This
      # little gem comes about when the lexer encounters a here
      # document. At this point we effectively need to split the input
      # stream into two parts: one to read the body of the here document,
      # the other to read the rest of the input line where the here
      # document was initially encountered. For example, we might have
      #
      #   do_something(<<-A, <<-B)
      #     stuff
      #     for
      #   A
      #     stuff
      #     for
      #   B
      #
      # When the lexer encounters the <<A, it reads until the end of the
      # line, and keeps it around for later. It then reads the body of the
      # here document.  Once complete, it needs to read the rest of the
      # original line, but then skip the here document body.
      #
  
      class BufferedReader
    
        attr_reader :line_num
    
        def initialize(content)
          if /\t/ =~ content
            tab_width = 2
            content = content.split(/\n/).map do |line|
              1 while line.gsub!(/\t+/) { ' ' * (tab_width*$&.length - $`.length % tab_width)}  && $~ #`
              line
            end .join("\n")
          end
          @content   = content
          @content << "\n" unless @content[-1,1] == "\n"
          @size      = @content.size
          @offset    = 0
          @hwm       = 0
          @line_num  = 1
          @read_back_offset = 0
          @last_newline = 0
          @newline_pending = false
        end
    
        def column
          @offset - @last_newline
        end
    
        def getc
          return nil if @offset >= @size
          ch = @content[@offset, 1]
      
          @offset += 1
          @hwm = @offset if @hwm < @offset
      
          if @newline_pending
            @line_num += 1
            @last_newline = @offset - 1
            @newline_pending = false
          end
      
          if ch == "\n"
            @newline_pending = true
          end
          ch
        end
    
        def getc_already_read
          getc
        end
    
        def ungetc(ch)
          raise "unget past beginning of file" if @offset <= 0
          @offset -= 1
          if @content[@offset] == ?\n
            @newline_pending = false
          end
        end
    
        def get_read
          res = @content[@read_back_offset...@offset]
          @read_back_offset = @offset
          res
        end
    
        def peek(at)
          pos = @offset + at
          if pos >= @size
            nil
          else
            @content[pos, 1]
          end
        end
    
        def peek_equal(str)
          @content[@offset, str.length] == str
        end
    
        def divert_read_from(reserve)
          @content[@offset, 0] = reserve
          @size      = @content.size
        end
      end

      # end of nested class BufferedReader

      extend Exception2MessageMapper
      def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
      def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
      def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
      def_exception(:TkReading2TokenDuplicateError, 
        "key duplicate(token_n='%s', key='%s')")
      def_exception(:SyntaxError, "%s")
  
      include RubyToken
      include IRB

      attr_reader :continue
      attr_reader :lex_state

      def RubyLex.debug?
        false
      end

      def initialize(content)
        lex_init

        @reader = BufferedReader.new(content)

        @exp_line_no = @line_no = 1
        @base_char_no = 0
        @indent = 0

        @ltype = nil
        @quoted = nil
        @lex_state = EXPR_BEG
        @space_seen = false
    
        @continue = false
        @line = ""

        @skip_space = false
        @read_auto_clean_up = false
        @exception_on_syntax_error = true
      end

      attr_accessor :skip_space
      attr_accessor :read_auto_clean_up
      attr_accessor :exception_on_syntax_error

      attr :indent

      # io functions
      def line_no
        @reader.line_num
      end

      def char_no
        @reader.column
      end

      def get_read
        @reader.get_read
      end

      def getc
        @reader.getc
      end

      def getc_of_rests
        @reader.getc_already_read
      end

      def gets
        c = getc or return
        l = ""
        begin
          l.concat c unless c == "\r"
          break if c == "\n"
        end while c = getc
        l
      end


      def ungetc(c = nil)
        @reader.ungetc(c)
      end

      def peek_equal?(str)
        @reader.peek_equal(str)
      end

      def peek(i = 0)
        @reader.peek(i)
      end

      def lex
        catch(:eof) do
          until (((tk = token).kind_of?(TkNL) || tk.kind_of?(TkEND_OF_SCRIPT)) &&
             !@continue or
             tk.nil?)
          end
          line = get_read

          if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil?
            nil
          else
            line
          end
        end
      end

      def token
        set_token_position(line_no, char_no)
        catch(:eof) do
          begin
            begin
              tk = @OP.match(self)
              @space_seen = tk.kind_of?(TkSPACE)
            rescue SyntaxError
              abort if @exception_on_syntax_error
              tk = TkError.new(line_no, char_no)
            end
          end while @skip_space and tk.kind_of?(TkSPACE)
          if @read_auto_clean_up
            get_read
          end
          # throw :eof unless tk
          p tk if $DEBUG
          tk.lex_state = lex_state if tk
          tk
        end
      end
  
      ENINDENT_CLAUSE = [
        "case", "class", "def", "do", "for", "if",
        "module", "unless", "until", "while", "begin" #, "when"
      ]
      ACCEPTS_COLON = ["if", "for", "unless", "until", "while"]
      DEINDENT_CLAUSE = ["end" #, "when"
      ]

      PERCENT_LTYPE = {
        "q" => "\'",
        "Q" => "\"",
        "x" => "\`",
        "r" => "/",
        "w" => "]",
        "W" => "]"
      }
  
      PERCENT_PAREN = {
        "{" => "}",
        "[" => "]",
        "<" => ">",
        "(" => ")"
      }

      Ltype2Token = {
        "\'" => TkSTRING,
        "\"" => TkSTRING,
        "\`" => TkXSTRING,
        "/" => TkREGEXP,
        "]" => TkDSTRING
      }
      Ltype2Token.default = TkSTRING

      DLtype2Token = {
        "\"" => TkDSTRING,
        "\`" => TkDXSTRING,
        "/" => TkDREGEXP,
      }

      def lex_init()
        @OP = SLex.new
        @OP.def_rules("\0", "\004", "\032") do |chars, io|
          Token(TkEND_OF_SCRIPT).set_text(chars)
        end

        @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |chars, io|
          @space_seen = TRUE
          while (ch = getc) =~ /[ \t\f\r\13]/
            chars << ch
          end
          ungetc
          Token(TkSPACE).set_text(chars)
        end

        @OP.def_rule("#") do
          |op, io|
          identify_comment
        end

        @OP.def_rule("=begin", proc{@prev_char_no == 0 && peek(0) =~ /\s/}) do
          |op, io|
          str = op
          @ltype = "="


          begin
            line = ""
            begin
              ch = getc
              line << ch
            end until ch == "\n"
            str << line
          end until line =~ /^=end/

          ungetc

          @ltype = nil

          if str =~ /\A=begin\s+rdoc/i
            str.sub!(/\A=begin.*\n/, '')
            str.sub!(/^=end.*/m, '')
            Token(TkCOMMENT).set_text(str)
          else
            Token(TkCOMMENT).set_text(str)
          end
        end

        @OP.def_rule("\n") do
          print "\\n\n" if RubyLex.debug?
          @colonblock_seen = false
          case @lex_state
          when EXPR_BEG, EXPR_FNAME, EXPR_DOT
            @continue = TRUE
          else
            @continue = FALSE
            @lex_state = EXPR_BEG
          end
          Token(TkNL).set_text("\n")
        end

        @OP.def_rules("*", "**",  
          "!", "!=", "!~",
          "=", "==", "===", 
          "=~", "<=>",  
          "<", "<=",
          ">", ">=", ">>") do |op, io|
          @lex_state = EXPR_BEG
          Token(op).set_text(op)
        end

        @OP.def_rules("<<") do |op, io|
          tk = nil
          if @lex_state != EXPR_END && @lex_state != EXPR_CLASS && 
             (@lex_state != EXPR_ARG || @space_seen)
            c = peek(0)
            tk = identify_here_document if /[-\w_\"\'\`]/ =~ c
          end
          if !tk
            @lex_state = EXPR_BEG
            tk = Token(op).set_text(op)
          end
          tk
        end

        @OP.def_rules("'", '"') do |op, io|
          identify_string(op)
        end

        @OP.def_rules("`") do |op, io|
          if @lex_state == EXPR_FNAME
            Token(op).set_text(op)
          else
            identify_string(op)
          end
        end

        @OP.def_rules('?') do |op, io|
          if @lex_state == EXPR_END
            @lex_state = EXPR_BEG
            Token(TkQUESTION).set_text(op)
          else
            ch = getc
            if @lex_state == EXPR_ARG && ch !~ /\s/
              ungetc
              @lex_state = EXPR_BEG
              Token(TkQUESTION).set_text(op)
            else
              str = op
              str << ch
              if (ch == '\\') #'
                str << read_escape
              end
              @lex_state = EXPR_END
              Token(TkINTEGER).set_text(str)
            end
          end
        end

        @OP.def_rules("&", "&&", "|", "||") do |op, io|
          @lex_state = EXPR_BEG
          Token(op).set_text(op)
        end
    
        @OP.def_rules("+=", "-=", "*=", "**=", 
          "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do |op, io|
          @lex_state = EXPR_BEG
          op =~ /^(.*)=$/
          Token(TkOPASGN, $1).set_text(op)
        end

        @OP.def_rule("+@", proc{@lex_state == EXPR_FNAME}) do |op, io|
          Token(TkUPLUS).set_text(op)
        end

        @OP.def_rule("-@", proc{@lex_state == EXPR_FNAME}) do |op, io|
          Token(TkUMINUS).set_text(op)
        end

        @OP.def_rules("+", "-") do |op, io|
          catch(:RET) do
            if @lex_state == EXPR_ARG
              if @space_seen and peek(0) =~ /[0-9]/
                throw :RET, identify_number(op)
              else
                @lex_state = EXPR_BEG
              end
            elsif @lex_state != EXPR_END and peek(0) =~ /[0-9]/
              throw :RET, identify_number(op)
            else
              @lex_state = EXPR_BEG
            end
            Token(op).set_text(op)
          end
        end

        @OP.def_rule(".") do
          @lex_state = EXPR_BEG
          if peek(0) =~ /[0-9]/
            ungetc
            identify_number("")
          else
            # for obj.if
            @lex_state = EXPR_DOT
            Token(TkDOT).set_text(".")
          end
        end

        @OP.def_rules("..", "...") do |op, io|
          @lex_state = EXPR_BEG
          Token(op).set_text(op)
        end

        lex_int2
      end
  
      def lex_int2
        @OP.def_rules("]", "}", ")") do
          |op, io|
          @lex_state = EXPR_END
          @indent -= 1
          Token(op).set_text(op)
        end

        @OP.def_rule(":") do 
          if (@colonblock_seen && @lex_state != EXPR_BEG) || peek(0) =~ /\s/
            @lex_state = EXPR_BEG
            tk = Token(TkCOLON)
          else
            @lex_state = EXPR_FNAME
            tk = Token(TkSYMBEG)
          end
          tk.set_text(":")
        end

        @OP.def_rule("::") do
          # p @lex_state.id2name, @space_seen
          if @lex_state == EXPR_BEG or @lex_state == EXPR_ARG && @space_seen
            @lex_state = EXPR_BEG
            tk = Token(TkCOLON3)
          else
            @lex_state = EXPR_DOT
            tk = Token(TkCOLON2)
          end
          tk.set_text("::")
        end

        @OP.def_rule("/") do |op, io|
          if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
            identify_string(op)
          elsif peek(0) == '='
            getc
            @lex_state = EXPR_BEG
            Token(TkOPASGN, :/).set_text("/=") #")
          elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
            identify_string(op)
          else 
            @lex_state = EXPR_BEG
            Token("/").set_text(op)
          end
        end

        @OP.def_rules("^") do
          @lex_state = EXPR_BEG
          Token("^").set_text("^")
        end

        # @OP.def_rules("^=") do
        #   @lex_state = EXPR_BEG
        #   Token(TkOPASGN, :^)
        # end
    
        @OP.def_rules(",", ";") do |op, io|
          @colonblock_seen = false
          @lex_state = EXPR_BEG
          Token(op).set_text(op)
        end

        @OP.def_rule("~") do
          @lex_state = EXPR_BEG
          Token("~").set_text("~")
        end

        @OP.def_rule("~@", proc{@lex_state = EXPR_FNAME}) do
          @lex_state = EXPR_BEG
          Token("~").set_text("~@")
        end
    
        @OP.def_rule("(") do
          @indent += 1
          # if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
          #  @lex_state = EXPR_BEG
          #  tk = Token(TkfLPAREN)
          # else
            @lex_state = EXPR_BEG
            tk = Token(TkLPAREN)
          # end
          tk.set_text("(")
        end

        @OP.def_rule("[]", proc{@lex_state == EXPR_FNAME}) do
          Token("[]").set_text("[]")
        end

        @OP.def_rule("[]=", proc{@lex_state == EXPR_FNAME}) do
          Token("[]=").set_text("[]=")
        end

        @OP.def_rule("[") do
          @indent += 1
          # if @lex_state == EXPR_FNAME
          #   t = Token(TkfLBRACK)
          # else
          #   if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
          #     t = Token(TkLBRACK)
          #   elsif @lex_state == EXPR_ARG && @space_seen
          #   else
          #     t = Token(TkfLBRACK)
          #   end
          # end
          t = Token(TkLBRACK)
          @lex_state = EXPR_BEG
          t.set_text("[")
        end

        @OP.def_rule("{") do
          @indent += 1
          # if @lex_state != EXPR_END && @lex_state != EXPR_ARG
          #   t = Token(TkLBRACE)
          # else
          #   t = Token(TkfLBRACE)
          # end
          t = Token(TkLBRACE)
          @lex_state = EXPR_BEG
          t.set_text("{")
        end

        @OP.def_rule('\\') do   #'
          if getc == "\n" 
            @space_seen = true
            @continue = true
            Token(TkSPACE).set_text("\\\n")
          else 
            ungetc
            Token("\\").set_text("\\")  #"
          end 
        end 

        @OP.def_rule('%') do
          |op, io|
          if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
            identify_quotation('%')
          elsif peek(0) == '='
            getc
            Token(TkOPASGN, "%").set_text("%=")
          elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
            identify_quotation('%')
          else
            @lex_state = EXPR_BEG
            Token("%").set_text("%")
          end
        end

        @OP.def_rule('$') do  #'
          identify_gvar
        end

        @OP.def_rule('@') do
          if peek(0) =~ /[@\w_]/
            ungetc
            identify_identifier
          else
            Token("@").set_text("@")
          end
        end

        # @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do 
        #   |op, io|
        #   @indent += 1
        #   @lex_state = EXPR_FNAME
        # # @lex_state = EXPR_END
        # # until @rests[0] == "\n" or @rests[0] == ";"
        # #   rests.shift
        # # end
        # end

        @OP.def_rule("__END__", proc{@prev_char_no == 0 && peek(0) =~ /[\r\n]/}) do
          throw :eof
        end

        @OP.def_rule("") do |op, io|
          printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug?
          if peek(0) =~ /[0-9]/
            t = identify_number("")
          elsif peek(0) =~ /[\w_]/
            t = identify_identifier
          end
          printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
          t
        end
    
        p @OP if RubyLex.debug?
      end
  
      def identify_gvar
        @lex_state = EXPR_END
        str = "$"

        tk = case ch = getc
             when /[~_*$?!@\/\\;,=:<>".]/   #"
               str << ch
               Token(TkGVAR, str)
           
             when "-"
               str << "-" << getc
               Token(TkGVAR, str)
           
             when "&", "`", "'", "+"
               str << ch
               Token(TkBACK_REF, str)
           
             when /[1-9]/
               str << ch
               while (ch = getc) =~ /[0-9]/
                 str << ch
               end
               ungetc
               Token(TkNTH_REF)
             when /\w/
               ungetc
               ungetc
               return identify_identifier
             else 
               ungetc
               Token("$")     
             end
        tk.set_text(str)
      end
  
      def identify_identifier
        token = ""
        token.concat getc if peek(0) =~ /[$@]/
        token.concat getc if peek(0) == "@"

        while (ch = getc) =~ /\w|_/
          print ":", ch, ":" if RubyLex.debug?
          token.concat ch
        end
        ungetc
    
        if ch == "!" or ch == "?"
          token.concat getc
        end
        # fix token

        # $stderr.puts "identifier - #{token}, state = #@lex_state"

        case token
        when /^\$/
          return Token(TkGVAR, token).set_text(token)
        when /^\@/
          @lex_state = EXPR_END
          return Token(TkIVAR, token).set_text(token)
        end
    
        if @lex_state != EXPR_DOT
          print token, "\n" if RubyLex.debug?

          token_c, *trans = TkReading2Token[token]
          if token_c
            # reserved word?

            if (@lex_state != EXPR_BEG &&
                @lex_state != EXPR_FNAME &&
                trans[1])
              # modifiers
              token_c = TkSymbol2Token[trans[1]]
              @lex_state = trans[0]
            else
              if @lex_state != EXPR_FNAME
                if ENINDENT_CLAUSE.include?(token)
                  @indent += 1
            
                  if ACCEPTS_COLON.include?(token)  
                    @colonblock_seen = true
                  else
                    @colonblock_seen = false
                  end
                elsif DEINDENT_CLAUSE.include?(token)
                  @indent -= 1
                  @colonblock_seen = false
                end
                @lex_state = trans[0]
              else
                @lex_state = EXPR_END
              end
            end
            return Token(token_c, token).set_text(token)
          end
        end

        if @lex_state == EXPR_FNAME
          @lex_state = EXPR_END
          if peek(0) == '='
            token.concat getc
          end
        elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
          @lex_state = EXPR_ARG
        else
          @lex_state = EXPR_END
        end

        if token[0, 1] =~ /[A-Z]/
          return Token(TkCONSTANT, token).set_text(token)
        elsif token[token.size - 1, 1] =~ /[!?]/
          return Token(TkFID, token).set_text(token)
        else
          return Token(TkIDENTIFIER, token).set_text(token)
        end
      end

      def identify_here_document
        ch = getc
        if ch == "-"
          ch = getc
          indent = true
        end
        if /['"`]/ =~ ch            # '
          lt = ch
          quoted = ""
          while (c = getc) && c != lt
            quoted.concat c
          end
        else
          lt = '"'
          quoted = ch.dup
          while (c = getc) && c =~ /\w/
            quoted.concat c
          end
          ungetc
        end

        ltback, @ltype = @ltype, lt
        reserve = ""

        while ch = getc
          reserve << ch
          if ch == "\\"    #"
            ch = getc
            reserve << ch
          elsif ch == "\n"
            break
          end
        end

        str = ""
        while (l = gets)
          l.chomp!
          l.strip! if indent
          break if l == quoted
          str << l.chomp << "\n"
        end

        @reader.divert_read_from(reserve)

        @ltype = ltback
        @lex_state = EXPR_END
        Token(Ltype2Token[lt], str).set_text(str.dump)
      end
  
      def identify_quotation(initial_char)
        ch = getc
        if lt = PERCENT_LTYPE[ch]
          initial_char += ch
          ch = getc
        elsif ch =~ /\W/
          lt = "\""
        else
          # RubyLex.fail SyntaxError, "unknown type of %string ('#{ch}')"
        end
        # if ch !~ /\W/
        #   ungetc
        #   next
        # end
        # @ltype = lt
        @quoted = ch unless @quoted = PERCENT_PAREN[ch]
        identify_string(lt, @quoted, ch, initial_char) if lt
      end

      def identify_number(start)
        str = start.dup

        if start == "+" or start == "-" or start == ""
          start = getc
          str << start
        end

        @lex_state = EXPR_END

        if start == "0"
          if peek(0) == "x"
            ch = getc
            str << ch
            match = /[0-9a-f_]/
          else
            match = /[0-7_]/
          end
          while ch = getc
            if ch !~ match
              ungetc
              break
            else
              str << ch
            end
          end
          return Token(TkINTEGER).set_text(str)
        end

        type = TkINTEGER
        allow_point = TRUE
        allow_e = TRUE
        while ch = getc
          case ch
          when /[0-9_]/
            str << ch

          when allow_point && "."
            type = TkFLOAT
            if peek(0) !~ /[0-9]/
              ungetc
              break
            end
            str << ch
            allow_point = false

          when allow_e && "e", allow_e && "E"
            str << ch
            type = TkFLOAT
            if peek(0) =~ /[+-]/
              str << getc
            end
            allow_e = false
            allow_point = false
          else
            ungetc
            break
          end
        end
        Token(type).set_text(str)
      end
  
      def identify_string(ltype, quoted = ltype, opener=nil, initial_char = nil)
        @ltype = ltype
        @quoted = quoted
        subtype = nil

        str = ""
        str << initial_char if initial_char
        str << (opener||quoted)

        nest = 0
        begin
          while ch = getc 
            str << ch
            if @quoted == ch 
              if nest == 0
                break
              else
                nest -= 1
              end
            elsif opener == ch
              nest += 1
            elsif @ltype != "'" && @ltype != "]" and ch == "#"
              ch = getc
              if ch == "{"
                subtype = true
                str << ch << skip_inner_expression
              else
                ungetc(ch)
              end
            elsif ch == '\\' #'
              str << read_escape
            end
          end
          if @ltype == "/"
            if peek(0) =~ /i|o|n|e|s/
              str << getc
            end
          end
          if subtype
            Token(DLtype2Token[ltype], str)
          else
            Token(Ltype2Token[ltype], str)
          end.set_text(str)
        ensure
          @ltype = nil
          @quoted = nil
          @lex_state = EXPR_END
        end
      end

      def skip_inner_expression
        res = ""
        nest = 0
        while (ch = getc)
          res << ch
          if ch == '}'
            break if nest.zero?
            nest -= 1
          elsif ch == '{'
            nest += 1
          end
        end
        res
      end

      def identify_comment
        @ltype = "#"
        comment = "#"
        while ch = getc
          if ch == "\\"
            ch = getc
            if ch == "\n"
              ch = " "
            else
              comment << "\\" 
            end
          else
            if ch == "\n"
              @ltype = nil
              ungetc
              break
            end
          end
          comment << ch
        end
        return Token(TkCOMMENT).set_text(comment)
      end
  
      def read_escape
        res = ""
        case ch = getc
        when /[0-7]/
          ungetc ch
          3.times do
          case ch = getc
          when /[0-7]/
          when nil
            break
          else
            ungetc
            break
          end
          res << ch
        end
      
        when "x"
          res << ch
          2.times do
          case ch = getc
          when /[0-9a-fA-F]/
          when nil
            break
          else
            ungetc
            break
          end
            res << ch
          end

        when "M"
          res << ch
          if (ch = getc) != '-'
            ungetc
          else
            res << ch
            if (ch = getc) == "\\" #"
              res << ch
              res << read_escape
            else
              res << ch
            end
          end

        when "C", "c" #, "^"
          res << ch
          if ch == "C" and (ch = getc) != "-"
            ungetc
          else
            res << ch
            if (ch = getc) == "\\" #"
              res << ch
              res << read_escape
            else
              res << ch
            end
          end
        else
          res << ch
        end
        res
      end
    end
  end
end