=begin copyright
    rubylexer - a ruby lexer written in ruby
    Copyright (C) 2004,2005  Caleb Clausen

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2.1 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
=end



require "assert"
require "charhandler"
#require "term"
require "rubycode"
require "io.each_til_charset"

#------------------------------------
class RuLexer

   WHSP=" \t\r\v\f"
   WHSPLF=WHSP+"\n"
   #maybe \r should be in WHSPLF instead

   LEGALCHARS=/[ -~#{WHSPLF}]/

   PAIRS={ '{'=>'}', '['=>']', '('=>')', '<'=>'>'}

   attr_reader :linenum,:last_operative_token

   #-----------------------------------
   def initialize(filename, file, line)
      @filename=filename
      String===file && file=IOext::FakeFile.new(file)
      file.binmode
      @file=file
      @linenum=line
      @toptable=nil   #descendants must fill this out
      @moretokens=[ FileAndLineToken.new(@filename, @linenum, @file.pos) ]
      @last_operative_token=nil
   end

   #-----------------------------------
   def endoffile_detected s=''
     EoiToken.new(s,@file,@file.pos-s.size)
   end

   #-----------------------------------
   def get1token
      @moretokens.empty? or return @moretokens.shift

      if @file.eof?
         #@moretokens<<nil
         return endoffile_detected()
      end

      @toptable.go( nextchar )
   end

   #-----------------------------------
   def no_more?
     @moretokens.each{|t| FileAndLineToken===t or return false }
     return true
   end

private
   #-----------------------------------
   def lexerror_errortoken(tok,str,file=@filename,line=@linenum)
      str or return tok
      tok.extend(ErrorToken).error=str
      tok
   end

   #-----------------------------------
   def lexerror_exception(tok,str,file=@filename,line=@linenum)
      str or return tok
      raise [file,line,'  '+str].join(':')
   end

   #-----------------------------------
   alias lexerror lexerror_errortoken

   #-----------------------------------
   def handler_loop(handler)
      @file.each_byte {|b|   handler.go(b) or break   }
   end

   #-----------------------------------
   def regex(ch=nil)
      result=RenderExactlyStringToken.new('/').
         append_token double_quote("/")

      return result
   end

   #-----------------------------------
   def single_char_token(str)   return @file.getc.chr   end

   #-----------------------------------
   def illegal_char(ch)
     pos=@file.pos
     LEGALCHARS===ch and return( lexerror WsToken.new(getchar,pos), "legal (?!) bad char (code: #{ch[0]})" )
     lexerror WsToken.new(til_charset(LEGALCHARS),pos), "bad char (code: #{ch[0]})"   
   end

   #-----------------------------------
   def fancy_quote (ch)
      assert ch=='%'
      oldpos=@file.pos
      eat_next_if(ch) or raise "fancy_quote, no "+ch

      ch=getchar
      #ch.tr!('qwQWrx','"["{/`')
      type=case ch
         when 'q' then "'"
         when 'w' then "[" #word array
         when 'Q' then '"' #regular string
         when 'W' then '{' #dquotish word array
         when 'r' then '/' #regex
         when 'x' then '`' #exec it
         when 's' then '"' #symbol
         #other letters, nums are illegal here
         when /^[a-z0-9]$/oi
            error= "unrecognized %string type: "+ch; '"'
         when ''
            return lexerror( StringToken.new('', oldpos), "unexpected eof in %string")
         else back1char; '"' #no letter means string too
      end

      beg=nextchar.chr
      if /^[\r\n]$/===beg  then 
           beg=INET_NL_REX
      end

      result=double_quote(beg, type, (PAIRS[beg] or beg))
      case ch
      when /^[Wwr]$/: result=RenderExactlyStringToken.new(type).append_token(result)
      when 's': result=SymbolToken.new(result.to_s)
      end
      result.offset=oldpos
      return lexerror(result,error)
   end

   #-----------------------------------
   #this method is now misnamed, since it handles single quotes as well
   def double_quote(nester, type=nester, delimiter=nester)
      all_quote(nester,type,delimiter)
   end

   #-----------------------------------
   def all_quote(nester, type, delimiter, bs_handler=nil)
      #string must start with nester
      if nester==INET_NL_REX
        readnl
      else
        eat_next_if(nester)
      end or return nil

      bs_handler ||= case type
        when '/','{' then :regex_esc_seq
        when '"','`',':' then :dquote_esc_seq
        when "'"     then :squote_esc_seq
        when "["     then :wquote_esc_seq
        else raise "unknown quote type: #{type}"
      end

      str=StringToken.new type
      old_linenum=@linenum
      nestlevel=1
      maybe_crunch= "'["[type] ? nil : "#"
      @file.each_byte {|b|
         b=b.chr
         if /^[\r\n]$/===b
           back1char
           b=readnl
         end
         case b
            when delimiter
               if (nestlevel-=1)==0
                  str.modifiers=til_charset(/[^eioumnsx]/) if '/'==type
                  #emit eol marker later if line has changed
                  @linenum != old_linenum and @moretokens <<
                     FileAndLineToken.new(@filename,@linenum,@file.pos)
                  return str
               end
            when nester
               #this branch ignored if nester==delimiter
               assert(nester!=delimiter)
               nestlevel+=1
            when "\\"
               b= send(bs_handler,'\\',nester,delimiter)
            when nil then raise "nil char from each_byte?" #never happens
            when maybe_crunch
               nc=nextchar.chr
               nc[/^[{@$]$/] and b=ruby_code(nc)
         end
         str.append b
      }

      assert @file.eof?
      lexerror str,"unterminated #{delimiter}-string"
   end

   #-----------------------------------
   ESCAPECHRS="abefnrstv"
   ESCAPESEQS="\a\b\e\f\n\r\s\t\v"
   def dquote_esc_seq(ch,nester=nil,delimiter=nil)
      assert ch == '\\'
      #see ruby book, p 205 for documentation of escape sequences
      return case k=getchar
         when "\n" then @linenum+=1; ""
         when "\\" then "\\"
         when '"' then '"'
         when '#' then '#'
         when /^[#{ESCAPECHRS}]$/o
            k.tr(ESCAPECHRS,ESCAPESEQS)
=begin not needed anymore
         when "a"  then "\a"
         when "b"  then "\b"
         when "e"  then "\e"
         when "f"  then "\f"
         when "n"  then "\n"
         when "r"  then "\r"
         when "s"  then "\ "
         when "t"  then "\t"
         when "v"  then "\v"
=end
         when "M"
            eat_next_if(?-) or lexerror 'bad \\M sequence'
            (getchar_maybe_escape | 0x80).chr

         when "C"
            eat_next_if(?-) or lexerror 'bad \\C sequence'
            (getchar_maybe_escape & 0x9F).chr

         when "c"
            (getchar_maybe_escape & 0x9F).chr

         when /^[0-7]$/
            str=k
            while str.length < 3
               str << (eat_next_if(/^[0-7]$/) or break)
            end
            (str.oct&0xFF).chr

         when "x"
            str=''
            while str.length < 2
               str << (eat_next_if(/^[0-9A-F]$/i) or break)
            end
            str=='' and lexerror "bad \\x sequence"
            str.hex.chr

         else
            '\\'+k
      end
   end

   #-----------------------------------
   def regex_esc_seq(ch,nester,delimiter)
      assert ch == '\\'
      c=getchar
      return case c
         when "\n"
            @linenum+=1
            ''#ch+c
         when nester,delimiter  ,"/"
            c
         #when "c"
         #   ch + c + getchar
         #when "M","C"
         #   eat_next_if(?-) or
         #         lexerror "illegal \\#{c}- esc sequence"
         #   ch + c + '-' + (eat_next_if(/^[^\\]$/)or'')
         #   #if this \M- or \C- sequence is continued by
         #   #another backslash, we'll just leave the
         #   #backslash on the input, to be read by the next pass
         else
            ch+c
      end
   end

   #-----------------------------------
   def wquote_esc_seq(ch,nester,delimiter)
      assert(ch=='\\')

      #get the escaped character
      escchar=getchar
      return (case escchar
         #all \ sequences but \delimiter, \nester
         #are passed thru unchanged; actual
         #newlines are counted but not changed
         when delimiter,nester
              ''
         when "\n"
              @linenum+=1; "\\"
         else '\\'
      end+escchar)
   end

   #-----------------------------------
   def squote_esc_seq(ch,nester,delimiter)
      assert(ch=='\\')

      #get the escaped character
      escchar=getchar
      return (case escchar
         #all \ sequences but \delimiter, \nester and \\
         #are passed thru unchanged; actual
         #newlines are counted but not changed
         when delimiter,nester,'\\'
              ''
         when "\n"
              @linenum+=1; "\\"
         else '\\'
      end+escchar)
   end

   #-----------------------------------
   def ruby_code(ch='{')
      assert ch[/^[{(@$]$/]
      klass= RubyLexer===self ? self.class : RubyLexer
      rl=klass.new(@filename,@file,@linenum)



      case ch
      when '@'
         tokens=[rl.at_identifier]
      when '$'
         tokens=[rl.dollar_identifier]
      when '{','('
         tokens=[]
         loop {
            tok=rl.get1token
            EoiToken===tok and lexerror tok,"unterminated string inclusion"
            tokens << tok
            break if tok===PAIRS[ch] and rl.no_more? and rl.balanced_braces?
         }
      else
         raise 'hell'
      end

      if @linenum != rl.linenum
        last=tokens.pop
        fal=FileAndLineToken.new(@filename,@linenum, last.offset)
        tokens.push fal,last
      end

      #need to verify that rl's @moretokens, @incomplete_here_tokens are empty
      rl.incomplete_here_tokens.empty? or 
        here_spread_over_ruby_code rl,tokens.last
      rl.no_more? or
        raise 'uh-oh, ruby tokens were lexed past end of ruby code'

      result=RubyCode.new(tokens,@filename,@linenum)
      @linenum=rl.linenum
      return result
   end
   
   #-----------------------------------
   def here_spread_over_ruby_code(rl,tok)
     lexerror tok, 'here body outside string inclusion'
   end
   

   #-----------------------------------
   BINCHARS=?0..?1
   OCTCHARS=?0..?7
   DECCHARS=?0..?9
   HEXCHARS=CharSet[?0..?9, ?A..?F, ?a..?f]
   #0-9
   #-----------------------------------
   def number(str)

      return nil unless /^[0-9+\-]$/===str

      interp=:to_i
      str=  (eat_next_if(/^[+\-]$/)or'')
      str<< (eat_next_if(/^[0-9]$/)or'')

      if str[-1] == ?0 and nextchar !=?.
         typechar=eat_next_if(/^[BOX]$/i)||'o'
         str << typechar
         interp=:oct
         allowed=case typechar
           when 'b','B': BINCHARS
           when 'x','X': HEXCHARS
           when 'o','O': OCTCHARS
           else raise  :impossible
         end
      else
         interp=:to_i
         allowed =DECCHARS
      end

      addl_dig_seqs= ((!typechar)? 2 : 0)      #den 210
      error=nil
      @file.each_byte { |b|
         if allowed === b or ?_ == b
            str << b
         else
            #digits must follow and precede . and e
            if ?.==b and addl_dig_seqs==2 and allowed===nextchar
               addl_dig_seqs=1
               str << b
            #digits must follow and precede . and e
            elsif (?e==b or ?E==b) and addl_dig_seqs>=1 and
                  readahead(2)[/^[-+]?[0-9]/]
               addl_dig_seqs=0
               str << b
               str << (eat_next_if(/[+\-]/)or'')
            else
               back1char
               #return(str.send(interp))
               break
            end
            #OCTCHARS allowed here to permit constants like this: 01.2
            allowed == DECCHARS or allowed == OCTCHARS or error= "floats are always decimal (currently)"
            allowed = DECCHARS
            interp=:to_s
         end
      }

      assert(str[/[0-9]/])
      lexerror NumberToken.new(str.send(interp)), error
   end

   #-----------------------------------
   def comment(str=nil)
      #assert str == '#'
      str=eat_next_if(?#) or return nil
      Process.kill("INT",0) if
         readahead(10)==%/breakpoint/ and defined? DEBUGGER__
if false
     @file.each_byte {|b|
         if b==?\n #leave \n's on input for newline to eat
            back1char
         else
            str << b
         end
         return IgnoreToken.new(str) if b==?\n or b==?#
      }
      #eof...
else
      str<<til_charset(/[\r\n#]/)
      eat_next_if ?# and str<<?#
end
      return IgnoreToken.new(str)
   end

   #-----------------------------------
   def whitespace(ch)
      assert ch[/^[#{WHSP}]$/o]
      oldpos=@file.pos
      str=til_charset(/[^#{WHSP}]/o)
      return WsToken.new(str,oldpos)
   end

   #-----------------------------------
   INET_NL_REX=/^(\r\n?|\n\r?)/
   def readnl
      #compatible with dos/mac style newlines...
      nl=readahead(2)[INET_NL_REX]
      nl or return nil
      assert((1..2)===nl.length)
      @linenum+=1
      @file.read nl.length
   end

   #-----------------------------------
   def newline(ch)
      offset=@file.pos
      nl=readnl
      @moretokens << FileAndLineToken.new( @filename, @linenum, @file.pos )
      return NewlineToken.new( nl,offset)
   end


   #-----------------------------------
   def getchar_maybe_escape
      @file.eof? and lexerror huh,"unterminated dq string"
      c=@file.getc

      c == ?\\ and
         (c = (dquote_esc_seq('\\')[-1] or ?\n))

      return c
   end

  #-----------------------------------
  def RuLexer.delegate_to(obj,*names)
    eval names.collect {|name|
      "define_method(:#{name}) do|*args|
        #{obj}.#{name}(*args)
      end
      "
    }.to_s
  end

protected
  delegate_to :@file, :eat_next_if,:prevchar,:nextchar,:getchar,:back1char,:readahead,:readback,:til_charset

  #-----------------------------------
  def RuLexer.save_offsets_in(*funcnames)
    eval funcnames.collect{|fn| <<-endeval }.to_s
      class ::#{self}
        alias #{fn}__no_offset #{fn}   #rename old ver of fn
        def #{fn}(*args)               #create new version
          pos=@file.pos
          result=#{fn}__no_offset(*args)
          assert Token===result
          result.offset||=pos
          return result
        end
      end
    endeval
  end



end