=begin copyright
    rubylexer - a ruby lexer written in ruby
    Copyright (C) 2004,2005  Caleb Clausen

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2.1 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
=end



require 'rubylexer/rulexer' #must be 1st!!!
require 'rubylexer/version'
require 'rubylexer/token'
require 'rubylexer/charhandler'
require 'rubylexer/symboltable'
#require "io.each_til_charset"
require 'rubylexer/context'
require 'rubylexer/tokenprinter'


#-----------------------------------
class RubyLexer
  include NestedContexts
 
   RUBYSYMOPERATORREX=
      %r{^([&|^/%~]|=(==?|~)|>[=>]?|<(<|=>?)?|[+\-]@?|\*\*?|\[\]=?)}
      # (nasty beastie, eh?)
      #these are the overridable operators
      #does not match flow-control operators like: || && ! or and if not
      #or op= ops like: += -= ||=
      #or .. ... ?:
      #for that use:
   RUBYNONSYMOPERATORREX=
      %r{^([%^~/\-+|&]=|(\|\||&&)=?|(<<|>>|\*\*?)=|\.{1,3}|[?:,;]|=>?|![=~]?)$}
   RUBYOPERATORREX=/#{RUBYSYMOPERATORREX}|#{RUBYNONSYMOPERATORREX}/o
   UNSYMOPS=/^[~!]$/ #always unary
   UBSYMOPS=/^([*&+-]|::)$/  #ops that could be unary or binary
   WHSPCHARS=WHSPLF+"\\#"
   OPORBEGINWORDS="(if|unless|while|until)"
   BEGINWORDS=/^(def|class|module|begin|for|case|do|#{OPORBEGINWORDS})$/o
   FUNCLIKE_KEYWORDS=/^(break|next|redo|return|raise|yield|defined\?|retry|super|BEGIN|END)$/
   VARLIKE_KEYWORDS=/^(__FILE__|__LINE__|false|nil|self|true)$/
   INNERBOUNDINGWORDS="(else|elsif|ensure|in|then|rescue|when)"
   BINOPWORDS="(and|or)"
   NEVERSTARTPARAMLISTWORDS=/^(#{OPORBEGINWORDS}|#{INNERBOUNDINGWORDS}|#{BINOPWORDS}|end)([^a-zA-Z0-9_!?=]|\Z)/o
   NEVERSTARTPARAMLISTFIRST=CharSet['aoeitrwu']  #chars that begin NEVERSTARTPARAMLIST
   NEVERSTARTPARAMLISTMAXLEN=7     #max len of a NEVERSTARTPARAMLIST
   
   RUBYKEYWORDS=%r{
     ^(alias|#{BINOPWORDS}|not|undef|end|
       #{VARLIKE_KEYWORDS}|#{FUNCLIKE_KEYWORDS}|
       #{INNERBOUNDINGWORDS}|#{BEGINWORDS}
     )$
   }xo
      #__END__ should not be in this set... its handled in start_of_line_directives

   CHARMAPPINGS = {
         ?$ => :dollar_identifier,
         ?@ => :at_identifier,
         ?a..?z => :identifier,
         ?A..?Z => :identifier,
         ?_     => :identifier,
         ?0..?9 => :number,
         %{"'} => :double_quote,
         ?` => :back_quote,

         WHSP => :whitespace, #includes \r
         ?, => :comma,
         ?; => :semicolon,

         ?^ => :biop,
         ?~ => :tilde,
         ?= => :equals,
         ?! => :exclam,
         ?. => :dot,

         #these ones could signal either an op or a term
         ?/ => :regex_or_div,
         "|" => :conjunction_or_goalpost,
         ">" => :quadriop,
         "*&" => :star_or_amp,        #could be unary
         "+-" => :plusminus, #could be unary
         ?< => :lessthan,
         ?% => :percent,
         ?? => :char_literal_or_op,  #single-char int literal
         ?: => :symbol_or_op,
         ?\n => :newline, #implicitly escaped after op
         #?\r => :newline, #implicitly escaped after op

         ?\\ => :escnewline,
         ?\0 => :eof,

         "[({" => :open_brace,
         "])}" => :close_brace,


         ?# => :comment
   }

   attr_reader :incomplete_here_tokens, :parsestack


   #-----------------------------------
   def initialize(filename,file,linenum=1)
      super(filename,file, linenum)
      @start_linenum=linenum
      @parsestack=[TopLevelContext.new]
      @incomplete_here_tokens=[]
      @localvars_stack=[SymbolTable.new]
      @defining_lvar=nil
      @in_def_name=false

      @toptable=CharHandler.new(self, :illegal_char, CHARMAPPINGS)

      start_of_line_directives
   end
   
   def localvars;
     @localvars_stack.last
   end

   #-----------------------------------
   def get1token
      result=super  #most of the action's here

      #now cleanup and housekeeping


      #check for bizarre token types
      case result
      when StillIgnoreToken#,nil
          result
      when Token#,String
          @last_operative_token=result
          assert !(IgnoreToken===@last_operative_token)
          result
      else
          raise "#{@filename}:#{linenum}:token is a #{result.class}, last is #{@last_operative_token}"
      end
   end


   #-----------------------------------
   def balanced_braces?

       #@parsestack.empty?
       @parsestack.size==1 and TopLevelContext===@parsestack.first
   end

   #-----------------------------------
   def dollar_identifier(ch=nil)
      s=eat_next_if(?$) or return nil

      if t=((identifier_as_string(?$) or special_global))
        s<<t
      else error= "missing $id name"
      end

      return lexerror(VarNameToken.new(s),error)
   end

   #-----------------------------------
   def at_identifier(ch=nil)
      result =  (eat_next_if(?@) or return nil)
      result << (eat_next_if(?@)or'')
      if t=identifier_as_string(?@)
        result<<t
      else error= "missing @id name"
      end
      return lexerror(VarNameToken.new(result),error)
   end

private
   #-----------------------------------
   def here_spread_over_ruby_code(rl,tok)
     assert(!rl.incomplete_here_tokens.empty?)
     @incomplete_here_tokens += rl.incomplete_here_tokens
   end

   #-----------------------------------
   def expect_do_or_end_or_nl!(st)
     @parsestack.push ExpectDoOrNlContext.new(st,/(do|;|:|\n)/,@linenum)
   end

  #-----------------------------------
  #match NoWstoken, ws, comment, or (escaped?) newline repeatedly
  def maybe_no_ws_token
    result=[]
    while IgnoreToken===(tok=get1token)
      EoiToken===tok and lexerror tok,"end of file not expected here"
      result << tok
    end
    assert((not IgnoreToken===tok))
    @moretokens.unshift tok
    return result
  end

  #-----------------------------------
  WSCHARSET=/[#\\\n\s\t\v\r\f]/
  def ignored_tokens(allow_eof=false,allow_eol=true)
    result=[]
    result<<@moretokens.shift while StillIgnoreToken===@moretokens.first
    @moretokens.empty? or return result
    loop do
      unless @moretokens.empty?
        case @moretokens.first
        when StillIgnoreToken
        when NewlineToken: allow_eol or break
        else break
        end 
      else
      
        break unless ch=nextchar
        ch=ch.chr
        break unless WSCHARSET===ch
        break if ch[/[\r\n]/] and !allow_eol
      end
      

      tok=get1token
      result << tok
      case tok
        when NewlineToken; assert allow_eol; block_given? and yield tok
        when EoiToken; allow_eof or lexerror tok,"end of file not expected here(2)"
        when StillIgnoreToken
        else raise "impossible token: #{tok.inspect}"
      end
    end

=begin
      @whsphandler||=CharHandler.new(self, :==,
         "#" => :comment,
         "\n" => :newline,
         "\\" => :escnewline,
         "\s\t\v\r\f" => :whitespace
      )
      #tok=nil
      while tok=@whsphandler.go((nextchar or return result))
         block_given? and NewlineToken===tok and yield tok
         result << tok
      end
=end
    return result
  end

   #-----------------------------------
   def safe_recurse
      old_moretokens=@moretokens
      #old_parsestack=@parsestack.dup
      @moretokens=[]
      result=   yield @moretokens
      #assert @incomplete_here_tokens.empty?
      #assert @parsestack==old_parsestack
      @moretokens= old_moretokens.concat @moretokens
      return result
      #need to do something with @last_operative_token?
   end

   #-----------------------------------
   def special_global   #handle $-a and friends
      assert prevchar=='$'
      result = ((
      #order matters here, but it shouldn't
      #(but til_charset must be last)
         eat_next_if(/[!@&+`'=~\/\\,.;<>*"$?:]/) or
         (eat_next_if('-') and ("-"+getchar)) or
         (?0..?9)===nextchar ? til_charset(/[^\d]/) : nil
      ))
   end

   #-----------------------------------
   def identifier(context=nil)
      oldpos= input_position
      str=identifier_as_string(context)

      #skip keyword processing if 'escaped' as it were, by def, . or ::
      #or if in a non-bare context
      #just asserts because those contexts are never encountered.
      #control goes through symbol(<...>,nil) 
      assert( /^[a-z_]$/i===context)
      assert !(@last_operative_token===/^(\.|::|(un)?def|alias)$/)

      @moretokens.unshift(*parse_keywords(str,oldpos) do
        #if not a keyword,
        case str
          when FUNCLIKE_KEYWORDS; #do nothing
          when VARLIKE_KEYWORDS,RUBYKEYWORDS; raise "shouldnt see keywords here, now"
        end
        safe_recurse { |a| var_or_meth_name(str,@last_operative_token,oldpos) }
      end)
      return @moretokens.shift
   end

   #-----------------------------------
   def identifier_as_string(context)
      #must begin w/ letter or underscore
      str=eat_next_if(/[_a-z]/i) or return nil

      #equals, question mark, and exclamation mark
      #might be allowed at the end in some contexts.
      #(in def headers and symbols)
      #otherwise, =,?, and ! are to be considered
      #separate tokens. confusing, eh?
      #i hope i've captured all right conditions....
      #context should always be ?: right after def, ., and :: now

      maybe_eq,maybe_qm,maybe_ex = case context
         when ?@,?$ then [nil,nil,nil]
         when ?:    then [?=, ??, ?!]
         else            [nil,??, ?!]
      end
      
      @in_def_name and maybe_eq= ?=

      str<<til_charset(/[^a-z0-9_]/i)

      #look for ?, !, or =, if allowed
      case b=getc
      when nil #means we're at eof
         #handling nil here prevents b from ever matching
         #a nil value of maybe_qm, maybe_ex or maybe_eq
      when maybe_qm
         str << b
      when maybe_ex
         nc=(nextchar unless eof?)
         #does ex appear to be part of a larger operator?
         if nc==?= #or nc==?~
           back1char
         else
           str << b
         end
      when maybe_eq
         nc=(nextchar unless eof?)
         #does eq appear to be part of a larger operator?
         if nc==?= or nc==?~ or nc==?>
           back1char
         else
           str << b
         end
      else
         back1char
      end


      return str
   end

  #-----------------------------------
  #contexts in which comma may appear in ruby:
    #multiple lhs (terminated by assign op)
    #multiple rhs (in implicit context) 
    #method actual param list (in ( or implicit context)
    #method formal param list (in ( or implicit context)
    #block  formal param list (in | context) 
    #nested multiple rhs 
    #nested multiple lhs 
    #nested block formal list 
    #element reference/assignment (in [] or []= method actual parameter context)
    #hash immediate (in imm{ context)
    #array immediate (in imm[ context)
    #list between 'for' and 'in'
    #list after rescue
    #list after when
    #list after undef

    #note: comma in parens not around a param list or lhs or rhs is illegal

   #-----------------------------------
   #a comma has been seen. are we in an
   #lvalue list or some other construct that uses commas?
   def comma_in_lvalue_list?
     @parsestack.last.lhs= (not ListContext===@parsestack.last)
   end
   
   #-----------------------------------
   def in_lvar_define_state
     #@defining_lvar is a hack
     @defining_lvar or case ctx=@parsestack.last
       when ForSMContext; ctx.state==:for
       when RescueSMContext; ctx.state==:arrow
       #when BlockParamListLhsContext; true
     end 
   end
   
   #-----------------------------------
   #determine if an alphabetic identifier refers to a variable
   #or method name. generates implicit parenthes(es) if it is a
   #call site and no explicit parens are present. starts an implicit param list
   #if appropriate. adds tok to the
   #local var table if its a local var being defined for the first time.
   
   #note: what we here call variables (rather, constants) following ::
   #might actually be methods at runtime, but that's immaterial to tokenization.
   
   #note: this routine should determine the correct token type for name and
   #create the appropriate token. currently this is not done because callers
   #typically have done it (perhaps incorrectly) already.
   def var_or_meth_name(name,lasttok,pos)
     #look for call site if not a keyword or keyword is function-like
     #look for and ignore local variable names

     assert String===name
     
     #maybe_local really means 'maybe local or constant'
     maybe_local=case name
       when /[^a-z_0-9]$/i; #do nothing
       when /^[a-z_]/;  (localvars===name or VARLIKE_KEYWORDS===name or in_lvar_define_state) and not lasttok===/^(\.|::)$/
       when /^[A-Z]/; is_const=true;not lasttok==='.'  #this is the right algorithm for constants... 
     end 
          
     assert(@moretokens.empty?)
     
     oldlast=@last_operative_token

     tok=@last_operative_token=VarNameToken.new(name,pos)

     oldpos= input_position
     sawnl=false
     result=ws_toks=ignored_tokens(true) {|nl| sawnl=true }
     if sawnl || eof? 
         if  maybe_local then 
           if in_lvar_define_state
             if /^[a-z_][a-zA-Z_0-9]*$/===name 
               assert !(lasttok===/^(\.|::)$/)
               localvars[name]=true
             else
               lexerror tok,"not a valid variable name: #{name}"
             end
             return result.unshift(tok)
           end
           return result.unshift(tok) #if is_const
         else 
           return result.unshift(
             MethNameToken.new(name,pos),  #insert implicit parens right after tok
             ImplicitParamListStartToken.new( oldpos),
             ImplicitParamListEndToken.new( oldpos) 
           )
         end
     end
     
     #if next op is assignment (or comma in lvalue list)
     #then omit implicit parens
     assignment_coming=case nc=nextchar
       when ?=;  not /^=[>=~]$/===readahead(2)
       when ?,; comma_in_lvalue_list? 
       when ?); last_context_not_implicit.lhs
       when ?>,?<; /^(.)\1=$/===readahead(3)
       when ?*,?&; /^(.)\1?=/===readahead(3)
       when ?|; /^\|\|?=/===readahead(3) or
                #is it a goalpost?
                BlockParamListLhsContext===last_context_not_implicit &&
                readahead(2)[1] != ?|
       when ?%,?/,?-,?+,?^; readahead(2)[1]== ?=
     end 
     if (assignment_coming && !(lasttok===/^(\.|::)$/) or in_lvar_define_state)
        tok=VarNameToken.new(name,pos)
        if /[^a-z_0-9]$/i===name 
          lexerror tok,"not a valid variable name: #{name}"
        elsif /^[a-z_]/===name and !(lasttok===/^(\.|::)$/)
          localvars[name]=true
        end
        return result.unshift(tok)
     end
     
     implicit_parens_to_emit=
     if assignment_coming
       @parsestack.push AssignmentContext.new(nil) if nc==?% or nc==?/
       0
     else
     case nc
       when nil: 2
       when ?!; readahead(2)=='!=' ? 2 : 1
       when NEVERSTARTPARAMLISTFIRST
         (NEVERSTARTPARAMLISTWORDS===readahead(NEVERSTARTPARAMLISTMAXLEN)) ? 2 : 1
       when ?",?',?`,?a..?z,?A..?Z,?0..?9,?_,?@,?$,?~; 1
       when ?{
         maybe_local=false
         x=2
         x-=1 if /\A(return|break|next)\Z/===name and 
                 !(KeywordToken===oldlast and oldlast===/\A(\.|::)\Z/)
         x
       when ?(; 
         maybe_local=false; !(ws_toks.empty? or lasttok===/^(\.|::)$/)? 1 : 0
       when ?},?],?),?;,?^, ?|, ?>, ?,, ?., ?=; 2
       when ?+, ?-, ?*, ?&, ?%, ?/; (ws_toks.empty? || readahead(2)[/^.[#{WHSPLF}]/]) ? 2 : 3
       when ?:,??; next2=readahead(2);
                   WHSPLF[next2[1].chr] || next2=='::' ? 2 : 3
#       when ?:,??; (readahead(2)[/^.[#{WHSPLF}]/]) ? 2 : 3
       when ?<; (ws_toks.empty? || readahead(3)[/^<<["'`a-zA-Z_0-9-]/]) ? 3 : 2 
       when ?[; ws_toks.empty? ? 2 : 3
       when ?\\, ?\s, ?\t, ?\n, ?\r, ?\v, ?#; raise 'failure'
       else raise "unknown char after ident: #{nc=nextchar ? nc.chr : "<<EOF>>"}"
     end
     end

     if is_const and implicit_parens_to_emit==3 then
       implicit_parens_to_emit=1
     end
     
     tok=if maybe_local and implicit_parens_to_emit>=2 
       implicit_parens_to_emit=0
       VarNameToken
     else
       MethNameToken
     end.new(name,pos)
     
     case implicit_parens_to_emit
     when 2;
       result.unshift ImplicitParamListStartToken.new(oldpos),
                 ImplicitParamListEndToken.new(oldpos)
     when 1,3;
       arr,pass=*param_list_coming_with_2_or_more_params?
       result.push( *arr )
       unless pass
         result.unshift ImplicitParamListStartToken.new(oldpos)
         @parsestack.push ParamListContextNoParen.new(@linenum)
       end
     when 0; #do nothing
     else raise 'invalid value of implicit_parens_to_emit'
     end
     return result.unshift(tok)
     # 'ok:'
     # 'if unless while until {'
     # '\n (unescaped) and or'
     # 'then else elsif rescue ensure (illegal in value context)'

     # 'need to pop noparen from parsestack on these tokens: (in operator context)'
     # 'not ok:'
     # 'not (but should it be?)'
   end

   #-----------------------------------
   def param_list_coming_with_2_or_more_params?
     WHSPCHARS[prevchar] && (?(==nextchar) or return [[],false]
       basesize=@parsestack.size
       result=[get1token]
       pass=loop{
         tok=get1token
         result<<tok
         if @parsestack.size==basesize
           break false
         elsif ','==tok.to_s and @parsestack.size==basesize+1
           break true 
         elsif EoiToken===tok
           lexerror tok, "unexpected eof in parameter list"
         end
       }
       return [result,pass]
   end

  #-----------------------------------
  CONTEXT2ENDTOK={AssignmentRhsContext=>AssignmentRhsListEndToken, 
                  ParamListContextNoParen=>ImplicitParamListEndToken,
                  WhenParamListContext=>KwParamListEndToken,
                  RescueSMContext=>KwParamListEndToken
                 }
  def abort_noparens!(str='')
    #assert @moretokens.empty?
    result=[]
    while klass=CONTEXT2ENDTOK[@parsestack.last.class]
      result << klass.new(input_position-str.length)
      break if RescueSMContext===@parsestack.last
      @parsestack.pop 
    end
    return result
  end

if false #no longer used
   #-----------------------------------
   def abort_1_noparen!(offs=0)
     assert @moretokens.empty?
     result=[]
     while AssignmentRhsContext===@parsestack.last
       @parsestack.pop
       result << AssignmentRhsListEndToken.new(input_position-offs)
     end
     ParamListContextNoParen===@parsestack.last or lexerror huh,'{} with no matching callsite'
       @parsestack.pop
       result << ImplicitParamListEndToken.new(input_position-offs)
     return result
   end
end

   #-----------------------------------
   #parse keywords now, to prevent confusion over bare symbols
   #and match end with corresponding preceding def or class or whatever.
   #if arg is not a keyword, the block is called
   def parse_keywords(str,offset)
      assert @moretokens.empty?
      result=[KeywordToken.new(str,offset)]

      case str
      when "end"
         result.unshift(*abort_noparens!(str))
         @parsestack.last.see self,:semi #sorta hacky... should make an :end event instead?

=begin not needed?
         if ExpectDoOrNlContext===@parsestack.last
            @parsestack.pop
            assert @parsestack.last.starter[/^(while|until|for)$/]
         end
=end

         WantsEndContext===@parsestack.last or lexerror result.last, 'unbalanced end'
         ctx=@parsestack.pop
         start,line=ctx.starter,ctx.linenum
         BEGINWORDS===start or lexerror result.last, "end does not match #{start or "nil"}"
         /^(do)$/===start and localvars.end_block
         /^(class|module|def)$/===start and @localvars_stack.pop

      when "class","module"
         result.first.has_end!
         @parsestack.push WantsEndContext.new(str,@linenum)
         @localvars_stack.push SymbolTable.new 
         
      when "if","unless" #could be infix form without end
         if after_nonid_op?{false} #prefix form
            result.first.has_end!
            @parsestack.push WantsEndContext.new(str,@linenum)
            

         else #infix form
           result.unshift(*abort_noparens!(str))
         end
      when "begin","case"
         result.first.has_end!
         @parsestack.push WantsEndContext.new(str,@linenum)
      when "while","until" #could be infix form without end
         if after_nonid_op?{false} #prefix form
           result.first.has_end!
           @parsestack.push WantsEndContext.new(str,@linenum)
           expect_do_or_end_or_nl! str

         else #infix form
           result.unshift(*abort_noparens!(str))
         end
      when "for"
         result.first.has_end!
         result.push KwParamListStartToken.new(offset+str.length)
         # corresponding EndToken emitted leaving ForContext ("in" branch, below)
         @parsestack.push WantsEndContext.new(str,@linenum)
         #expect_do_or_end_or_nl! str #handled by ForSMContext now
         @parsestack.push ForSMContext.new(@linenum)
      when "do"
         result.unshift(*abort_noparens!(str))
         if ExpectDoOrNlContext===@parsestack.last
            @parsestack.pop
            assert WantsEndContext===@parsestack.last
         else
            result.last.has_end!
            @parsestack.push WantsEndContext.new(str,@linenum)
            localvars.start_block
            block_param_list_lookahead
         end
      when "def"
         result.first.has_end!
         @parsestack.push WantsEndContext.new("def",@linenum)
         @localvars_stack.push SymbolTable.new
         safe_recurse { |aa|
            @last_operative_token=KeywordToken.new "def" #hack
            result.concat ignored_tokens

            #read an expr like a.b.c or a::b::c
            #or (expr).b.c
            if nextchar==?( #look for optional parenthesised head
              old_size=@parsestack.size
              parencount=0
              begin
                tok=get1token
                case tok
                when/^\($/.token_pat then parencount+=1
                when/^\)$/.token_pat then parencount-=1
                end
                EoiToken===tok and lexerror tok, "eof in def header"
                result<<tok
              end until  parencount==0 #@parsestack.size==old_size
            else #no parentheses, all tail
              @last_operative_token=KeywordToken.new "." #hack hack
              tokindex=result.size
              result << tok=symbol(false,false)
              name=tok.to_s
              assert !in_lvar_define_state
     
              #maybe_local really means 'maybe local or constant'
              maybe_local=case name
                when /[^a-z_0-9]$/i; #do nothing
                when /^[@$]/; true
                when VARLIKE_KEYWORDS,FUNCLIKE_KEYWORDS; ty=KeywordToken
                when /^[a-z_]/;  localvars===name 
                when /^[A-Z]/; is_const=true  #this is the right algorithm for constants... 
              end
              if !ty and maybe_local 
                result.push(  *ignored_tokens(false,false)  )
                nc=nextchar
                if nc==?: || nc==?.
                  ty=VarNameToken
                end
              end  
              unless ty
                ty=MethNameToken
                endofs=tok.offset+tok.to_s.length
                result[tokindex+1...tokindex+1]=
                  [ImplicitParamListStartToken.new(endofs),ImplicitParamListEndToken.new(endofs)]
              end

              assert result[tokindex].equal?(tok)
              result[tokindex]=ty.new(tok.to_s,tok.offset)
              

              #if a.b.c.d is seen, a, b, and c
              #should be considered maybe varname instead of methnames.
              #the last (d in the example) is always considered a methname;
              #it's what's being defined.
              #b and c should be considered varnames only if 
              #they are capitalized and preceded by :: .
              #a could even be a keyword (eg self or block_given?).
            end
            #read tail: .b.c.d etc
            result.reverse_each{|res| break @last_operative_token=res unless StillIgnoreToken===res}
            ###@last_operative_token=result.last #naive
            assert !(IgnoreToken===@last_operative_token)
            state=:expect_op
            @in_def_name=true
            loop do

               #look for start of parameter list
               nc=(@moretokens.empty? ? nextchar.chr : @moretokens.first.to_s[0,1])
               if state==:expect_op and /^[a-z_(&*]/i===nc
                  result.concat def_param_list
                  break
               end

               tok=get1token
               result<<tok
               case tok
               when EoiToken
                  lexerror tok,'unexpected eof in def header'
               when StillIgnoreToken
               when MethNameToken ,VarNameToken # /^[a-z_]/i.token_pat
                  lexerror tok,'expected . or ::' unless state==:expect_name
                  state=:expect_op
               when /^(\.|::)$/.token_pat
                  lexerror tok,'expected ident' unless state==:expect_op
                  state=:expect_name
               when /^(;|end)$/.token_pat, NewlineToken #are we done with def name?
                  state==:expect_op or lexerror tok,'expected identifier'
                  break
               else
                  lexerror(tok, "bizarre token in def name: " +
                           "#{tok}:#{tok.class}")
               end
            end
            @in_def_name=false
         }
      when "alias"
         safe_recurse { |a|
            @last_operative_token=KeywordToken.new "alias" #hack
            result.concat ignored_tokens
            res=symbol(eat_next_if(?:),false) 
            res ? result<<res : lexerror(result.first,"bad symbol in alias")
            @last_operative_token=KeywordToken.new "alias" #hack
            result.concat ignored_tokens
            res=symbol(eat_next_if(?:),false) 
            res ? result<<res : lexerror(result.first,"bad symbol in alias")
         }
      when "undef"
         safe_recurse { |a|
            loop do
               @last_operative_token=KeywordToken.new "," #hack
               result.concat ignored_tokens
               tok=symbol(eat_next_if(?:),false)
               tok or lexerror(result.first,"bad symbol in undef")
               result<< tok
               @last_operative_token=tok
               assert !(IgnoreToken===@last_operative_token)

               sawnl=false
               result.concat ignored_tokens(true){|nl| sawnl=true}

               break if sawnl or nextchar != ?,
               tok= single_char_token(?,)
               result<< tok
            end
         }
         
#      when "defined?"
         #defined? might have a baresymbol following it
         #does it need to be handled specially?
         #it would seem not.....

      when "when"
         #abort_noparens! emits EndToken on leaving context
         result.unshift(*abort_noparens!(str))
         result.push KwParamListStartToken.new( offset+str.length)
         @parsestack.push WhenParamListContext.new(str,@linenum)

      when "rescue"
         unless after_nonid_op? {false}
           #rescue needs to be treated differently when in operator context... 
           #i think no RescueSMContext should be pushed on the stack...
           #plus, the rescue token should be marked as infix
           result.first.set_infix!
         else         
           result.push KwParamListStartToken.new(offset+str.length)
           #corresponding EndToken emitted by abort_noparens! on leaving rescue context
           result.unshift(*abort_noparens!(str))
           @parsestack.push RescueSMContext.new(@linenum)
         end

      when "then"
         result.unshift(*abort_noparens!(str))
         @parsestack.last.see self,:then
         
      when "in"
         result.unshift KwParamListEndToken.new( offset)
         result.unshift(*abort_noparens!(str))
         @parsestack.last.see self,:in
         
      when /\A(#{BINOPWORDS}|#{INNERBOUNDINGWORDS})\Z/o
         result.unshift(*abort_noparens!(str))

      when /\A(return|break|next)\Z/
         result=yield
         result.first.has_no_block! unless KeywordToken===@last_operative_token and @last_operative_token===/\A(\.|::)\Z/

      when FUNCLIKE_KEYWORDS
         result=yield

      when RUBYKEYWORDS
        #do nothing
        
      else result=yield

      end

      return result
   end


   #-----------------------------------
   def parsestack_lastnonassign_is?(obj)
     @parsestack.reverse_each{|ctx|
       case ctx
  #    when klass: return true
       when AssignmentRhsContext
       else return ctx.object_id==obj.object_id
       end
     }
   end

   #-----------------------------------
   #what's inside goalposts (the block formal parameter list) 
   #is considered the left hand side of an assignment.
   #inside goalposts, a local variable is declared if
   #it has one of the following tokens on both sides:
   #   ,  (if directly inside goalposts or nested lhs) 
   #   |  (as a goalpost)
   #   * or & (unary only)
   #   ( or ) (if they form a nested left hand side)
   #parens form a nested lhs if they're not part of an actual
   #parameter list and have a comma directly in them somewhere
   #a nested lhs _must_ have a comma in it somewhere. this is
   #not legal:
   #  (foo)=[1]
   #whereas this is:
   #  (foo,)=[1]


   
   #-----------------------------------
   def block_param_list_lookahead
      safe_recurse{ |la|
         @last_operative_token=KeywordToken.new  ';'
         a=ignored_tokens

         if eat_next_if(?|)
           a<<KeywordToken.new("|", input_position-1)
if true
           @parsestack.push mycontext=BlockParamListLhsContext.new(@linenum)
           nextchar==?| and a.push NoWsToken.new(input_position)
else
           if eat_next_if(?|)
             a.concat [NoWsToken.new(input_position-1),
                       KeywordToken.new('|', input_position-1)]
           else
             assert !@defining_lvar
             @defining_lvar=true
             assert((@last_operative_token===';' or NewlineToken===@last_operative_token))
             @parsestack.push mycontext=BlockParamListLhsContext.new(@linenum)
             #block param initializers ARE supported here, even tho ruby doesn't allow them!
             tok=nil
             loop do
               tok=get1token
               case tok
               when EoiToken; lexerror tok,"eof in block parameter list"
               when AssignmentRhsListStartToken; @defining_lvar=false
               when AssignmentRhsListEndToken; parsestack_lastnonassign_is?(mycontext) and @defining_lvar=true
               end
               
               tok==='|' and parsestack_lastnonassign_is?(mycontext) and break
               a<<tok
             end 
             assert@defining_lvar || AssignmentRhsContext===@parsestack.last
             @defining_lvar=false
             while AssignmentRhsContext===@parsestack.last 
               a.push( *abort_noparens!('|') )
             end
             
             @parsestack.last.object_id==mycontext.object_id or raise 'expected my BlockParamListLhsContext atop @parsestack'
             @parsestack.pop
             
             a<<KeywordToken.new('|',tok.offset)
             @moretokens.empty? or
               fixme %#moretokens might be set from get1token call above...might be bad#
end
           end
         end

         @last_operative_token=KeywordToken.new ';'
         #a.concat ignored_tokens

         #assert @last_operative_token===';'
         #a<<get1token

         la[0,0]=a
      }
   end

   #-----------------------------------
   #handle parameter list of a method declaration.
   #parentheses are optional... if missing param list
   #is ended by (unescaped) newline or semicolon (at the same bracing level)
   #expect a brace as the next token,
   #then match the following tokens until
   #the matching endbrace is found
   def def_param_list
      @in_def_name=false
      result=[]
      normal_comma_level=old_parsestack_size=@parsestack.size
      safe_recurse { |a|
         assert(@moretokens.empty?)
         assert((not IgnoreToken===@moretokens[0]))
         assert((@moretokens[0] or not nextchar.chr[WHSPCHARS]))

         #have parentheses?
         if '('==@moretokens[0] or nextchar==?(
            #get open paren token
            result.concat maybe_no_ws_token
            result << tok=get1token
            assert(tok==='(')


            #parsestack was changed by get1token above...
            normal_comma_level+=1
            assert(normal_comma_level==@parsestack.size)
            endingblock=proc{|tok| tok===')' }
         else
            endingblock=proc{|tok| tok===';' or NewlineToken===tok}
         end
         class << endingblock
            alias === call
         end

         @last_operative_token=KeywordToken.new ',' #hack
         #read local parameter names
         loop do
            expect_name=(@last_operative_token===',' and
                         normal_comma_level==@parsestack.size)
            expect_name and @defining_lvar||=true
            result << tok=get1token
            lexerror tok, "unexpected eof in def header" if EoiToken===tok

            #break if at end of param list
               endingblock===tok and 
                 old_parsestack_size>=@parsestack.size and break

            #next token is a local var name
            #(or the one after that if unary ops present)
            #result.concat ignored_tokens
            if expect_name 
              case tok
                when IgnoreToken #, /^[A-Z]/ #do nothing
                when /^,$/.token_pat #hack
               
               
                when VarNameToken
                  assert@defining_lvar
                  @defining_lvar=false
                  assert((not @last_operative_token===','))
                when /^[&*]$/.token_pat #unary form...
                  #a NoWsToken is also expected... read it now
                  result.concat maybe_no_ws_token #not needed?
                  @last_operative_token=KeywordToken.new ','
                else 
                  lexerror tok,"unfamiliar var name '#{tok}'"
              end
            elsif /^,$/.token_pat===tok and
                  normal_comma_level+1==@parsestack.size and
                  AssignmentRhsContext===@parsestack.last
              #seeing comma here should end implicit rhs started within the param list
              result[-1,0]=AssignmentRhsListEndToken.new(tok.offset)
              @parsestack.pop
            end
         end
         
         @defining_lvar=false
                   

         assert(@parsestack.size <= old_parsestack_size)
         assert(endingblock[tok])

         #hack: force next token to look like start of a
         #new stmt, if the last ignored_tokens
         #call above did not find a newline
         #(just in case the next token parsed
         #happens to call quote_expected? or after_nonid_op)
         result.concat ignored_tokens
         if  nextchar.chr[/[iuw\/<|>+\-*&%?:]/] and
             !(NewlineToken===@last_operative_token) and
             !(/^(end|;)$/===@last_operative_token)
           @last_operative_token=KeywordToken.new ';'
           result<< get1token
         end
      }

      return result
   end


   #-----------------------------------
   #handle % in ruby code. is it part of fancy quote or a modulo operator?
   def percent(ch)
      if quote_expected? ch
         fancy_quote ch
      else
         biop ch
      end
   end

   #-----------------------------------
   #handle * & in ruby code. is unary or binary operator?
   def star_or_amp(ch)
      assert('*&'[ch])
      want_unary=unary_op_expected? ch
      result=(quadriop ch)
      if want_unary
         #readahead(2)[1..1][/[\s\v#\\]/] or #not needed?
         assert OperatorToken===result
         result.unary=true         #result should distinguish unary+binary *&
         WHSPLF[nextchar.chr] or
           @moretokens << NoWsToken.new(input_position)
      end
      result
   end

   #-----------------------------------
   #handle ? in ruby code. is it part of ?..: or a character literal?
   def char_literal_or_op(ch)
      if colon_quote_expected? ch
         getchar
         NumberToken.new getchar_maybe_escape
      else
         @parsestack.push TernaryContext.new(@linenum)
         KeywordToken.new getchar   #operator
      end
   end

   #-----------------------------------
   def regex_or_div(ch) 
   #space after slash always means / operator, rather than regex start
      if after_nonid_op?{ !is_var_name? and WHSPLF[prevchar] and !readahead(2)[%r{^/\s}] }
        return regex(ch)
      else #/ is operator
        result=getchar
        if eat_next_if(?=)
          result << '='
        end
        return(operator_or_methname_token result)
      end
   end

   #-----------------------------------
   #return true if last tok corresponds to a variable or constant, 
   #false if its for a method, nil for something else
   #we assume it is a valid token with a correctly formed name.
   #...should really be called was_var_name
   def is_var_name?
     (tok=@last_operative_token)

     s=tok.to_s
     case s
     when /[^a-z_0-9]$/i; false
     when /^[a-z_]/; localvars===s or VARLIKE_KEYWORDS===s
     when /^[A-Z]/; VarNameToken===tok
     when /^[@$<]/; true
     else raise "not var or method name: #{s}"
     end   
   end
   
   #-----------------------------------
   def colon_quote_expected?(ch) #yukko hack
     assert ':?'[ch]
     readahead(2)[/^(\?[^#{WHSPLF}]|:[^\s\r\n\t\f\v :])$/o]   or return false

     after_nonid_op? {
       #possible func-call as operator

       not is_var_name? and
         if ch==':'
           not TernaryContext===@parsestack.last
         else
           !readahead(3)[/^\?[a-z0-9_]{2}/i]
         end
     }
   end

   #-----------------------------------
   def symbol_or_op(ch)
      startpos= input_position
      qe= colon_quote_expected?(ch)
      lastchar=prevchar
      eat_next_if(ch[0]) or raise "needed: "+ch

      #handle quoted symbols like  :"foobar",  :"[]"
      qe and return symbol(':')

      #look for another colon; return single : if not found
      unless eat_next_if(?:) 
        #cancel implicit contexts...
        @moretokens.push(*abort_noparens!(':'))
        
        #end ternary context, if any
        @parsestack.last.see self,:colon
        
        TernaryContext===@parsestack.last and @parsestack.pop #should be in the context's see handler
        
        if ExpectDoOrNlContext===@parsestack.last #should be in the context's see handler
          @parsestack.pop
          assert @parsestack.last.starter[/^(while|until|for)$/]
        end
        
        @moretokens.push KeywordToken.new(':',startpos)
        return @moretokens.shift
      end
      
      #we definately found a ::

      colon2=KeywordToken.new( '::',startpos)
      lasttok=@last_operative_token
      assert !(String===lasttok)
      if (VarNameToken===lasttok or MethNameToken===lasttok) and
          lasttok===/^[$@a-zA-Z_]/ and !WHSPCHARS[lastchar]
      then
         @moretokens << colon2
         result= NoWsToken.new(startpos)
      else
         result=colon2
      end
      dot_rhs(colon2)
      return result
   end

   #-----------------------------------
   def symbol(notbare,couldbecallsite=!notbare)
     assert !couldbecallsite
     start= input_position
     notbare and start-=1
     klass=(notbare ? SymbolToken : MethNameToken)
     
     #look for operators
     opmatches=readahead(3)[RUBYSYMOPERATORREX]
     result= opmatches ? read(opmatches.size) :
       case nc=nextchar
         when ?" then assert notbare;double_quote('"')
         when ?' then assert notbare;double_quote("'")
         when ?` then read(1)
         when ?@ then at_identifier.to_s
         when ?$ then dollar_identifier.to_s
         when ?_,?a..?z then identifier_as_string(?:)
         when ?A..?Z then 
           result=identifier_as_string(?:)
           if @last_operative_token==='::' 
             assert klass==MethNameToken
             /[A-Z_0-9]$/i===result and klass=VarNameToken
           end
           result
         else error= "unexpected char starting symbol: #{nc.chr}"
       end
     return lexerror(klass.new(result,start),error)
   end

   def merge_assignment_op_in_setter_callsites?
     false
   end
   #-----------------------------------
   def callsite_symbol(tok_to_errify)
     start= input_position
     
     #look for operators
     opmatches=readahead(3)[RUBYSYMOPERATORREX]
     return [opmatches ? read(opmatches.size) :
       case nc=nextchar
         when ?` then read(1)
         when ?_,?a..?z,?A..?Z then 
           context=merge_assignment_op_in_setter_callsites? ? ?: : nc
           identifier_as_string(context)
         else 
           @last_operative_token=KeywordToken.new(';')
           lexerror(tok_to_errify,"unexpected char starting callsite symbol: #{nc.chr}, tok=#{tok_to_errify.inspect}")
           nil
       end, start
      ]
   end

   #-----------------------------------
   def here_header
      read(2)=='<<' or raise "parser insanity"

      dash=eat_next_if(?-)
      quote=eat_next_if( /['"`]/)
      if quote
        ender=til_charset(/[#{quote}]/)
        (quote==getchar) or 
          return lexerror(HerePlaceholderToken.new( dash, quote, ender ), "mismatched quotes in here doc")
      else
        quote='"'
        ender=til_charset(/[^a-zA-Z0-9_]/)
        ender.length >= 1  or 
          return lexerror(HerePlaceholderToken.new( dash, quote, ender ), "invalid here header")
      end

      res= HerePlaceholderToken.new( dash, quote, ender )
      @incomplete_here_tokens.push res

      #hack: normally this should just be in get1token
      #this fixup is necessary because the call the get1token below
      #makes a recursion.
      @last_operative_token=res

      safe_recurse { |a|
         assert(a.object_id==@moretokens.object_id)
         toks=[]
         begin
           #yech. 
           #handle case of here header in a string inclusion, but
           #here body outside it.
           cnt=0
           1.upto @parsestack.size do |i|
             case @parsestack[-i]
               when AssignmentRhsContext,ParamListContextNoParen,TopLevelContext
               else cnt+=1
             end
           end
           if nextchar==?} and cnt==1
             res.bodyclass=OutlinedHereBodyToken
             break
           end
           
           tok=get1token
           assert(a.equal?( @moretokens))
           toks<<tok
           EoiToken===tok and lexerror tok, "here body expected before eof"
         end while res.unsafe_to_use
         assert(a.equal?( @moretokens))
         a[0,0]= toks   #same as a=toks+a, but keeps a's id
      }

      return res

      #the action continues in newline, where
      #the rest of the here token is read after a
      #newline has been seen and res.affix is eventually called
   end

   #-----------------------------------
   def lessthan(ch) #match quadriop('<') or here doc or spaceship op
      case readahead(3)
        when /^<<['"`\-a-z0-9_]$/i
           if quote_expected?(ch) #and @last_operative_token!='class' #not needed?
              here_header
           else
              operator_or_methname_token read(2)
           end
        when "<=>" then operator_or_methname_token read(3)
        else quadriop(ch)
      end
   end

   #-----------------------------------
   def escnewline(ch)
      assert ch == '\\'
      
      pos= input_position
      result=getchar
      if nl=readnl
        result+=nl
      else
        error='illegal escape sequence'
      end
      
      @moretokens.unshift FileAndLineToken.new(@filename,ln=@linenum,input_position)
      optional_here_bodies

      lexerror EscNlToken.new(@filename,ln-1,result,pos), error
   end
  
   #-----------------------------------
   def optional_here_bodies

      #handle here bodies queued up by previous line
      #(we should be more compatible with dos/mac style newlines...)
      while tofill=@incomplete_here_tokens.shift
         tofill.string.offset= input_position
         loop {
            assert("\r\n"[prevchar])

            #here body terminator?
            oldpos= input_position
            if tofill.dash
              til_charset(/[^#{WHSP}]/o)
            end
            break if eof?
            break if read(tofill.ender.size)==tofill.ender and readnl
            input_position_set oldpos
            
            if tofill.quote=="'" 
              line=til_charset(/[\r\n]/)+readnl
              line.gsub! "\\\\", "\\"
              tofill.append line
              assert(line[-1..-1][/[\r\n]/])
            else

              back1char  #-1 to make newline char the next to read
              @linenum-=1
  
              #retr evrything til next nl
              line=all_quote(INET_NL_REX, tofill.quote, INET_NL_REX)
              #(you didn't know all_quote could take a regex, did you?)
  
              #get rid of fals that otherwise appear to be in the middle of
              #a string (and are emitted out of order)
              fal=@moretokens.pop
              assert FileAndLineToken===fal || fal.nil?
  
              back1char
              @linenum-=1
              assert("\r\n"[nextchar.chr])
              tofill.append_token line
              tofill.append readnl
            end
         }

         assert(eof?  || "\r\n"[prevchar])
         tofill.unsafe_to_use=false
         tofill.line=@linenum-1

         @moretokens.push \
           tofill.bodyclass.new(tofill), 
           FileAndLineToken.new(@filename,@linenum,input_position)
      end
 
   end

   #-----------------------------------
   def newline(ch)
      assert("\r\n"[nextchar.chr])

      

      #ordinary newline handling (possibly implicitly escaped)
      assert("\r\n"[nextchar.chr])
                   assert !@parsestack.empty?
      assert @moretokens.empty?
      result=if NewlineToken===@last_operative_token or #hack
                @last_operative_token===/^(;|begin|do|#{INNERBOUNDINGWORDS})$/ or #hack
                !after_nonid_op?{false}
             then   #hack-o-rama: probly cases left out above
                   a= abort_noparens!
                   ExpectDoOrNlContext===@parsestack.last and @parsestack.pop
                   assert !@parsestack.empty?
                   @parsestack.last.see self,:semi

                   a << super(ch)
                   @moretokens.replace a+@moretokens
                   @moretokens.shift
             else
                   offset= input_position
                   nl=readnl
                   @moretokens << FileAndLineToken.new(@filename,@linenum,input_position)
                   EscNlToken.new(@filename,@linenum-1,nl,offset)
                   #WsToken.new ' ' #why?  #should be "\\\n" ?
             end

      optional_here_bodies

      start_of_line_directives

      return result
   end

   #-----------------------------------
   EQBEGIN=%r/^=begin[ \t\v\r\n\f]$/
   EQBEGINLENGTH=7
   EQEND='=end'
   EQENDLENGTH=4
   ENDMARKER=/^__END__[\r\n]?\Z/
   ENDMARKERLENGTH=8
   def start_of_line_directives
      #handle =begin...=end (at start of a line)
      while EQBEGIN===readahead(EQBEGINLENGTH)
         startpos= input_position
         more= read(EQBEGINLENGTH-1)   #get =begin

         begin
           eof? and raise "eof before =end"
           more<<til_charset(/[\r\n]/)
           more<<readnl
         end until readahead(EQENDLENGTH)==EQEND

         #read rest of line after =end
         more << til_charset(/[\r\n]/)  
         assert((?\r===nextchar or ?\n===nextchar))
         assert !(/[\r\n]/===more[-1,1])
         more<< readnl

#         newls= more.scan(/\r\n?|\n\r?/)
#         @linenum+= newls.size

         #inject the fresh comment into future token results
         @moretokens.push IgnoreToken.new(more,startpos),
                          FileAndLineToken.new(@filename,@linenum,input_position)
      end

      #handle __END__
      if ENDMARKER===readahead(ENDMARKERLENGTH)
         assert !(ImplicitContext===@parsestack.last)
         @moretokens.unshift endoffile_detected(read(7))
#         input_position_set @file.size
      end
   end



  #-----------------------------------
  #used to resolve the ambiguity of
  # unary ops (+, -, *, &, ~ !) in ruby
  #returns whether current token is to be the start of a literal
  IDBEGINCHAR=/^[a-zA-Z_$@]/
  def unary_op_expected?(ch) #yukko hack
    '*&='[readahead(2)[1..1]] and return false

    after_nonid_op? {
      #possible func-call as operator

      not is_var_name? and
        WHSPLF[prevchar]
    }
  end

   #-----------------------------------
   #used to resolve the ambiguity of
   # <<, %, ? in ruby
   #returns whether current token is to be the start of a literal
   def quote_expected?(ch) #yukko hack
     if AssignmentContext===@parsestack.last
       @parsestack.pop
       return false
     end

     case ch[0]
          when ?? then readahead(2)[/^\?[#{WHSPLF}]$/o] #not needed?
          when ?% then readahead(3)[/^%([a-pt-vyzA-PR-VX-Z]|[QqrswWx][a-zA-Z0-9])/]
          when ?< then !readahead(4)[/^<<-?['"`a-z0-9_]/i]
          else raise 'unexpected ch (#{ch}) in quote_expected?'
     #     when ?+,?-,?&,?*,?~,?! then '*&='[readahead(2)[1..1]]
     end and return false

     after_nonid_op? {
       #possible func-call as operator

       not is_var_name? and
         WHSPLF[prevchar] and not WHSPLF[readahead(2)[1..1]]
     }
   end

   #-----------------------------------
   #returns false if last token was an value, true if it was an operator.
   #returns what block yields if last token was a method name.
   #used to resolve the ambiguity of
   # <<, %, /, ?, :, and newline (among others) in ruby
   def after_nonid_op?
      case @last_operative_token
         when MethNameToken, FUNCLIKE_KEYWORDS.token_pat ,VarNameToken
         #VarNameToken should really be left out of this case... 
         #should be in next branch instread
         #callers all check for last token being not a variable if they pass anything
         #but {false} in the block
            return yield
         when StringToken, SymbolToken, NumberToken, HerePlaceholderToken,
              %r{^(
                class|module|end|self|true|false|nil|
                __FILE__|__LINE__|[\})\]]|alias|(un)?def|for
              )$}x.token_pat
            #dunno about def/undef
            #maybe class/module shouldn't he here either?  
            #for is also in NewlineToken branch, below.
            #what about rescue?
            return false
         when /^(#{RUBYOPERATORREX}|#{INNERBOUNDINGWORDS}|do)$/o.token_pat
            #regexs above must match whole string
            #assert(@last_operative_token==$&) #disabled 'cause $& is now always nil :(
            return true
         when NewlineToken, nil,   #nil means we're still at beginning of file
              /^([({\[]|or|not|and|if|unless|then|elsif|else|
                 while|until|begin|for|in|case|when|ensure)$
              /x.token_pat
            return true
         #when KeywordToken
         #   return true
         when IgnoreToken
            raise "last_operative_token shouldn't be ignoreable"
         else
            raise "after_nonid_op? after #{@last_operative_token}:#{@last_operative_token.class} -- now what"
      end
   end




   #-----------------------------------
   #returns the last context on @parsestack which isn't an ImplicitContext
   def last_context_not_implicit
     @parsestack.reverse_each{|ctx|
       return ctx unless ImplicitContext===ctx
     }
     fail
   end

   #-----------------------------------
   #a | has been seen. is it an operator? or a goalpost?
   #(goalpost == delimiter of block param list)
   #if it is a goalpost, end the BlockParamListLhsContext on
   #the context stack, as well as any implicit contexts on top of it.
   def conjunction_or_goalpost(ch)
     result=quadriop(ch)
     if result===/^|$/ and BlockParamListLhsContext===last_context_not_implicit 
       @moretokens.push( *abort_noparens!("|"))
       assert(BlockParamListLhsContext===@parsestack.last)
       @parsestack.pop
       @moretokens.push KeywordToken.new("|", input_position-1)
       result=@moretokens.shift
     end
     result
   end
   
   #-----------------------------------
   def quadriop(ch) #match /&&?=?/ (&, &&, &=, or &&=)
      assert(%w[& * | < >].include?(ch))
      result=getchar + (eat_next_if(ch)or'')
      if eat_next_if(?=)
         result << ?=
      end
      return operator_or_methname_token(result)
   end


   #-----------------------------------
   def biop(ch) #match /%=?/ (% or %=)
      assert(ch[/^[%^~]$/])
      result=getchar
      if eat_next_if(?=)
         result <<?=
      end
      return operator_or_methname_token( result)
   end
   #-----------------------------------
   def tilde(ch) #match ~
      assert(ch=='~')
      result=getchar
#      eat_next_if(?=) ?   #ack, spppft, I'm always getting this backwards
#         result <<?= :
         WHSPLF[nextchar.chr] ||
           @moretokens << NoWsToken.new(input_position)
      #why is the NoWsToken necessary at this point?    
      result=operator_or_methname_token result
      result.unary=true         #result should distinguish unary ~
      result
   end

   #-----------------------------------
   def want_op_name
      KeywordToken===@last_operative_token and
         @last_operative_token===/^(alias|(un)?def|\.|::)$/
   end

   #-----------------------------------
   #match /[+\-]=?/ (+ or +=)
   #could be beginning of number, too
   #fixme: handle +@ and -@ here as well... (currently, this is done in symbol()?)
   def plusminus(ch)
      assert(/^[+\-]$/===ch)
      if unary_op_expected?(ch)
        if (?0..?9)===readahead(2)[1]
          return number(ch)
        else #unary operator
          result=getchar
          WHSPLF[nextchar.chr] or
            @moretokens << NoWsToken.new(input_position)
          result=(operator_or_methname_token result)
          result.unary=true
          #todo: result should distinguish unary+binary +-
        end
      else #binary operator
         assert(! want_op_name)
         result=getchar
         if eat_next_if(?=)
            result << ?=
         end
         result=(operator_or_methname_token result)
         #todo: result should distinguish unary+binary +-
      end
      result
   end

   #-----------------------------------
   def equals(ch) #match /=(>|~|==?)?/ (= or == or =~ or === or =>)
      offset= input_position
      str=getchar
      assert str=='='
      c=(eat_next_if(/[~=>]/)or'')
      str << c
      result= operator_or_methname_token( str,offset)
      case c
      when '=': str<< (eat_next_if(?=)or'')
      
      when '>': 
        unless ParamListContextNoParen===@parsestack.last
          @moretokens.unshift result
          @moretokens.unshift( *abort_noparens!("=>"))
          result=@moretokens.shift
        end
        @parsestack.last.see self,:arrow
      when '': #record local variable definitions

        @parsestack.push AssignmentRhsContext.new(@linenum)
        @moretokens.unshift AssignmentRhsListStartToken.new( offset+1)
      end
      return result
   end

   #-----------------------------------
   def exclam(ch) #match /![~=]?/ (! or != or !~)
      assert nextchar==?!
      result=getchar
      k=eat_next_if(/[~=]/)
      if k
        result+=k
      else
        WHSPLF[nextchar.chr] or
          @moretokens << NoWsToken.new(input_position)
      end
      return KeywordToken.new(result, input_position-result.size)
      #result should distinguish unary !
   end


   #-----------------------------------
   def dot(ch)
      str=''
      eat_next_if(?.) or raise "lexer confusion"

      #three lumps of sugar or two?
      eat_next_if(?.) and
         return KeywordToken.new(eat_next_if(?.)? "..." : "..")

      #else saw just single .
      #match a valid ruby id after the dot
      result= KeywordToken.new( ".")
      dot_rhs(result)
      return result
   end
   #-----------------------------------
   def dot_rhs(prevtok)
      safe_recurse { |a|
         @last_operative_token=prevtok
         aa= ignored_tokens
         tok,pos=callsite_symbol(prevtok)
         tok and aa.push(*var_or_meth_name(tok,prevtok,pos)) 
         a.unshift(*aa)
      }     
   end

  #-----------------------------------
  def back_quote(ch=nil)
    if @last_operative_token===/^(def|::|\.)$/
      oldpos= input_position
      MethNameToken.new(eat_next_if(?`), oldpos)
    else
      double_quote(ch)
    end
  end

if false
   #-----------------------------------
   def comment(str)
     result=""
     #loop{
       result<<super(nil).to_s

       if /^\#.*\#$/===result #if comment was ended by a crunch

         #that's not a legal comment end in ruby, so just keep reading
         assert(result.to_s[-1]==?#)
         result.chomp! '#'

         #back up one char in input so that the
         #super will see that # on the next go round.
         #this hack makes the ruma comment lexer work with ruby too.
         back1char

         assert nextchar==?#
       #else break #not a crunch... just exit
       end
     #}

     return IgnoreToken.new(result)
   end
end
   #-----------------------------------
   def open_brace(ch)
      assert((ch!='[' or !want_op_name))
      assert(@moretokens.empty?)
      lastchar=prevchar
      ch=eat_next_if(/[({\[]/)or raise "lexer confusion"
      tokch=KeywordToken.new(ch, input_position-1)
      

      #maybe emitting of NoWsToken can be moved into var_or_meth_name ??
      case tokch.ident
      when '['
        # in contexts expecting an (operator) method name, we 
        #       would want to match [] or []= at this point
        #but control never comes this way in those cases... goes 
        #to custom parsers for alias, undef, and def in #parse_keywords
        tokch.set_infix! unless after_nonid_op?{WHSPLF[lastchar]}
        @parsestack.push ListImmedContext.new(ch,@linenum)
        lasttok=last_operative_token
        #could be: lasttok===/^[a-z_]/i
        if (VarNameToken===lasttok or ImplicitParamListEndToken===lasttok or MethNameToken===lasttok) and !WHSPCHARS[lastchar]
               @moretokens << (tokch)
               tokch= NoWsToken.new(input_position-1)
        end
      when '('
        lasttok=last_operative_token
        #could be: lasttok===/^[a-z_]/i
        if (VarNameToken===lasttok or MethNameToken===lasttok or
            lasttok===FUNCLIKE_KEYWORDS)
          unless WHSPCHARS[lastchar]
               @moretokens << tokch
               tokch= NoWsToken.new(input_position-1)
          end
          @parsestack.push ParamListContext.new(@linenum)
        else
          @parsestack.push ParenContext.new(@linenum)
        end

      when '{'
      #check if we are in a hash literal or string inclusion (#{}),
      #in which case below would be bad.
      if after_nonid_op?{false} or @last_operative_token.has_no_block?
        @parsestack.push ListImmedContext.new(ch,@linenum) #that is, a hash
      else
        tokch.set_infix!
=begin not needed now, i think
        # 'need to find matching callsite context and end it if implicit'
        lasttok=last_operative_token
        unless lasttok===')' and lasttok.callsite?
          @moretokens.push *(abort_1_noparen!(1).push tokch)
          tokch=@moretokens.shift
        end
=end

        localvars.start_block
        @parsestack.push BlockContext.new(@linenum)
        block_param_list_lookahead
      end
      end
      return (tokch)
   end

   #-----------------------------------
   def close_brace(ch)
      ch==eat_next_if(/[)}\]]/) or raise "lexer confusion"
      @moretokens.concat abort_noparens!(ch)
      @parsestack.last.see self,:semi #hack
      @moretokens<< kw=KeywordToken.new( ch, input_position-1)
      if @parsestack.empty? 
        lexerror kw,"unmatched brace: #{ch}"
        return @moretokens.shift
      end
      ctx=@parsestack.pop
      origch,line=ctx.starter,ctx.linenum
      ch==PAIRS[origch] or
        lexerror kw,"mismatched braces: #{origch}#{ch}\n" +
                 "matching brace location", @filename, line
      BlockContext===ctx and localvars.end_block
      if ParamListContext==ctx.class
        assert ch==')'
        #kw.set_callsite! #not needed?
      end
      return @moretokens.shift
   end

   #-----------------------------------
   def eof(ch=nil)
     #this must be the very last character...
     oldpos= input_position
     assert(?\0==getc)

     result= "\0#{ignored_tokens(true).delete_if{|t|FileAndLineToken===t}}"

     eof? or
        lexerror result,'nul character is not at the end of file'
     input_position_set @file.size
     return(endoffile_detected result)
   end

   #-----------------------------------
   def endoffile_detected(s='')
     @moretokens.push( *(abort_noparens!.push super(s)))
     result= @moretokens.shift
     balanced_braces? or (lexerror result,"unbalanced braces at eof. parsestack=#{@parsestack.inspect}")
     result
   end

  #-----------------------------------
  def single_char_token(ch)
    KeywordToken.new super(ch), input_position-1
  end

  #-----------------------------------
  def comma(ch)
    single_char_token(ch)
  end
  
  #-----------------------------------
  def semicolon(ch)
    assert @moretokens.empty?
    @moretokens.push(*abort_noparens!)
    @parsestack.last.see self,:semi
    if ExpectDoOrNlContext===@parsestack.last #should be in context's see:semi handler
      @parsestack.pop
      assert @parsestack.last.starter[/^(while|until|for)$/]
    end
    @moretokens.push single_char_token(ch)
    return @moretokens.shift
  end

  #-----------------------------------
  def operator_or_methname_token(s,offset=nil)
    assert RUBYOPERATORREX===s
    if RUBYNONSYMOPERATORREX===s
      KeywordToken
    elsif @last_operative_token===/^(\.|::|def|undef|alias|defined\?)$/
      MethNameToken
    else 
      OperatorToken
    end.new(s,offset)
  end

  #-----------------------------------
  #tokenify_results_of  :identifier
  save_offsets_in(*CHARMAPPINGS.values.uniq-[
    :symbol_or_op,:open_brace,:whitespace,:exclam,:backquote
  
  
  ])
  #save_offsets_in :symbol

end