#!/usr/bin/ruby
$Debug=true
require "rubylexer"
require "getoptlong"
require "pp"

class RubyLexer
class Token
  def verify_offset(fd); false end
  
  def check_for_error; end
end

class LexerError<Exception; end

module ErrorToken
  def check_for_error; raise LexerError,@error end
end

class FileAndLineToken
  def verify_offset(fd); true  end
end
class ImplicitParamListStartToken
  def verify_offset(fd); true  end
end
class ImplicitParamListEndToken
  def verify_offset(fd); true  end
end

module SimpleVerify
  def verify_offset(fd)
    fd.read(@ident.length)==@ident
  end
end

class WToken;      include SimpleVerify; end
class NewlineToken;     include SimpleVerify; end
class IgnoreToken; include SimpleVerify; end
class MethNameToken; include SimpleVerify; end

class SymbolToken
  def verify_offset(fd)
    la=fd.read(2)
    case la
      when '%s': #stay right here
      when /^:/: fd.pos-=1
      else raise 'unrecognized symbol type'
    end
    @ident[1]==?" or @ident[1]==?' or fd.read(@ident.length-1)==@ident[1..-1]

  end
end

class EoiToken
  def verify_offset(fd)
    result=super(fd)
    fd.eof?
    return result
  end
end

class NoWsToken
  def verify_offset(fd)
    orig=fd.pos
    fd.pos=orig-1
    result= (/^[^\s\v\t\n\r\f]{2}$/===fd.read(2))
    fd.pos=orig
    return result
  end
end

class HereBodyToken
  def verify_offset(fd)
    @ident.verify_subtoken_offsets(fd)
  end
end

class HerePlaceholderToken
  def verify_offset(fd)
    '<<'==fd.read(2) or return false
    @dash and (?-==fd.read1 or return false)
    case ch=fd.read1
      when ?', ?`, ?"
        @quote==ch.chr and
        fd.read(@ender.size)==@ender and
        return fd.read(1)==@quote
      when ?a..?z, ?A..?Z, ?_, ?0..?9
        @quote=='"' or return false
        fd.pos-=1
        fd.read(@ender.size)==@ender or return false
      else
        return false
    end
  end
end

class StringToken
  FANCY_QUOTE_BEGINNINGS= {'`'=>'%x', '['=>'%w', '{'=>'%W',
                           '"'=>/('|%[^a-pr-z0-9])/i, '/'=>'%r'}
  def verify_offset(fd)
    str=fd.read(2)
    @char==str[0,1] or FANCY_QUOTE_BEGINNINGS[@char]===str or return false
    verify_subtoken_offsets(fd)
  end

  def verify_subtoken_offsets(fd)
    #verify offsets of subtokens
    1.step(@elems.length-1,2) { |i| @elems[i].verify_offset(fd) or raise LexerError}
    return true
  end
  
  def check_for_error
    1.step(@elems.size-1,2){|idx|
      @elems[idx].check_for_error
    }
    super
  end
end

class RubyCode
  def verify_offset(fd)
    thistok=nexttok=endpos=nil
    @ident.each_index{ |tok_i|
      thistok,nexttok=@ident[tok_i,2]
      endpos=nexttok ? nexttok.offset : thistok.offset+thistok.to_s.size
      check_offset(thistok,fd,endpos)
    }
    assert nexttok.nil?
    assert thistok.object_id==@ident.last.object_id
    assert WToken===thistok
    fd.pos=endpos
  end
  
  def check_for_error
    @ident.each{|tok| tok.check_for_error }
  end
end


class NumberToken
  def verify_offset(fd)
    /^[0-9?+-]$/===fd.read(1)
  end
end


#class ZwToken
#  def to_s
#    $ShowImplicit ? explicit_form : super
#  end
#end
end
public

def check_offset(tok,file=nil,endpos=nil)
  file||=@file
  endpos||=(@moretokens.empty?)? file.pos : @moretokens[0].offset
  oldpos=file.pos

  assert Integer===tok.offset
  assert Integer===endpos
  endpos>=tok.offset or 
    raise RubyLexer::LexerError, "expected >=#{tok.offset}, got #{endpos}, "\
                      "token #{tok}:#{tok.class}"

  file.pos=tok.offset
  tok.verify_offset(file) or raise RubyLexer::LexerError, "couldn't check offset of token #{tok.class}: #{tok}"
  case tok
    when RubyLexer::StringToken,RubyLexer::NumberToken,
         RubyLexer::HereBodyToken,RubyLexer::SymbolToken: #do nothing
    else (file.pos==endpos) or raise RubyLexer::LexerError, "positions don't line up, expected #{endpos}, got #{file.pos}, token: #{tok}"
  end
  file.pos=oldpos
end






def tokentest(name,lexertype,pprinter,input=File.open(name),output=$stdout)
  input ||= File.open(name)
  if output!=$stdout
    output=File.open(output,'w')
  end

  fd=input
  #File.open(name) {|fd|
    lxr=lexertype.new(name,fd,1)

    begin
      tok=lxr.get1token
      lxr.check_offset(tok)
      tok.check_for_error
      pprinter.pprint(tok,output)
    end until RubyLexer::EoiToken===tok

    #hack for SimpleTokenPrinter....
    print "\n" if RubyLexer::NewlineToken===lxr.last_operative_token and
                  RubyLexer::SimpleTokenPrinter===pprinter

#    unless lxr.balanced_braces? 
#      raise "unbalanced braces at eof"
#    end
  #}
   output.close unless output==$stdout

end

#$ShowImplicit=false
if __FILE__==$0

  sep,line,showzw='',1,0
#  lexertype= RumaLexer if defined? RumaLexer
  lexertype=RubyLexer
  insertnils=fd=name=nil
  pprinter=RubyLexer::SimpleTokenPrinter

  opts=GetoptLong.new \
    ["--eval","-e", GetoptLong::REQUIRED_ARGUMENT],
#    ["--ruby","-r", GetoptLong::NO_ARGUMENT],
    ["--keepws","-k", GetoptLong::NO_ARGUMENT],
    ["--maxws","-m", GetoptLong::NO_ARGUMENT],
    ["--implicit","-i", GetoptLong::NO_ARGUMENT],
    ["--implicit-all", GetoptLong::NO_ARGUMENT]

  saweval=nil
  opts.each do|opt,arg|
    case opt
    when '--eval'   then 
       tokentest('-e',lexertype,pprinter.new(sep,line,showzw),arg)
       saweval=true
#    when '--ruby'   then lexertype=RubyLexer
    when '--keepws' then pprinter= RubyLexer::KeepWsTokenPrinter
    when '--maxws'  then pprinter= RubyLexer::KeepWsTokenPrinter;sep=' '
    when '--implicit' then showzw=1
    when '--implicit-all' then showzw=2
    else raise :impossible
    end
  end

  pprinter =pprinter.new(sep,line,showzw)
  
    ARGV.empty?   ?    saweval || tokentest('-',lexertype,pprinter,$stdin) : 
        ARGV.each{|fn| tokentest(fn,lexertype,pprinter) } 
#  ARGV.first[/[_.]rb$/i] and lexertype=RubyLexer  #filename with _rb are special hack

end