#!/usr/bin/ruby -dw
require "rubylexer"
#  require "rumalexer"
require "token"
require "tokenprinter"
require "getoptlong"
require "pp"

class Token
  def verify_offset(fd); false end
  
  def check_for_error; end
end

module ErrorToken
  def check_for_error; raise @error end
end

class FileAndLineToken
  def verify_offset(fd); true  end
end

module SimpleVerify
  def verify_offset(fd)
    fd.read(@ident.length)==@ident
  end
end

class WToken;      include SimpleVerify; end
class NewlineToken;     include SimpleVerify; end
class IgnoreToken; include SimpleVerify; end
class MethNameToken; include SimpleVerify; end

class SymbolToken
  def verify_offset(fd)
    readsym=fd.read(@ident.length)
    @ident[1]==?" or @ident[1]==?' or readsym==@ident

  end
end

class EoiToken
  def verify_offset(fd)
    result=super(fd)
    fd.pos=fd.stat.size
    return result
  end
end

class NoWsToken
  def verify_offset(fd)
    orig=fd.pos
    fd.pos=orig-1
    result= (/^[^\s\v\t\n\r\f]{2}$/===fd.read(2))
    fd.pos=orig
    return result
  end
end

class HereBodyToken
  def verify_offset(fd)
    @ident.verify_subtoken_offsets(fd)
  end
end

class HerePlaceholderToken
  def verify_offset(fd)
    '<<'==fd.read(2) or return false
    @dash and (?-==fd.getc or return false)
    case ch=fd.getc
      when ?', ?`, ?"
        @quote==ch.chr or return false
        fd.read(@ender.size)==@ender or return false
        return fd.getc.chr==@quote
      when ?a..?z, ?A..?Z, ?_, ?0..?9
        @quote=='"' or return false
        fd.pos-=1
        fd.read(@ender.size)==@ender or return false
      else
        return false
    end
  end
end

class StringToken
  FANCY_QUOTE_BEGINNINGS= {'`'=>'%x', '['=>'%w', '{'=>'%W',
                           '"'=>/('|%[^a-pr-z0-9])/i, '/'=>'%r'}
  def verify_offset(fd)
    str=fd.read(2)
    @char==str[0,1] or FANCY_QUOTE_BEGINNINGS[@char]===str or return false
    verify_subtoken_offsets(fd)
  end

  def verify_subtoken_offsets(fd)
    #verify offsets of subtokens
    1.step(@elems.length-1,2) { |i| @elems[i].verify_offset(fd) }
    return true
  end
  
  def check_for_error
    1.step(@elems.size-1,2){|idx|
      @elems[idx].check_for_error
    }
    super
  end
end

class RubyCode
  def verify_offset(fd)
    thistok=nexttok=endpos=nil
    @ident.each_index{ |tok_i|
      thistok,nexttok=@ident[tok_i,2]
      endpos=nexttok ? nexttok.offset : thistok.offset+thistok.to_s.size
      check_offset(thistok,fd,endpos)
    }
    assert nexttok.nil?
    assert thistok.object_id==@ident.last.object_id
    assert WToken===thistok
    fd.pos=endpos
  end
  
  def check_for_error
    @ident.each{|tok| tok.check_for_error }
  end
end


class NumberToken
  def verify_offset(fd)
    /^[0-9?+-]$/===fd.read(1)
  end
end


#class ZwToken
#  def to_s
#    $ShowImplicit ? explicit_form : super
#  end
#end

class RuLexer
  def check_offset(tok,file=@file)
    endpos=(@moretokens.empty?)? file.pos : @moretokens[0].offset
    super(tok,file,endpos)
  end
end

def check_offset(tok,file,endpos)
    oldpos=file.pos

    assert Integer===tok.offset
    assert Integer===endpos
    assert endpos>=tok.offset

    file.pos=tok.offset
    assert tok.verify_offset(file)
    case tok
      when StringToken,NumberToken,HereBodyToken
      else assert(file.pos==endpos)
    end
    file.pos=oldpos
end






def tokentest(name,lexertype,pprinter,input=File.open(name),output=nil)
  input ||= File.open(name)
  if output
    old_stdout=$stdout
    $stdout=File.open(output,'w')
  end
  File.open(name) {|fd|
    lxr=lexertype.new(name,fd,1)

    begin
      tok=lxr.get1token
      lxr.check_offset(tok)
      tok.check_for_error
      pprinter.pprint(tok)
    end until EoiToken===tok

    #hack for SimpleTokenPrinter....
    print "\n" if NewlineToken===lxr.last_operative_token and
                  SimpleTokenPrinter===pprinter

#    unless lxr.balanced_braces? 
#      raise "unbalanced braces at eof"
#    end
  }
  if output
    $stdout.close
    $stdout=old_stdout
  end
end

#$ShowImplicit=false
if __FILE__==$0

  sep,line,showzw='',1,0
#  lexertype= RumaLexer if defined? RumaLexer
  lexertype=RubyLexer
  insertnils=fd=name=nil
  pprinter=SimpleTokenPrinter

  opts=GetoptLong.new \
    ["--eval","-e", GetoptLong::REQUIRED_ARGUMENT],
#    ["--ruby","-r", GetoptLong::NO_ARGUMENT],
    ["--keepws","-k", GetoptLong::NO_ARGUMENT],
    ["--maxws","-m", GetoptLong::NO_ARGUMENT],
    ["--implicit","-i", GetoptLong::NO_ARGUMENT],
    ["--implicit-all", GetoptLong::NO_ARGUMENT]

  opts.each do|opt,arg|
    case opt
    when '--eval'   then 
       pprinter =pprinter.new(sep,line,showzw)
       tokentest('-e',lexertype,pprinter,arg)
       saweval=true
#    when '--ruby'   then lexertype=RubyLexer
    when '--keepws' then pprinter= KeepWsTokenPrinter
    when '--maxws'  then pprinter= KeepWsTokenPrinter;sep=' '
    when '--implicit' then showzw=1
    when '--implicit-all' then showzw=2
    else raise :impossible
    end
  end

  pprinter =pprinter.new(sep,line,showzw)
  
    ARGV.empty?   ?    saweval || tokentest('-',lexertype,pprinter,$stdin) : 
        ARGV.each{|fn| tokentest(fn,lexertype,pprinter) } 
#  ARGV.first[/[_.]rb$/i] and lexertype=RubyLexer  #filename with _rb are special hack

end