$:.unshift "../../lib"

require 'test/unit'
require 'syntax/ruby'

class TC_Syntax_Ruby < Test::Unit::TestCase

  def setup
    @ruby = Syntax::Ruby.new
  end

  def test_empty
    called = false
    @ruby.tokenize( "" ) { |tok| called = true }
    assert !called
  end

  def test_constant
    called = false
    @ruby.tokenize( "Foo" ) do |tok|
      called = true
      assert_equal :constant, tok.group
      assert_equal "Foo", tok
    end
    assert called
  end

  def test_ident
    called = false
    @ruby.tokenize( "foo" ) do |tok|
      called = true
      assert_equal :ident, tok.group
      assert_equal "foo", tok
    end
    assert called
  end

  def test_comment_eol
    called = false
    @ruby.tokenize( "# a comment" ) do |tok|
      called = true
      assert_equal :comment, tok.group
      assert_equal "# a comment", tok
    end
    assert called
  end

  def test_comment_block
    called = false
    @ruby.tokenize( "=begin\nthis is a comment\n=end" ) do |tok|
      called = true
      assert_equal :comment, tok.group
      assert_equal "=begin\nthis is a comment\n=end", tok
    end
    assert called
  end

  def test_keyword
    Syntax::Ruby::KEYWORDS.each do |word|
      tok = []
      @ruby.tokenize( word ) { |t| tok << t }
      assert_equal [ :keyword, word ], [ tok.first.group, tok.first ]
    end
    Syntax::Ruby::KEYWORDS.each do |word|
      tok = []
      @ruby.tokenize( "foo.#{word}" ) { |t| tok << t }
      tok.shift
      tok.shift
      assert_equal [ :ident, word ], [ tok.first.group, tok.first ]
    end
  end

  def test__END__
    called = false
    @ruby.tokenize( "__END__\n\nblah blah blah" ) do |tok|
      called = true
      assert_equal :comment, tok.group
      assert_equal "__END__\n\nblah blah blah", tok
    end
    assert called
  end

  def test_def_paren
    tok = nil
    @ruby.start( "def  foo(bar)" ) { |t| tok = t }
    @ruby.step
    assert_equal "def  ", tok
    assert_equal :keyword, tok.group
    @ruby.step
    assert_equal "foo", tok
    assert_equal :method, tok.group
  end

  def test_def_space
    tok = nil
    @ruby.start( "def  foo bar" ) { |t| tok = t }
    @ruby.step
    assert_equal "def  ", tok
    assert_equal :keyword, tok.group
    @ruby.step
    assert_equal "foo", tok
    assert_equal :method, tok.group
  end

  def test_def_semicolon
    tok = nil
    @ruby.start( "def  foo;" ) { |t| tok = t }
    @ruby.step
    assert_equal "def  ", tok
    assert_equal :keyword, tok.group
    @ruby.step
    assert_equal "foo", tok
    assert_equal :method, tok.group
  end

  def test_class_space
    tok = nil
    @ruby.start( "class    Foo\n" ) { |t| tok = t }
    @ruby.step
    assert_equal "class    ", tok
    assert_equal :keyword, tok.group
    @ruby.step
    assert_equal "Foo", tok
    assert_equal :class, tok.group
  end

  def test_class_semicolon
    tok = nil
    @ruby.start( "class    Foo;" ) { |t| tok = t }
    @ruby.step
    assert_equal "class    ", tok
    assert_equal :keyword, tok.group
    @ruby.step
    assert_equal "Foo", tok
    assert_equal :class, tok.group
  end

  def test_class_extend
    tok = nil
    @ruby.start( "class    Foo< Bang" ) { |t| tok = t }
    @ruby.step
    assert_equal "class    ", tok
    assert_equal :keyword, tok.group
    @ruby.step
    assert_equal "Foo", tok
    assert_equal :class, tok.group
  end

  def test_module_space
    tok = nil
    @ruby.start( "module    Foo\n" ) { |t| tok = t }
    @ruby.step
    assert_equal "module    ", tok
    assert_equal :keyword, tok.group
    @ruby.step
    assert_equal "Foo", tok
    assert_equal :module, tok.group
  end

  def test_module_semicolon
    tok = nil
    @ruby.start( "module    Foo;" ) { |t| tok = t }
    @ruby.step
    assert_equal "module    ", tok
    assert_equal :keyword, tok.group
    @ruby.step
    assert_equal "Foo", tok
    assert_equal :module, tok.group
  end

  def test_module_other
    tok = nil
    @ruby.start( "module    Foo!\n" ) { |t| tok = t }
    @ruby.step
    assert_equal "module    ", tok
    assert_equal :keyword, tok.group
    @ruby.step
    assert_equal "Foo!", tok
    assert_equal :module, tok.group
  end

  def test_scope_operator
    tok = []
    @ruby.tokenize( "Foo::Bar" ) { |t| tok << t }
    assert_equal "Foo", tok.shift
    assert_equal "::", tok.first
    assert_equal :punct, tok.shift.group
    assert_equal "Bar", tok.first
  end

  def test_symbol_dquote
    tok = []
    @ruby.tokenize( ':"foo"' ) { |t| tok << t }
    assert_equal ':"foo"', tok.first
    assert_equal :symbol, tok.first.group
  end

  def test_symbol_squote
    tok = []
    @ruby.tokenize( ":'foo'" ) { |t| tok << t }
    assert_equal ":'foo'", tok.first
    assert_equal :symbol, tok.first.group
  end

  def test_symbol
    tok = []
    @ruby.tokenize( ":foo_bar?" ) { |t| tok << t }
    assert_equal ":foo_bar?", tok.first
    assert_equal :symbol, tok.first.group
  end

  def test_char
    tok = []
    @ruby.tokenize( "?." ) { |t| tok << t }
    assert_equal "?.", tok.first
    assert_equal :char, tok.first.group

    tok.clear
    @ruby.tokenize( '?\n' ) { |t| tok << t }
    assert_equal '?\n', tok.first
    assert_equal :char, tok.first.group
  end

  def test_specials
    %w{__FILE__ __LINE__ true false nil self}.each do |word|
      tok = []
      @ruby.tokenize( word ) { |t| tok << t }
      assert_equal word, tok.first
      assert_equal :constant, tok.first.group
    end

    %w{__FILE__ __LINE__ true false nil self}.each do |word|
      tok = []
      @ruby.tokenize( "#{word}?" ) { |t| tok << t }
      assert_equal "#{word}?", tok.first
      assert_equal :ident, tok.first.group
    end

    %w{__FILE__ __LINE__ true false nil self}.each do |word|
      tok = []
      @ruby.tokenize( "#{word}!" ) { |t| tok << t }
      assert_equal "#{word}!", tok.first
      assert_equal :ident, tok.first.group
    end

    %w{__FILE__ __LINE__ true false nil self}.each do |word|
      tok = []
      @ruby.tokenize( "x.#{word}" ) { |t| tok << t }
      tok.shift
      tok.shift
      assert_equal word, tok.first
      assert_equal :ident, tok.first.group
    end
  end

  def test_pct_r
    tok = []
    @ruby.tokenize( '%r{foo#{x}bar}' ) { |t| tok << t }
    assert_equal [ :punct, "%r{" ], [ tok.first.group, tok.shift ]
    assert_equal [ :regex, "foo" ], [ tok.first.group, tok.shift ]
    assert_equal [ :expr, '#{x}' ], [ tok.first.group, tok.shift ]
    assert_equal [ :regex, "bar" ], [ tok.first.group, tok.shift ]
    assert_equal [ :punct, "}" ], [ tok.first.group, tok.shift ]

    tok = []
    @ruby.tokenize( '%r-foo#{x}bar-' ) { |t| tok << t }
    assert_equal [ :punct, "%r-" ], [ tok.first.group, tok.shift ]
    assert_equal [ :regex, "foo" ], [ tok.first.group, tok.shift ]
    assert_equal [ :expr, '#{x}' ], [ tok.first.group, tok.shift ]
    assert_equal [ :regex, "bar" ], [ tok.first.group, tok.shift ]
    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
  end

  def test_pct_w
    tok = []
    @ruby.tokenize( '%w-foo#{x} bar baz-' ) { |t| tok << t }
    assert_equal [ :punct, "%w-" ], [ tok.first.group, tok.shift ]
    assert_equal [ :string, 'foo#{x} bar baz' ], [ tok.first.group, tok.shift ]
    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
  end

  def test_pct_q
    tok = []
    @ruby.tokenize( '%q-hello #{world}-' ) { |t| tok << t }
    assert_equal [ :punct, "%q-" ], [ tok.first.group, tok.shift ]
    assert_equal [ :string, 'hello #{world}' ], [ tok.first.group, tok.shift ]
    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
  end
 
  def test_pct_s
    tok = []
    @ruby.tokenize( '%s-hello #{world}-' ) { |t| tok << t }
    assert_equal [ :punct, "%s-" ], [ tok.first.group, tok.shift ]
    assert_equal [ :symbol, 'hello #{world}' ], [ tok.first.group, tok.shift ]
    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
  end

  def test_pct_W
    tok = []
    @ruby.tokenize( '%W-foo#{x} bar baz-' ) { |t| tok << t }
    assert_equal [ :punct, "%W-" ], [ tok.first.group, tok.shift ]
    assert_equal [ :string, 'foo' ], [ tok.first.group, tok.shift ]
    assert_equal [ :expr, '#{x}' ], [ tok.first.group, tok.shift ]
    assert_equal [ :string, ' bar baz' ], [ tok.first.group, tok.shift ]
    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
  end

  def test_pct_Q
    tok = []
    @ruby.tokenize( '%Q-hello #{world}-' ) { |t| tok << t }
    assert_equal [ :punct, "%Q-" ], [ tok.first.group, tok.shift ]
    assert_equal [ :string, 'hello ' ], [ tok.first.group, tok.shift ]
    assert_equal [ :expr, '#{world}' ], [ tok.first.group, tok.shift ]
    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
  end

  def test_pct_x
    tok = []
    @ruby.tokenize( '%x-ls /blah/#{foo}-' ) { |t| tok << t }
    assert_equal [ :punct, "%x-" ], [ tok.first.group, tok.shift ]
    assert_equal [ :string, 'ls /blah/' ], [ tok.first.group, tok.shift ]
    assert_equal [ :expr, '#{foo}' ], [ tok.first.group, tok.shift ]
    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
  end

  def test_pct_string
    tok = []
    @ruby.tokenize( '%-hello #{world}-' ) { |t| tok << t }
    assert_equal [ :punct, "%-" ], [ tok.first.group, tok.shift ]
    assert_equal [ :string, 'hello ' ], [ tok.first.group, tok.shift ]
    assert_equal [ :expr, '#{world}' ], [ tok.first.group, tok.shift ]
    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
  end

  def test_bad_pct_string
    tok = []
    @ruby.tokenize( '%0hello #{world}0' ) { |t| tok << t }
    assert_equal [ :punct, "%" ], [ tok.first.group, tok.shift ]
    assert_equal [ :number, '0' ], [ tok.first.group, tok.shift ]
    assert_equal [ :ident, 'hello' ], [ tok.first.group, tok.shift ]
    assert_equal [ :normal, ' ' ], [ tok.first.group, tok.shift ]
    assert_equal [ :comment, '#{world}0' ], [ tok.first.group, tok.shift ]
  end

  def test_shift_left
    tok = []
    @ruby.tokenize( 'foo << 5' ) { |t| tok << t }
    assert_equal [ :ident, "foo" ], [ tok.first.group, tok.shift ]
    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
    assert_equal [ :punct, "<<" ], [ tok.first.group, tok.shift ]
    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
    assert_equal [ :number, "5" ], [ tok.first.group, tok.shift ]
  end

  def test_here_doc_no_opts
    tok = []
    @ruby.tokenize( "foo <<EOF\n  foo\n  bar\n  baz\nEOF" ) { |t| tok << t }
    assert_equal [ :ident, "foo" ], [ tok.first.group, tok.shift ]
    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
    assert_equal [ :punct, "<<" ], [ tok.first.group, tok.shift ]
    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
    assert_equal [ :string, "\n  foo\n  bar\n  baz\n" ], [ tok.first.group, tok.shift ]
    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
  end

  def test_here_doc_no_opts_missing_end
    tok = []
    @ruby.tokenize( "foo <<EOF\n  foo\n  bar\n  baz\n EOF" ) { |t| tok << t }
    assert_equal [ :ident, "foo" ], [ tok.first.group, tok.shift ]
    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
    assert_equal [ :punct, "<<" ], [ tok.first.group, tok.shift ]
    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
    assert_equal [ :string, "\n  foo\n  bar\n  baz\n EOF" ], [ tok.first.group, tok.shift ]
  end

  def test_here_doc_float_right
    tok = []
    @ruby.tokenize( "foo <<-EOF\n  foo\n  bar\n  baz\n  EOF" ) { |t| tok << t }
    assert_equal [ :ident, "foo" ], [ tok.first.group, tok.shift ]
    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
    assert_equal [ :punct, "<<-" ], [ tok.first.group, tok.shift ]
    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
    assert_equal [ :string, "\n  foo\n  bar\n  baz\n" ], [ tok.first.group, tok.shift ]
    assert_equal [ :constant, "  EOF" ], [ tok.first.group, tok.shift ]
  end

  def test_here_doc_single_quotes
    tok = []
    @ruby.tokenize( "foo <<'EOF'\n  foo\#{x}\n  bar\n  baz\nEOF" ) { |t| tok << t }
    assert_equal [ :ident, "foo" ], [ tok.first.group, tok.shift ]
    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
    assert_equal [ :punct, "<<'" ], [ tok.first.group, tok.shift ]
    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
    assert_equal [ :punct, "'" ], [ tok.first.group, tok.shift ]
    assert_equal [ :string, "\n  foo\#{x}\n  bar\n  baz\n" ], [ tok.first.group, tok.shift ]
    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
  end

  def test_here_doc_double_quotes
    tok = []
    @ruby.tokenize( "foo <<\"EOF\"\n  foo\#{x}\n  bar\n  baz\nEOF" ) { |t| tok << t }
    assert_equal [ :ident, "foo" ], [ tok.first.group, tok.shift ]
    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
    assert_equal [ :punct, "<<\"" ], [ tok.first.group, tok.shift ]
    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
    assert_equal [ :punct, "\"" ], [ tok.first.group, tok.shift ]
    assert_equal [ :string, "\n  foo" ], [ tok.first.group, tok.shift ]
    assert_equal [ :expr, '#{x}' ], [ tok.first.group, tok.shift ]
    assert_equal [ :string, "\n  bar\n  baz\n" ], [ tok.first.group, tok.shift ]
    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
  end

  def test_space
    tok = []
    @ruby.tokenize( "\n  \t\t\n\n\r\n" ) { |t| tok << t }
    assert_equal [ :normal, "\n  \t\t\n\n\r\n" ], [ tok.first.group, tok.shift ]
  end

  def test_number
    tok = []
    @ruby.tokenize( "1 1.0 1e5 1.0e5 1_2.5 1_2.5_2 1_2.5_2e3_2" ) { |t| tok << t }
    assert_equal [ :number, "1" ], [ tok.first.group, tok.shift ]
    tok.shift
    assert_equal [ :number, "1.0" ], [ tok.first.group, tok.shift ]
    tok.shift
    assert_equal [ :number, "1e5" ], [ tok.first.group, tok.shift ]
    tok.shift
    assert_equal [ :number, "1.0e5" ], [ tok.first.group, tok.shift ]
    tok.shift
    assert_equal [ :number, "1_2.5" ], [ tok.first.group, tok.shift ]
    tok.shift
    assert_equal [ :number, "1_2.5_2" ], [ tok.first.group, tok.shift ]
    tok.shift
    assert_equal [ :number, "1_2.5_2e3_2" ], [ tok.first.group, tok.shift ]
  end

  def test_dquoted_string
    tok = []
    @ruby.tokenize( '"foo #{x} bar\"\n\tbaz\xA5b\5\1234"' ) { |t| tok << t }
    assert_equal [ :punct, '"' ], [ tok.first.group, tok.shift ]
    assert_equal [ :string, 'foo ' ], [ tok.first.group, tok.shift ]
    assert_equal [ :expr, '#{x}' ], [ tok.first.group, tok.shift ]
    assert_equal [ :string, ' bar' ], [ tok.first.group, tok.shift ]
    assert_equal [ :expr, '\"\n\t' ], [ tok.first.group, tok.shift ]
    assert_equal [ :string, 'baz' ], [ tok.first.group, tok.shift ]
    assert_equal [ :expr, '\xA5' ], [ tok.first.group, tok.shift ]
    assert_equal [ :string, 'b' ], [ tok.first.group, tok.shift ]
    assert_equal [ :expr, '\5\123' ], [ tok.first.group, tok.shift ]
    assert_equal [ :string, '4' ], [ tok.first.group, tok.shift ]
    assert_equal [ :punct, '"' ], [ tok.first.group, tok.shift ]
  end

  def test_squoted_string
    tok = []
    @ruby.tokenize( '\'foo #{x} bar\\\'\n\tbaz\\\\\xA5b\5\1234\'' ) { |t| tok << t }
    assert_equal [ :punct, "'" ], [ tok.first.group, tok.shift ]
    assert_equal [ :string, 'foo #{x} bar' ], [ tok.first.group, tok.shift ]
    assert_equal [ :expr, '\\\'' ], [ tok.first.group, tok.shift ]
    assert_equal [ :string, '\n\tbaz' ], [ tok.first.group, tok.shift ]
    assert_equal [ :expr, '\\\\' ], [ tok.first.group, tok.shift ]
    assert_equal [ :string, '\xA5b\5\1234' ], [ tok.first.group, tok.shift ]
    assert_equal [ :punct, "'" ], [ tok.first.group, tok.shift ]
  end

  def test_dot_selector
    tok = []
    @ruby.tokenize( 'foo.nil' ) { |t| tok << t }
    tok.shift
    assert_equal [ :punct, "." ], [ tok.first.group, tok.shift ]
    assert_equal [ :ident, "nil" ], [ tok.first.group, tok.shift ]
  end

  def test_dot_range_inclusive
    tok = []
    @ruby.tokenize( 'foo..nil' ) { |t| tok << t }
    tok.shift
    assert_equal [ :punct, ".." ], [ tok.first.group, tok.shift ]
    assert_equal [ :constant, "nil" ], [ tok.first.group, tok.shift ]
  end

  def test_dot_range_exclusive
    tok = []
    @ruby.tokenize( 'foo...nil' ) { |t| tok << t }
    tok.shift
    assert_equal [ :punct, "..." ], [ tok.first.group, tok.shift ]
    assert_equal [ :constant, "nil" ], [ tok.first.group, tok.shift ]
  end

  def test_dot_range_many
    tok = []
    @ruby.tokenize( 'foo.....nil' ) { |t| tok << t }
    tok.shift
    assert_equal [ :punct, "....." ], [ tok.first.group, tok.shift ]
    assert_equal [ :constant, "nil" ], [ tok.first.group, tok.shift ]
  end

  def test_attribute
    tok = []
    @ruby.tokenize( '@var_foo' ) { |t| tok << t }
    assert_equal [ :attribute, "@var_foo" ], [ tok.first.group, tok.shift ]
  end

  def test_global
    tok = []
    @ruby.tokenize( '$var_foo' ) { |t| tok << t }
    assert_equal [ :global, "$var_foo" ], [ tok.first.group, tok.shift ]
    tok = []
    @ruby.tokenize( '$12' ) { |t| tok << t }
    assert_equal [ :global, "$12" ], [ tok.first.group, tok.shift ]
    tok = []
    @ruby.tokenize( '$/f' ) { |t| tok << t }
    assert_equal [ :global, "$/" ], [ tok.first.group, tok.shift ]
    tok = []
    @ruby.tokenize( "$\n" ) { |t| tok << t }
    assert_equal [ :global, "$" ], [ tok.first.group, tok.shift ]
  end

end