Contents

# -*- coding: utf-8 -*-
#
# 文字列から1トークンずつ取得
#

$KCODE = 'utf8'

module ReExpand
  class Scanner
    def initialize(s)
      @s = s
      @a = s.split(//)
      @p = 0
      @t = ''
      @u = ''
    end
    
    def gettoken
      if @u != '' then
        @t = @u
        @u = ''
        return @t
      end
      if @p >= @a.length then
        @t = ''
        return ''
      end
      @t = @a[@p]
      if @t =~ /^[\(\|\)\*\?\[\]]$/ then
        @p += 1
        return @t
      elsif @t == '\\' then
        @p += 1
        @t = @a[@p]
        @t = "\n" if @t == 'n'
        @t = "\t" if @t == 't'
        @p += 1
        return @t
      else
        @p += 1
        while @p < @a.length && @a[@p] !~ /^[\(\|\)\*\+\?\[\]\\]$/ do
          @t += @a[@p]
          @p += 1
        end
        return @t
      end
    end
    
    def ungettoken
      if @u == '' then
        @u = @t
      else
        puts "Can't ungettoken(#{token})"
      end
    end
    
    def nexttoken
      return @t
    end
  end
end

Version data entries

2 entries across 2 versions & 1 rubygems

Version	Path
re_expand-0.0.3	lib/Scanner.rb
re_expand-0.0.2	lib/Scanner.rb