parser.rb in asciimath-1.0.3

- old
+ new

@@ -51,11 +51,10 @@
   # This determines if the accent should be rendered over or under the expression to which
   # it applies.
   #
   class Tokenizer
     WHITESPACE = /^\s+/
-    NUMBER_START = /[-0-9]/
     NUMBER = /-?[0-9]+(?:\.[0-9]+)?/
     TEXT = /"[^"]+"/
 
     # Public: Initializes an ASCIIMath tokenizer.
     #
@@ -85,11 +84,11 @@
       return {:value => nil, :type => :eof} if @string.eos?
 
       case @string.peek(1)
         when '"'
           read_text
-        when NUMBER_START
+        when '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'
           read_number() || read_symbol
         else
           read_symbol()
       end
     end
@@ -123,10 +122,21 @@
       read_value(NUMBER) do |number|
         {:value => number, :type => :number}
       end
     end
 
+    if String.method_defined?(:bytesize)
+      def bytesize(s)
+        s.bytesize
+      end
+    else
+      def bytesize(s)
+        s.length
+      end
+    end
+
+
     # Private: Reads a symbol token from the input string. This method first creates
     # a String from the input String starting from the current position with a length
     # that matches that of the longest key in the symbol table. It then looks up that
     # substring in the symbol table. If the substring is present in the symbol table, the
     # associated value is returned and the position is moved ahead by the length of the
@@ -141,11 +151,11 @@
       position = @string.pos
       read_value(@symbol_regexp) do |s|
         until s.length == 1 || @symbols.include?(s)
           s.chop!
         end
-        @string.pos = position + s.length
+        @string.pos = position + bytesize(s)
         @symbols[s] || {:value => s, :type => :identifier}
       end
     end
 
     # Private: Reads a String from the input String that matches the given RegExp
@@ -158,9 +168,15 @@
       s = @string.scan(regexp)
       if s
         yield s
       else
         s
+      end
+    end
+
+    if String.respond_to?(:byte_size)
+      def byte_size(s)
+        s.byte_size
       end
     end
   end
 
   class Parser