lib/protobuf/compiler/proto.y in ruby_protobuf-0.3.2 vs lib/protobuf/compiler/proto.y in ruby_protobuf-0.3.3

@@ -1,7 +1,6 @@
 class Protobuf::ProtoParser
-  #options no_result_var
 rule
   proto : proto_item
           { result = Protobuf::Node::ProtoNode.new val }
         | proto proto_item
           { result.children << val[1] }
@@ -103,15 +102,17 @@
                       { }
 
   group : label 'group' CAMEL_IDENT '=' integer_literal message_body
           { result = Protobuf::Node::GroupNode.new val[0], val[2], val[4], val[5] }
 
-  field : label type IDENT '=' integer_literal ';'
+  field : label type field_name '=' integer_literal ';'
           { result = Protobuf::Node::FieldNode.new val[0], val[1], val[2], val[4] }
-        | label type IDENT '=' integer_literal '[' field_option_list ']' ';'
+        | label type field_name '=' integer_literal '[' field_option_list ']' ';'
           { result = Protobuf::Node::FieldNode.new val[0], val[1], val[2], val[4], val[6] }
 
+  field_name : IDENT | "required" | "optional" | "repeated" | "import" | "package" | "option" | "message" | "extend" | "enum" | "service" | "rpc" | "returns" | "group" | "default" | "extensions" | "to" | "max" | "double" | "float" | "int32" | "int64" | "uint32" | "uint64" | "sint32" | "sint64" | "fixed32" | "fixed64" | "sfixed32" | "sfixed64" | "bool" | "string" | "bytes"
+
   field_option_list : field_option
                       { result = val }
                     | field_option_list ',' field_option
                       { result << val[2] }
 
@@ -158,44 +159,63 @@
                   | OCT_INTEGER
 end
 
 ---- inner
 
+  require 'strscan'
+
   def parse(f)
-    @q = []
-    f.each do |line|
-      until line.empty? do
-        case line
-        when /\A\s+/, /\A\/\/.*/
-          ;
-        when /\A(required|optional|repeated|import|package|option|message|extend|enum|service|rpc|returns|group|default|extensions|to|max|double|float|int32|int64|uint32|uint64|sint32|sint64|fixed32|fixed64|sfixed32|sfixed64|bool|string|bytes)\b/
-          @q.push [$&, $&.to_sym]
-        when /\A[1-9]\d*(?!\.)/, /\A0(?![.xX0-9])/
-          @q.push [:DEC_INTEGER, $&.to_i]
-        when /\A0[xX]([A-Fa-f0-9])+/
-          @q.push [:HEX_INTEGER, $&.to_i(0)]
-        when /\A0[0-7]+/
-          @q.push [:OCT_INTEGER, $&.to_i(0)]
-        when /\A\d+(\.\d+)?([Ee][\+-]?\d+)?/
-          @q.push [:FLOAT_LITERAL, $&.to_f]
-        when /\A(true|false)/
-          @q.push [:BOOLEAN_LITERAL, $& == 'true']
-        when /\A"(?:[^"\\]+|\\.)*"/, /\A'(?:[^'\\]+|\\.)*'/
-          @q.push [:STRING_LITERAL, eval($&)]
-        when /\A[a-zA-Z_][\w_]*/
-          @q.push [:IDENT, $&.to_sym]
-        when /\A[A-Z][\w_]*/
-          @q.push [:CAMEL_IDENT, $&.to_sym]
-        when /\A./
-          @q.push [$&, $&]
-        else
-          raise ArgumentError.new(line) 
-        end
-        line = $'
+    @scanner = StringScanner.new(f.read)
+    yyparse(self, :scan)
+  end
+
+  def scan_debug
+    scan do |token, value|
+      p [token, value]
+      yield [token, value]
+    end
+  end
+
+  def scan
+    until @scanner.eos?
+      case
+      when match(/\s+/, /\/\/.*/)
+        # skip
+      when match(/\/\*/)
+        # C-like comment
+        raise 'EOF inside block comment' until @scanner.scan_until(/\*\//)
+      when match(/(?:required|optional|repeated|import|package|option|message|extend|enum|service|rpc|returns|group|default|extensions|to|max|double|float|int32|int64|uint32|uint64|sint32|sint64|fixed32|fixed64|sfixed32|sfixed64|bool|string|bytes)\b/)
+        yield [@token, @token.to_sym]
+      when match(/[+-]?\d*\.\d+([Ee][\+-]?\d+)?/)
+        yield [:FLOAT_LITERAL, @token.to_f]
+      when match(/[+-]?[1-9]\d*(?!\.)/, /0(?![.xX0-9])/)
+        yield [:DEC_INTEGER, @token.to_i]
+      when match(/0[xX]([A-Fa-f0-9])+/)
+        yield [:HEX_INTEGER, @token.to_i(0)]
+      when match(/0[0-7]+/)
+        yield [:OCT_INTEGER, @token.to_i(0)]
+      when match(/(true|false)\b/)
+        yield [:BOOLEAN_LITERAL, @token == 'true']
+      when match(/"(?:[^"\\]+|\\.)*"/, /'(?:[^'\\]+|\\.)*'/)
+        yield [:STRING_LITERAL, eval(@token)]
+      when match(/[a-zA-Z_][\w_]*/)
+        yield [:IDENT, @token.to_sym]
+      when match(/[A-Z][\w_]*/)
+        yield [:CAMEL_IDENT, @token.to_sym]
+      when match(/./)
+        yield [@token, @token]
+      else
+        raise "parse error around #{@scanner.string[@scanner.pos, 32].inspect}"
       end
     end
-    do_parse
+    yield [false, nil]
   end
 
-  def next_token
-    @q.shift
+  def match(*regular_expressions)
+    regular_expressions.each do |re|
+      if @scanner.scan(re)
+        @token = @scanner[0]
+        return true
+      end
+    end
+    false
   end