lexer.rb in rdf-turtle-0.1.1

- old
+ new

@@ -198,16 +198,24 @@
 
         token = match_token
 
         if token.nil?
           lexme = (scanner.rest.split(/#{@whitespace}|#{@comment}/).first rescue nil) || scanner.rest
-          raise Error.new("Invalid token #{lexme.inspect} on line #{lineno + 1}",
+          raise Error.new("Invalid token #{lexme[0..100].inspect}",
             :input => scanner.rest[0..100], :token => lexme, :lineno => lineno)
         end
 
         token
       end
+    rescue ArgumentError, Encoding::CompatibilityError => e
+      raise Error.new("#{e.message} on line #{lineno + 1}",
+        :input => (scanner.rest[0..100] rescue '??'), :token => lexme, :lineno => lineno)
+    rescue Error
+      raise
+    rescue
+      STDERR.puts "Expected ArgumentError, got #{$!.class}"
+      raise
     end
 
     ##
     # Returns first token and shifts to next
     #
@@ -221,17 +229,17 @@
     ##
     # Skip input until a token is matched
     #
     # @return [Token]
     def recover
-      scanner.skip(/./)
       until scanner.eos? do
         begin
+          shift
           return first
-        rescue Error
+        rescue Error, ArgumentError
           # Ignore errors until something scans, or EOS.
-          scanner.skip(/./)
+          scanner.pos = scanner.pos + 1
         end
       end
     end
   protected
 
@@ -452,7 +460,14 @@
         @token  = options[:token]
         @lineno = options[:lineno]
         super(message.to_s)
       end
     end # class Error
+    
+    unless "".respond_to?(:force_encoding)
+      # Compatibility with 1.9 Encoding
+      module Encoding
+        class CompatibilityError < StandardError; end
+      end
+    end
   end # class Lexer
 end # module RDF::Turtle