lib/rouge/lexer.rb in rouge-0.0.13 vs lib/rouge/lexer.rb in rouge-0.0.14

@@ -2,39 +2,14 @@
 require 'strscan'
 
 module Rouge
   class Lexer
     class << self
-      def make(opts={}, &b)
-        _sup = self
-
-        Class.new(self) do
-          @lazy_load_proc = b
-          @default_options = _sup.default_options.merge(opts)
-          @parent = _sup
-        end
-      end
-
       def lex(stream, opts={}, &b)
         new(opts).lex(stream, &b)
       end
 
-    protected
-      def force_load!
-        return self if @force_load
-        @force_load = true
-        @lazy_load_proc && instance_eval(&@lazy_load_proc)
-
-        self
-      end
-    public
-
-      def new(*a, &b)
-        force_load!
-        super(*a, &b)
-      end
-
       def default_options
         @default_options ||= {}
       end
 
       def find(name)
@@ -119,11 +94,10 @@
 
     # -*- instance methods -*- #
 
     def initialize(opts={}, &b)
       options(opts)
-      @lazy_load_proc = b
     end
 
     def options(o={})
       (@options ||= {}).merge!(o)
 
@@ -144,13 +118,19 @@
 
     def get_tokens(stream)
       lex(stream).to_a
     end
 
-    def lex(string, &b)
+    def reset!
+      # noop, called after each lex is finished
+    end
+
+    def lex(string, opts={}, &b)
       return enum_for(:lex, string) unless block_given?
 
+      reset! unless opts[:continue]
+
       last_token = nil
       last_val = nil
       stream_tokens(StringScanner.new(string)) do |tok, val|
         next if val.empty?
 
@@ -242,11 +222,11 @@
       end
 
       attr_accessor :scanner
       attr_accessor :stack
       attr_accessor :lexer
-      def initialize(lexer, scanner, stack=nil)
+      def initialize(lexer, scanner=nil, stack=nil)
         @lexer = lexer
         @scanner = scanner
         @stack = stack || [lexer.get_state(:root)]
       end
 
@@ -256,12 +236,19 @@
         debug { "    popping stack" }
         stack.pop
       end
 
       def push(state_name=nil, &b)
-        debug { "    pushing #{state_name || 'anonymous state'}" }
-        stack.push(state.relative_state(state_name, &b))
+        # use the top of the stack by default
+        if state_name || b
+          push_state = state.relative_state(state_name, &b)
+        else
+          push_state = self.state
+        end
+
+        debug { "    pushing #{push_state.name}" }
+        stack.push(push_state)
       end
 
       def in_state?(state_name)
         stack.map(&:name).include? state_name.to_s
       end
@@ -298,14 +285,14 @@
       def group(tok)
         token(tok, scanner[@group_count += 1])
       end
 
       def delegate(lexer, text=nil)
-        debug { "    delegating to #{lexer.name}" }
+        debug { "    delegating to #{lexer.inspect}" }
         text ||= scanner[0]
 
-        lexer.lex(text) do |tok, val|
+        lexer.lex(text, :continue => true) do |tok, val|
           debug { "    delegated token: #{tok.inspect}, #{val.inspect}" }
           token(tok, val)
         end
       end
 
@@ -350,12 +337,12 @@
         if block_given?
           next_state = tok
         else
           tok = Token[tok]
 
-          callback = proc do |ss|
-            token tok, ss[0]
+          callback = proc do
+            token tok
             case next_state
             when :pop!
               pop!
             when Symbol
               push next_state
@@ -386,20 +373,10 @@
     def self.state(name, &b)
       name = name.to_s
       states[name] = State.new(self, name, &b)
     end
 
-    def initialize(parent=nil, opts={}, &defn)
-      if parent.is_a? Hash
-        opts = parent
-        parent = nil
-      end
-
-      @parent = parent
-      super(opts, &defn)
-    end
-
     def self.get_state(name)
       return name if name.is_a? State
 
       state = states[name.to_s]
       raise "unknown state: #{name}" unless state
@@ -412,21 +389,26 @@
 
     def get_state(name)
       self.class.get_state(name)
     end
 
-    def stream_tokens(stream, &b)
-      scan_state = ScanState.new(self, stream)
+    def scan_state
+      @scan_state ||= ScanState.new(self)
+    end
 
+    def reset!
+      @scan_state = nil
+
       self.class.start_procs.each do |pr|
         scan_state.instance_eval(&pr)
       end
-
-      stream_with_state(scan_state, &b)
     end
 
-    def stream_with_state(scan_state, &b)
+    def stream_tokens(stream, &b)
+      scan_state.scanner = stream
+
       until scan_state.eos?
+        debug { "lexer: #{self.class.tag}" }
         debug { "stack: #{scan_state.stack.map(&:name).inspect}" }
         debug { "stream: #{scan_state.scanner.peek(20).inspect}" }
         success = step(get_state(scan_state.state), scan_state, &b)
 
         if !success