tokeniser.rb in nydp-0.0.5

- old
+ new

@@ -2,19 +2,24 @@
 
 module Nydp
   class Tokeniser
     attr_accessor :state, :finished
 
-    def initialize stream
-      @stream = stream.is_a?(String) ? nil : stream
-      @scanner = StringScanner.new(stream.is_a?(String) ? stream : "")
+    def initialize reader
+      @reader = reader
+      @scanner = StringScanner.new("")
       @state = :lisp
     end
 
     def no_more?
-      @scanner << @stream.readline if @scanner.eos? && @stream && !@stream.eof?
-      @scanner.eos?
+      if @scanner.eos?
+        nextline = @reader.nextline
+        return true if nextline.nil?
+        @scanner << nextline
+      end
+
+      false
     end
 
     def close_delimiter? scanner, delim
       return (no_more? ? '' : nil) if (delim == :eof)
       scanner.scan(delim)
@@ -35,12 +40,12 @@
           end
           rep << ch
         elsif closer = close_delimiter?(s, close_delimiter)
           rep << closer
           return StringFragmentCloseToken.new(string, rep)
-        elsif embed_suffix = s.scan(/%%/)
-          rep << embed_suffix
+        elsif start_interpolation = s.scan(/~/)
+          rep << start_interpolation
           return StringFragmentToken.new(string, rep)
         else
           ch = s.getch
           string << ch
           rep    << ch
@@ -59,12 +64,9 @@
           return nil
         elsif comment = s.scan(/;.*$/)
           tok = [:comment, comment[1..-1].strip]
         elsif open_str = s.scan(/"/)
           tok = [:string_open_delim, open_str]
-        elsif embed_suffix = s.scan(/\]#/)
-          self.state = :external_text
-          tok = [:embed_suffix, embed_suffix]
         elsif list_prefix = s.scan(/[^\s()]*\(/)
           tok = [:left_paren, list_prefix[0...-1]]
         elsif list_prefix = s.scan(/[^\s()]*\{/)
           tok = [:left_brace, list_prefix[0...-1]]
         elsif s.scan(/\)/)