lib/vowpalwabbit/model.rb in vowpalwabbit-0.1.0 vs lib/vowpalwabbit/model.rb in vowpalwabbit-0.1.1

@@ -49,24 +49,24 @@
 
     def load_model(filename)
       bin_str = File.binread(filename)
       model_data = ::FFI::MemoryPointer.new(:char, bin_str.bytesize)
       model_data.put_bytes(0, bin_str)
-      @handle = FFI.VW_InitializeWithModel(param_str, model_data, bin_str.bytesize)
+      @handle = FFI.VW_InitializeWithModel(param_str(@params), model_data, bin_str.bytesize)
       nil
     end
 
     private
 
     # TODO clean-up handle
     def handle
-      @handle ||= FFI.VW_InitializeA(param_str)
+      @handle ||= FFI.VW_InitializeA(param_str(@params))
     end
 
-    def param_str
+    def param_str(params)
       args =
-        @params.map do |k, v|
+        params.map do |k, v|
           check_param(k.to_s)
           check_param(v.to_s)
 
           if v == true
             "--#{k}"
@@ -106,34 +106,42 @@
       else
         [predict(x), y]
       end
     end
 
-    # TODO support compressed files
     def each_example(x, y = nil)
-      each_line(x, y) do |line|
-        example = FFI.VW_ReadExampleA(handle, line)
-        yield example
-        FFI.VW_FinishExample(handle, example)
-      end
-    end
-
-    def each_line(x, y)
+      # path to a file
       if x.is_a?(String)
         raise ArgumentError, "Cannot pass y with file" if y
 
-        File.foreach(x) do |line|
-          yield line
+        file_handle = FFI.VW_InitializeA(param_str(data: x, quiet: true))
+        FFI.VW_StartParser(file_handle)
+        loop do
+          example = FFI.VW_GetExample(file_handle)
+          break if example.read_pointer.null?
+          yield example
+          FFI.VW_FinishExample(file_handle, example)
         end
+        FFI.VW_EndParser(file_handle)
+        FFI.VW_Finish(file_handle)
       else
-        raise ArgumentError, "x and y must have same size" if y && x.size != y.size
+        x = x.to_a
+        if y
+          y = y.to_a
+          raise ArgumentError, "x and y must have same size" if x.size != y.size
+        end
 
         x.zip(y || []) do |xi, yi|
-          if xi.is_a?(String)
-            yield xi
-          else
-            yield "#{yi} 1 | #{xi.map.with_index { |v, i| "#{i}:#{v}" }.join(" ")}"
-          end
+          line =
+            if xi.is_a?(String)
+              xi
+            else
+              "#{yi} 1 | #{xi.map.with_index { |v, i| "#{i}:#{v}" }.join(" ")}"
+            end
+
+          example = FFI.VW_ReadExampleA(handle, line)
+          yield example
+          FFI.VW_FinishExample(handle, example)
         end
       end
     end
   end
 end