lkqd.rb in embulk-input-lkqd-0.3.0

- old
+ new

@@ -1,8 +1,9 @@
 require 'base64'
 require 'json'
 require 'csv'
+require 'fileutils'
 require 'tempfile'
 require 'http'
 
 module Embulk
   module Input
@@ -18,14 +19,16 @@
           "report_parameters" => config.param("report_parameters", :hash, default: {}),
         }
         task['authorization'] = Base64.urlsafe_encode64("#{task['secret_key_id']}:#{task['secret_key']}")
 
         response = request_lkqd({authorization: task['authorization'], endpoint: task['endpoint'], report_parameters: task['report_parameters']})
-        tempfile = Tempfile.new('lkqd_')
-        tempfile.write response.body
-        task['tempfile_path'] = tempfile.path
+        tempfile = Tempfile.new('emublk-input-lkqd_')
+        while chunk = response.body.readpartial
+          tempfile.write chunk
+        end
         tempfile.close
+        task['tempfile_path'] = tempfile.path
 
         columns = []
         ::CSV.foreach(tempfile.path).first.each_with_index do |column_name, index|
           column_type = guess_column_type(task['report_parameters']['metrics'], column_name)
           columns << Column.new({index: index}.merge(column_type))
@@ -87,9 +90,10 @@
       def run
         CSV.foreach(@task['tempfile_path'], {headers: true}).each do |row|
           page_builder.add(Lkqd.try_convert(row))
         end
         page_builder.finish
+        FileUtils.rm_rf(@task['tempfile_path'])
 
         task_report = {}
         return task_report
       end
 