lib/pupa/runner.rb in pupa-0.0.4 vs lib/pupa/runner.rb in pupa-0.0.5
- old
+ new
@@ -160,26 +160,43 @@
end
end
exit if options.dry_run
+ report = {
+ plan: {
+ processor: @processor_class,
+ arguments: options.to_h,
+ options: rest,
+ },
+ start: Time.now.utc,
+ }
+
Pupa.session = Moped::Session.new([options.host_with_port], database: options.database)
if options.actions.delete('scrape')
FileUtils.mkdir_p(options.output_dir)
FileUtils.mkdir_p(options.cache_dir)
Dir[File.join(options.output_dir, '*.json')].each do |path|
FileUtils.rm(path)
end
+ report[:scrape] = {}
options.tasks.each do |task_name|
- processor.dump_scraped_objects(task_name)
+ report[:scrape][task_name] = processor.dump_scraped_objects(task_name)
end
end
options.actions.each do |action|
processor.send(action)
+ if processor.report.key?(action.to_sym)
+ report.update(action.to_sym => processor.report[action.to_sym])
+ end
end
+
+ report[:end] = Time.now.utc
+ report[:time] = report[:end] - report[:start]
+ puts JSON.dump(report)
end
end
end