module Github module Archive class StatCollector @queue = :github_archive class << self attr_reader :queue def perform(stat_url) archived = ArchivedUrl.where(url: stat_url).first if archived.nil? puts "cannot archive untracked url #{stat_url}" elsif archived.finished_processing puts "already processed #{stat_url}" else begin ArchivedUrl.transaction do puts "processing #{stat_url}" gz = open(stat_url) js = Zlib::GzipReader.new(gz).read Yajl::Parser.parse(js) do |event| Github::Archive::Event.create_with_json(event) end archived.finished_processing = true archived.save end rescue puts "There was some kind of error while parsing #{stat_url}" archived.finished_processing = true archived.save end end end end end end end