module CropDuster class Importer REGIONS = { aws: ['ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-west-1', 'sa-east-1', 'us-east-1', 'us-west-1', 'us-west-2'] } class << self def import import_id = SecureRandom.hex self.import_servers(import_id) if CropDuster.aws_billing_bucket.present? self.import_billing(import_id) else puts "No billing bucket provided, so precise AWS billing information is unknown." end end def import_servers(import_id = SecureRandom.hex) REGIONS[:aws].each do |region| compute = Fog::Compute.new CropDuster.aws_keys.merge(region: region) compute.servers.each do |s| server = Server.import(s) server.update_attributes(last_import_id: import_id) end compute.volumes.each do |v| volume = Volume.import(v) volume.update_attributes(last_import_id: import_id) end #compute.snapshots.each do |hash| #snapshot = Snapshot.import(hash) #snapshot.update_attributes(last_import_id: import_id) #end #Snapshot.where("last_import_id != ? AND deleted_at IS NULL", import_id).update_all(deleted_at: Time.now) end Server.where("last_import_id != ? AND deleted_at IS NULL", import_id).update_all(deleted_at: Time.now) Volume.where("last_import_id != ? AND deleted_at IS NULL", import_id).update_all(deleted_at: Time.now) end def import_billing(import_id = SecureRandom.hex) ActiveRecord::Base.connection.execute("TRUNCATE aws_billing_imports") s = Fog::Storage.new CropDuster.aws_keys path = "/tmp/#{import_id}" Dir.mkdir path s.directories.get(CropDuster.aws_billing_bucket).files.find_all { |f| f.key =~ /-aws-billing-detailed-line-items-with-resources-and-tags-/ }.each do |file| `curl -o #{path}/file.zip '#{file.url(15.minutes.since)}'` `unzip -p #{path}/file.zip > #{path}/#{Time.now.strftime('%Y%m%d%H%M%S.csv')}` `rm #{path}/file.zip` end Dir.new(path).each do |file| next if file =~ /^\./ headers = nil ar_conn = ActiveRecord::Base.connection_pool.checkout conn = ar_conn.raw_connection conn.copy_data "COPY aws_billing_imports FROM STDIN CSV" do File.new(File.join(path, file)).each_line do |line| if headers.nil? headers = line.strip.split(/,/).map(&:underscore) else conn.put_copy_data line end end end ActiveRecord::Base.connection_pool.checkin(ar_conn) end `rm #{path}/*.csv` end end end end