data_miner.rb in earth-0.6.0

- old
+ new

@@ -8,28 +8,10 @@
     def in_july_2009?(row)
       row ['MONTH'] == 7 and row['YEAR'] == 2009
     end
   end
   
-  def self.update_averages!
-    # Derive load factor, which is passengers divided by available seats
-    update_all 'load_factor = passengers / seats', 'seats > 0'
-
-    # Assume a load factor of 1 where passengers > available seats
-    update_all 'load_factor = 1', 'passengers > seats AND seats > 0'
-  
-    # TODO: what is 90.718474
-    # Derive freight share as a fraction of the total weight carried
-    update_all 'freight_share = (freight + mail) / (freight + mail + (passengers * 90.718474))', '(freight + mail + passengers) > 0'
-  
-    # Derive average seats per flight
-    update_all 'seats_per_flight = seats / flights', 'flights > 0'
-    
-    # Add a useful date field
-    update_all 'approximate_date = DATE(CONCAT_WS("-", year, month, "14"))', 'month IS NOT NULL'
-  end
-  
   URL = 'http://www.transtats.bts.gov/DownLoad_Table.asp?Table_ID=293&Has_Group=3&Is_Zipped=0'
   FORM_DATA = %{
     UserTableName=T_100_Segment__All_Carriers&
     DBShortName=Air_Carriers&
     RawDataTable=T_T100_SEGMENT_ALL_CARRIER&
@@ -185,24 +167,27 @@
     VarName=DATA_SOURCE&
     VarDesc=DataSource&
     VarType=Char
   }.gsub /[\s]+/,''
   
-  data_miner do
-    months = Hash.new
-    (2009..2011).each do |year|
+  def self.form_data_per_month(year_range)
+    months = {}
+    year_range.each do |year|
       (1..12).each do |month|
-        time = Time.gm year, month
+        time = ::Time.gm year, month
         form_data = FORM_DATA.dup
         form_data.gsub! '__YEAR__', time.year.to_s
         form_data.gsub! '__MONTH_NUMBER__', time.month.to_s
         form_data.gsub! '__MONTH_NAME__', time.strftime('%B')
         months[time] = form_data
       end
     end
-    
-    months.each do |month, form_data|
+    months
+  end
+  
+  data_miner do
+    form_data_per_month(2009..2011).each do |month, form_data|
       import "T100 flight segment data for #{month.strftime('%B %Y')}",
              :url => URL,
              :form_data => form_data,
              :compression => :zip,
              :glob => '/*.csv',
@@ -235,18 +220,18 @@
     end
     
     process "Look up airline name based on BTS code" do
       connection.select_values("SELECT DISTINCT airline_bts_code FROM flight_segments WHERE airline_bts_code IS NOT NULL").each do |bts_code|
         if airline = Airline.find_by_bts_code(bts_code)
-          update_all %{ airline_name = "#{airline.name}" }, %{ airline_bts_code = "#{bts_code}" }
+          update_all({ :airline_name => airline.name }, :airline_bts_code => bts_code)
         end
       end
     end
     
     process "Look up aircraft description based on BTS code" do
       connection.select_values("SELECT DISTINCT aircraft_bts_code FROM flight_segments WHERE aircraft_bts_code IS NOT NULL").each do |bts_code|
         if aircraft = BtsAircraft.find_by_bts_code(bts_code)
-          update_all %{ aircraft_description = "#{aircraft.description.downcase}" }, %{ aircraft_bts_code = "#{bts_code}" }
+          update_all({ :aircraft_description => aircraft.description.downcase }, :aircraft_bts_code => bts_code)
         end
       end
     end
     
     process :update_averages!