lib/echi-converter.rb in echi-converter-0.1.1 vs lib/echi-converter.rb in echi-converter-0.2.0

@@ -1,9 +1,10 @@
 require 'rubygems'
 require 'active_record'
 require 'faster_csv'
 require 'net/ftp'
+require 'net/smtp'
 require 'fileutils'
 
 class Logger
   #Change the logging format to include a timestamp
   def format_message(severity, timestamp, progname, msg)
@@ -32,10 +33,11 @@
     begin
       ActiveRecord::Base.establish_connection(YAML::load(File.open(databaseconfig))) 
       @log.info "Successfully connected to the database"
     rescue => err
       @log.fatal "Could not connect to the database - " + err
+      send_email_alert "DATABASE"
     end
   end
   
   #Method to open our application log
   def initiate_logger
@@ -53,10 +55,50 @@
       when 'DEBUG'
         @log.level = Logger::DEBUG
     end
   end
   
+  #Method to send alert emails
+  def send_email_alert reason
+    begin
+      Net::SMTP.start(@config["smtp_server"], @config["smtp_port"]) do |smtp|
+        smtp.open_message_stream('[e-mail]', [@config["alert_email_address"]]) do |f|
+          f.puts "From: [e-mail]"
+          f.puts "To: " + @config['alert_email_address']
+          f.puts "Subject: ECHI-Converter Failure"
+          case reason 
+          when "DATABASE"
+            f.puts "Failed to connect to the database."
+          when "FTP"
+            f.puts "Failed to connect to the ftp server."
+          end
+            f.puts " "
+            f.puts "Please check the ECHI-Converter environment as soon as possible."
+        end
+      end
+    rescue => err
+      @log.warn err
+    end
+  end
+  
+  #Set the working directory to copy processed files to, if it does not exist creat it
+  #Directory names based on year/month so as not to exceed 5K files in a single directory
+  def set_directory working_directory
+    time = Time.now
+    directory_year = working_directory + "/../files/processed/" + time.year.to_s 
+    directory_month = directory_year + "/" + time.month.to_s
+    
+    if File.exists?(directory_month) == false
+      if File.exists?(directory_year) == false
+        Dir.mkdir(directory_year)
+      end
+      Dir.mkdir(directory_month)
+    end      
+    
+    return directory_month
+  end
+  
   #Method for parsing the various datatypes from the ECH file
   def dump_binary type, length
     case type
     when 'int' 
       #Process integers, assigning appropriate profile based on length
@@ -114,65 +156,68 @@
       echi_log.filename = filename
       echi_log.filenumber = filenumber
       echi_log.version = fileversion
     end
     
-    bool_cnt = 0
-    record_cnt = 0
-    while @binary_file.eof == FALSE do 
-      @log.debug '<====================START RECORD ' + record_cnt.to_s + ' ====================>'
-      echi_record = EchiRecord.new
-      @echi_schema["fields"].each do | field |
-        #We handle the 'boolean' fields differently, as they are all encoded as bits in a single 8-bit byte
-        if field["type"] == 'bool'
-          if bool_cnt == 0
-            bytearray = dump_binary field["type"], field["length"]
-          end
-          #Ensure we parse the bytearray and set the appropriate flags
-          #We need to make sure the entire array is not nil, in order to do Y/N
-          #if Nil we then set all no
-          if bytearray != nil
-            if bytearray.slice(bool_cnt,1) == 1
-              value = 'Y'
-            else
+    #Perform a transaction for each file, including the log table
+    #in order to commit as one atomic action upon success
+    EchiRecord.transaction do
+      bool_cnt = 0
+      @record_cnt = 0
+      while @binary_file.eof == FALSE do 
+        @log.debug '<====================START RECORD ' + @record_cnt.to_s + ' ====================>'
+        echi_record = EchiRecord.new
+        @echi_schema["fields"].each do | field |
+          #We handle the 'boolean' fields differently, as they are all encoded as bits in a single 8-bit byte
+          if field["type"] == 'bool'
+            if bool_cnt == 0
+              bytearray = dump_binary field["type"], field["length"]
+            end
+            #Ensure we parse the bytearray and set the appropriate flags
+            #We need to make sure the entire array is not nil, in order to do Y/N
+            #if Nil we then set all no
+            if bytearray != nil
+              if bytearray.slice(bool_cnt,1) == 1
+                value = 'Y'
+              else
+                value = 'N'
+              end
+            else 
               value = 'N'
             end
-          else 
-            value = 'N'
+            bool_cnt += 1
+            if bool_cnt == 8
+              bool_cnt = 0
+            end
+          else
+            #Process 'standard' fields
+            value = dump_binary field["type"], field["length"]
+            @log.debug field["name"] + " { type => #{field["type"]} & length => #{field["length"]} } value => " + value.to_s
           end
-          bool_cnt += 1
-          if bool_cnt == 8
-            bool_cnt = 0
-          end
-        else
-          #Process 'standard' fields
-          value = dump_binary field["type"], field["length"]
-          @log.debug field["name"] + " { type => #{field["type"]} & length => #{field["length"]} } value => " + value.to_s
+          echi_record[field["name"]] = value
         end
-        echi_record[field["name"]] = value
-      end
-      echi_record.save
+        echi_record.save
       
-      #Scan past the end of line record
-      @binary_file.read(1)
-      @log.debug '<====================STOP RECORD ' + record_cnt.to_s + ' ====================>'
-      record_cnt += 1
+        #Scan past the end of line record
+        @binary_file.read(1)
+        @log.debug '<====================STOP RECORD ' + @record_cnt.to_s + ' ====================>'
+        @record_cnt += 1
+      end
+      @binary_file.close
     end
-    @binary_file.close
     
     #Move the file to the processed directory
-    destination_directory = @workingdirectory + '/../files/processed/'
-    FileUtils.mv(echi_file, destination_directory)
+    FileUtils.mv(echi_file, @processeddirectory)
     
     if @config["echi_process_log"] == "Y"
       #Finish logging the details on the file
-      echi_log.records = record_cnt
+      echi_log.records = @record_cnt
       echi_log.processed_at = Time.now
       echi_log.save
     end
     
-    return record_cnt
+    return @record_cnt
   end
   
   def connect_ftpsession
     #Open ftp connection
     begin
@@ -186,10 +231,11 @@
         @log.fatal "SSH currently not supported, please use FTP for accessing the ECHI server"
         exit
       end
     rescue => err
       @log.fatal "Could not connect with the FTP server - " + err
+      send_email_alert "FTP"
       return -1
     end
     return ftp_session
   end
   
@@ -208,11 +254,13 @@
        ftp_session = 0
      end
    end
    if ftp_session != 0
      begin
-       ftp_session.chdir(@config["echi_ftp_directory"])
+       if @config["echi_ftp_directory"] != nil
+         ftp_session.chdir(@config["echi_ftp_directory"])
+       end
        files = ftp_session.list('chr*')
        file_cnt = 0
        files.each do | file |
          #ACTION:  Need to detect which OS we are running on and then parse the ftp data appropriately
          file_data = file.split(' ')
@@ -247,49 +295,52 @@
     echi_log.filename = filename
     #echi_log.filenumber = filenumber
     #echi_log.version = fileversion
   end
   
-  record_cnt = 0
-  FasterCSV.foreach(echi_file) do |row|
-    if row != nil
-      @log.debug '<====================START RECORD ' + record_cnt.to_s + ' ====================>'
-      echi_record = EchiRecord.new
-      cnt = 0
-      @echi_schema["fields"].each do | field |
-        if field["type"] == "bool" || field["type"] == "bool_int"
-          case row[cnt]
-          when "0"
-            echi_record[field["name"]] = "N"
-          when "1"
-            echi_record[field["name"]] = "Y"
-          end
-          @log.debug field["name"] + ' == ' + row[cnt]
-        else
-          echi_record[field["name"]] = row[cnt]
-          if row[cnt] != nil
+  #Perform a transaction for each file, including the log table
+  #in order to commit as one atomic action upon success
+  EchiRecord.transaction do
+    @record_cnt = 0
+    FasterCSV.foreach(echi_file) do |row|
+      if row != nil
+        @log.debug '<====================START RECORD ' + @record_cnt.to_s + ' ====================>'
+        echi_record = EchiRecord.new
+        cnt = 0
+        @echi_schema["fields"].each do | field |
+          if field["type"] == "bool" || field["type"] == "bool_int"
+            case row[cnt]
+            when "0"
+              echi_record[field["name"]] = "N"
+            when "1"
+              echi_record[field["name"]] = "Y"
+            end
             @log.debug field["name"] + ' == ' + row[cnt]
+          else
+            echi_record[field["name"]] = row[cnt]
+            if row[cnt] != nil
+              @log.debug field["name"] + ' == ' + row[cnt]
+            end
           end
+          cnt += 1
         end
-        cnt += 1
+        echi_record.save
+        @log.debug '<====================STOP RECORD ' + @record_cnt.to_s + ' ====================>'
+        @record_cnt += 1
       end
-      echi_record.save
-      @log.debug '<====================STOP RECORD ' + record_cnt.to_s + ' ====================>'
-      record_cnt += 1
     end
   end
   
   #Move the file to the processed directory
-  destination_directory = @workingdirectory + '/../files/processed/' + filename
-  FileUtils.mv(echi_file, destination_directory)
+  FileUtils.mv(echi_file, @processeddirectory)
   
   if @config["echi_process_log"] == "Y"
     #Finish logging the details on the file
-    echi_log.records = record_cnt
+    echi_log.records = @record_cnt
     echi_log.processed_at = Time.now
     echi_log.save
   end
   
-  return record_cnt
+  return @record_cnt
 end
 
 require @workingdirectory + '/echi-converter/version.rb'