lib/echi-converter.rb in echi-converter-0.3.3 vs lib/echi-converter.rb in echi-converter-0.3.4
- old
+ new
@@ -3,10 +3,12 @@
require 'faster_csv'
require 'net/ftp'
require 'net/smtp'
require 'fileutils'
require 'uuidtools'
+require 'thread'
+require $workingdir + '/ftp_fetcher.rb'
class Logger
#Change the logging format to include a timestamp
def format_message(severity, timestamp, progname, msg)
"#{timestamp} (#{$$}) #{msg}\n"
@@ -30,11 +32,11 @@
when 'DEBUG'
ActiveRecord::Base.logger.level = Logger::DEBUG
end
begin
ActiveRecord::Base.establish_connection(YAML::load(File.open(databaseconfig)))
- @log.info "Successfully connected to the database"
+ @log.info "Initialized the database"
rescue => err
@log.fatal "Could not connect to the database - " + err
send_email_alert "DATABASE"
end
end
@@ -95,10 +97,74 @@
end
return directory_month
end
+ #Method to get FTP files
+ def get_ftp_files
+ filelist_fetcher = FtpFetcher.new
+ filequeue = filelist_fetcher.fetch_list @log
+
+ if filequeue == nil
+ return -1
+ end
+
+ if $config["max_ftp_sessions"] > 1 && filequeue.length > 4
+ if $config["max_ftp_sessions"] > filequeue.length
+ @log.info "Using " + filequeue.length.to_s + " ftp sessions to fetch files"
+ my_threads = []
+ cnt = 0
+ while cnt < filequeue.length
+ my_threads << Thread.new do
+ fetcher = Fetcher.new
+ result = fetcher.fetch_ftp_files filequeue, @log
+ end
+ cnt += 1
+ end
+ my_threads.each { |aThread| aThread.join }
+ else
+ @log.info "Using " + $config["max_ftp_sessions"].to_s + " ftp sessions to fetch files"
+ my_threads = []
+ cnt = 0
+ while cnt < $config["max_ftp_sessions"]
+ my_threads << Thread.new do
+ fetcher = FtpFetcher.new
+ result = fetcher.fetch_ftp_files filequeue, @log
+ end
+ cnt += 1
+ end
+ my_threads.each { |aThread| aThread.join }
+ end
+ else
+ @log.info "Using a single ftp session to fetch the files"
+ fetcher = FtpFetcher.new
+ result = fetcher.fetch_ftp_files filequeue, @log
+ end
+ if result == false
+ send_email_alert "FTP"
+ end
+ end
+
+ #Method to write to the log table
+ def log_processed_file type, filedata
+ begin
+ echi_log = EchiLog.new
+ echi_log.filename = filedata["name"]
+ if type == 'BINARY'
+ echi_log.filenumber = filedata["number"]
+ echi_log.version = filedata["version"]
+ end
+ echi_log.records = filedata["cnt"]
+ echi_log.processedat = Time.now
+ echi_log.save
+ rescue => err
+ @log.info "Error creating ECHI_LOG entry - " + err
+ return -1
+ end
+ return 0
+ end
+
#Method for parsing the various datatypes from the ECH file
def dump_binary type, length
case type
when 'int'
#Process integers, assigning appropriate profile based on length
@@ -147,290 +213,235 @@
#Read header information first
filenumber = dump_binary 'int', 4
@log.debug "File_number " + filenumber.to_s
fileversion = dump_binary 'int', 4
@log.debug "Version " + fileversion.to_s
-
- if $config["echi_process_log"] == "Y"
- #Log the file
- begin
- echi_log = EchiLog.new
- echi_log.filename = filename
- echi_log.filenumber = filenumber
- echi_log.version = fileversion
- rescue => err
- @log.info "Error creating ECHI_LOG entry - " + err
- end
- end
- #Perform a transaction for each file, including the log table
- #in order to commit as one atomic action upon success
- EchiRecord.transaction do
- bool_cnt = 0
- @record_cnt = 0
- while @binary_file.eof == FALSE do
- @log.debug '<====================START RECORD ' + @record_cnt.to_s + ' ====================>'
- echi_record = EchiRecord.new
- @echi_schema["fields"].each do | field |
- #We handle the 'boolean' fields differently, as they are all encoded as bits in a single 8-bit byte
- if field["type"] == 'bool'
- if bool_cnt == 0
- bytearray = dump_binary field["type"], field["length"]
- end
- #Ensure we parse the bytearray and set the appropriate flags
- #We need to make sure the entire array is not nil, in order to do Y/N
- #if Nil we then set all no
- if bytearray != nil
- if bytearray.slice(bool_cnt,1) == 1
- value = 'Y'
- else
+ begin
+ #Perform a transaction for each file, including the log table
+ #in order to commit as one atomic action upon success
+ EchiRecord.transaction do
+ bool_cnt = 0
+ @record_cnt = 0
+ while @binary_file.eof == FALSE do
+ @log.debug '<====================START RECORD ' + @record_cnt.to_s + ' ====================>'
+ echi_record = EchiRecord.new
+ @echi_schema["echi_records"].each do | field |
+ #We handle the 'boolean' fields differently, as they are all encoded as bits in a single 8-bit byte
+ if field["type"] == 'bool'
+ if bool_cnt == 0
+ bytearray = dump_binary field["type"], field["length"]
+ end
+ #Ensure we parse the bytearray and set the appropriate flags
+ #We need to make sure the entire array is not nil, in order to do Y/N
+ #if Nil we then set all no
+ if bytearray != nil
+ if bytearray.slice(bool_cnt,1) == 1
+ value = 'Y'
+ else
+ value = 'N'
+ end
+ else
value = 'N'
end
- else
- value = 'N'
+ bool_cnt += 1
+ if bool_cnt == 8
+ bool_cnt = 0
+ end
+ else
+ #Process 'standard' fields
+ value = dump_binary field["type"], field["length"]
+ @log.debug field["name"] + " { type => #{field["type"]} & length => #{field["length"]} } value => " + value.to_s
end
- bool_cnt += 1
- if bool_cnt == 8
- bool_cnt = 0
- end
- else
- #Process 'standard' fields
- value = dump_binary field["type"], field["length"]
- @log.debug field["name"] + " { type => #{field["type"]} & length => #{field["length"]} } value => " + value.to_s
+ echi_record[field["name"]] = value
end
- echi_record[field["name"]] = value
- end
- echi_record.save
+ echi_record.save
- #Scan past the end of line record
- @binary_file.read(1)
- @log.debug '<====================STOP RECORD ' + @record_cnt.to_s + ' ====================>'
- @record_cnt += 1
+ #Scan past the end of line record
+ @binary_file.read(1)
+ @log.debug '<====================STOP RECORD ' + @record_cnt.to_s + ' ====================>'
+ @record_cnt += 1
+ end
+ @binary_file.close
end
- @binary_file.close
+ rescue => err
+ @log.info "Error processing ECHI file - " + err
end
-
+
#Move the file to the processed directory
FileUtils.mv(echi_file, @processeddirectory)
if $config["echi_process_log"] == "Y"
- #Finish logging the details on the file
- begin
- echi_log.records = @record_cnt
- echi_log.processedat = Time.now
- echi_log.save
- rescue => err
- @log.info "Error inserting ECHI_LOG entry - " + err
- end
+ log_processed_file "BINARY", { "name" => filename, "number" => filenumber, "version" => fileversion, "cnt" => @record_cnt }
end
return @record_cnt
end
-
- def connect_ftpsession
- #Open ftp connection
- begin
- if $config["echi_connect_type"] == 'ftp'
- ftp_session = Net::FTP.new($config["echi_host"])
- ftp_session.login $config["echi_username"], $config["echi_password"]
- @log.info "Successfully connected to the ECHI FTP server"
- else
- #Stub for possible SSH support in the future
- #session = Net::SSH.start(config["echi_host"], config["echi_port"], config["echi_username"], config["echi_password"])
- @log.fatal "SSH currently not supported, please use FTP for accessing the ECHI server"
- exit
- end
- rescue => err
- @log.fatal "Could not connect with the FTP server - " + err
- send_email_alert "FTP"
- return -1
- end
- return ftp_session
- end
-
- #Connect to the ftp server and fetch the files each time
- def fetch_ftp_files
- attempts = 0
- ftp_session = -1
- while ftp_session == -1 do
- ftp_session = connect_ftpsession
- if ftp_session == -1
- sleep 5
- end
- attempts += 1
- if $config["echi_ftp_retry"] == attempts
- ftp_session = 0
- end
- end
- if ftp_session != 0
- begin
- if $config["echi_ftp_directory"] != nil
- ftp_session.chdir($config["echi_ftp_directory"])
- end
- files = ftp_session.list('chr*')
-
- #Also fetch the agname.dat file if it is configured to be processed
- if $config["echi_update_agent_data"] == "Y"
- files = files + ftp_session.list("agname.dat")
- end
-
- file_cnt = 0
- files.each do | file |
- file_data = file.split(' ')
-
- local_filename = $workingdir + '/../files/to_process/' + file_data[8]
- ftp_session.getbinaryfile(file_data[8], local_filename)
- if $config["echi_ftp_delete"] == 'Y'
- begin
- ftp_session.delete(file_data[8])
- rescue => err
- @log.fatal err
- end
- end
- file_cnt += 1
- end
- ftp_session.close
- rescue => err
- @log.fatal "Could not fetch from ftp server - " + err
- end
- end
- return
- end
def process_ascii filename
echi_file = $workingdir + "/../files/to_process/" + filename
- if $config["echi_process_log"] == "Y"
- #Log the file
- begin
- echi_log = EchiLog.new
- echi_log.filename = filename
- #echi_log.filenumber = filenumber
- #echi_log.version = fileversion
- rescue => err
- @log.info "Error creating ECHI_LOG entry - " + err
- end
- end
-
- #Perform a transaction for each file, including the log table
- #in order to commit as one atomic action upon success
- EchiRecord.transaction do
- @record_cnt = 0
- FasterCSV.foreach(echi_file) do |row|
- if row != nil
- @log.debug '<====================START RECORD ' + @record_cnt.to_s + ' ====================>'
- echi_record = EchiRecord.new
- cnt = 0
- @echi_schema["fields"].each do | field |
- if field["type"] == "bool" || field["type"] == "bool_int"
- case row[cnt]
- when "0"
- echi_record[field["name"]] = "N"
- when "1"
- echi_record[field["name"]] = "Y"
- end
- @log.debug field["name"] + ' == ' + row[cnt]
- else
- echi_record[field["name"]] = row[cnt]
- if row[cnt] != nil
+ begin
+ #Perform a transaction for each file, including the log table
+ #in order to commit as one atomic action upon success
+ EchiRecord.transaction do
+ @record_cnt = 0
+ FasterCSV.foreach(echi_file) do |row|
+ if row != nil
+ @log.debug '<====================START RECORD ' + @record_cnt.to_s + ' ====================>'
+ echi_record = EchiRecord.new
+ cnt = 0
+ @echi_schema["echi_records"].each do | field |
+ if field["type"] == "bool" || field["type"] == "bool_int"
+ case row[cnt]
+ when "0"
+ echi_record[field["name"]] = "N"
+ when "1"
+ echi_record[field["name"]] = "Y"
+ end
@log.debug field["name"] + ' == ' + row[cnt]
+ else
+ echi_record[field["name"]] = row[cnt]
+ if row[cnt] != nil
+ @log.debug field["name"] + ' == ' + row[cnt]
+ end
end
+ cnt += 1
end
- cnt += 1
+ echi_record.save
+ @log.debug '<====================STOP RECORD ' + @record_cnt.to_s + ' ====================>'
+ @record_cnt += 1
end
- echi_record.save
- @log.debug '<====================STOP RECORD ' + @record_cnt.to_s + ' ====================>'
- @record_cnt += 1
end
end
+ rescue => err
+ @log.info "Error processing ECHI file - " + err
end
#Move the file to the processed directory
FileUtils.mv(echi_file, @processeddirectory)
if $config["echi_process_log"] == "Y"
- #Finish logging the details on the file
- begin
- echi_log.records = @record_cnt
- echi_log.processedat = Time.now
- echi_log.save
- rescue => err
- @log.info "Error inserting ECHI_LOG entry - " + err
- end
+ log_processed_file nil, { "name" => filename, "cnt" => @record_cnt }
end
return @record_cnt
end
- def insert_agent_data field
+ def insert_dat_data tablename, row
begin
- echi_agent = EchiAgent.new
- echi_agent.group_id = field[0]
- echi_agent.login_id = field[1]
- echi_agent.name = field[2]
- echi_agent.save
+ case tablename
+ when "echi_agents"
+ echi_dat_record = EchiAgent.new
+ when "echi_aux_reasons"
+ echi_dat_record = EchiAuxReason.new
+ when "echi_cwcs"
+ echi_dat_record = EchiCwc.new
+ when "echi_vdns"
+ echi_dat_record = EchiVdn.new
+ end
+ cnt = 0
+ @echi_schema[tablename].each do | field |
+ echi_dat_record[field["name"]] = row[cnt]
+ cnt += 1
+ end
+ echi_dat_record.save
rescue => err
- @log.info "Unable to insert agent record - " + err
+ @log.info "Unable to insert " + tablename + " file record - " + err
end
end
- #Method to insert data into 'echi_agents' based on agname.dat
- def process_agent_data
- agent_file = $workingdir + "/../files/to_process/agname.dat"
-
- if File.exists?(agent_file)
- EchiAgent.transaction do
- @record_cnt = 0
- File.open(agent_file).each do |row|
- if row != nil
- field = row.rstrip.split('|')
- @log.debug '<====================START AGENT RECORD ' + @record_cnt.to_s + ' ====================>'
- agent = EchiAgent.find(:first, :conditions => [ "login_id = ? AND group_id = ?", field[1], field[0]])
- if agent != nil
- if agent.name != field[2]
- agent.name = field[2]
- agent.update
- @record_cnt += 1
- @log.debug "Updated record - " + field.inspect
- else
- @log.debug "No update required for - " + field.inspect
- end
- else
- insert_agent_data field
- @record_cnt += 1
- @log.debug "Inserted new record - " + field.inspect
- end
- end
- @log.debug '<====================STOP AGENT RECORD ' + @record_cnt.to_s + ' ====================>'
+ #Process the appropriate table name
+ def process_proper_table file
+ @record_cnt = 0
+ process_file = File.open(file["filename"])
+ process_file.each do |row|
+ if row != nil
+ field = row.rstrip.split('|')
+ @log.debug '<====================START ' + file["name"] + ' RECORD ' + @record_cnt.to_s + ' ====================>'
+ case file["name"]
+ when "echi_agents"
+ record = EchiAgent.find(:first, :conditions => [ "login_id = ? AND group_id = ?", field[1], field[0]])
+ when "echi_aux_reasons"
+ record = EchiAuxReason.find(:first, :conditions => [ "aux_reason = ? AND group_id = ?", field[1], field[0]])
+ when "echi_cwcs"
+ record = EchiCwc.find(:first, :conditions => [ "cwc = ? AND group_id = ?", field[1], field[0]])
+ when "echi_vdns"
+ record = EchiVdn.find(:first, :conditions => [ "vdn = ? AND group_id = ?", field[1], field[0]])
+ end
+ if record != nil
+ if record.name != field[2]
+ record.name = field[2]
+ record.update
+ @record_cnt += 1
+ @log.debug "Updated record - " + field.inspect
+ else
+ @log.debug "No update required for - " + field.inspect
end
+ else
+ insert_dat_data file["name"], field
+ @record_cnt += 1
+ @log.debug "Inserted new record - " + field.inspect
end
- @agent_file_processed = Time.now
- #Move the file to the processed directory
- begin
- agname_new_filename = "agname_" + UUID.timestamp_create.to_s + ".dat"
- target_file = @processeddirectory + "/" + agname_new_filename
- FileUtils.mv(agent_file, target_file)
- rescue => err
- @log.info "Issue with agname_*.dat filename - " + err
- end
- if $config["echi_process_log"] == "Y"
- #Log the file
- begin
- echi_log = EchiLog.new
- echi_log.filename = agname_new_filename
- #echi_log.filenumber = filenumber
- #echi_log.version = fileversion
- #Finish logging the details on the file
- echi_log.records = @record_cnt
- echi_log.processedat = Time.now
- echi_log.save
- rescue => err
- @log.info "Error creating ECHI_LOGS entry - " + err
+ end
+ @log.debug '<====================STOP ' + file["name"] + ' RECORD ' + @record_cnt.to_s + ' ====================>'
+ end
+ process_file.close
+
+ case file["name"]
+ when "echi_agents"
+ filename_elements = $config["echi_agent_dat"].split(".")
+ when "echi_aux_reasons"
+ filename_elements = $config["echi_aux_rsn_dat"].split(".")
+ when "echi_cwcs"
+ filename_elements = $config["echi_cwc_dat"].split(".")
+ when "echi_vdns"
+ filename_elements = $config["echi_vdn_dat"].split(".")
+ end
+ new_filename = filename_elements[0] + "_" + UUID.timestamp_create.to_s + "." + filename_elements[1]
+ target_file = @processeddirectory + "/" + new_filename
+ begin
+ FileUtils.mv(file["filename"], target_file)
+ if $config["echi_process_log"] == "Y"
+ log_processed_file nil, { "name" => new_filename, "cnt" => @record_cnt }
+ end
+ rescue => err
+ @log.info "Unable to move processed file - " + err
+ end
+ end
+
+ #Method to insert data into 'echi_agents' based on agname.dat
+ def process_dat_files
+ dat_files = Array.new
+ dat_files[0] = { "name" => "echi_agents", "filename" => $workingdir + "/../files/to_process/" + $config["echi_agent_dat"] }
+ dat_files[1] = { "name" => "echi_aux_reasons", "filename" => $workingdir + "/../files/to_process/" + $config["echi_aux_rsn_dat"] }
+ dat_files[2] = { "name" =>"echi_cwcs", "filename" => $workingdir + "/../files/to_process/" + $config["echi_cwc_dat"] }
+ dat_files[3] = { "name" =>"echi_vdns", "filename" => $workingdir + "/../files/to_process/" + $config["echi_vdn_dat"] }
+
+ dat_files.each do |file|
+ if File.exists?(file["filename"])
+ case file["name"]
+ when "echi_agents"
+ EchiAgent.transaction do
+ process_proper_table file
end
+ when "echi_aux_reasons"
+ EchiAuxReason.transaction do
+ process_proper_table file
+ end
+ when "echi_cwcs"
+ EchiCwc.transaction do
+ process_proper_table file
+ end
+ when "echi_vdns"
+ EchiVdn.transaction do
+ process_proper_table file
+ end
end
end
end
+ end
require $workingdir + '/echi-converter/version.rb'
end
\ No newline at end of file