lib/automate-em.rb in automate-em-0.0.2 vs lib/automate-em.rb in automate-em-0.0.3

@@ -30,10 +30,11 @@
 # Library Files
 #
 require 'automate-em/constants.rb'
 require 'automate-em/utilities.rb'
 require 'automate-em/status.rb'
+require 'automate-em/module_core.rb'
 
 require 'automate-em/core/resolver_pool.rb'
 require 'automate-em/core/modules.rb'
 require 'automate-em/core/communicator.rb'
 require 'automate-em/core/system.rb'
@@ -120,20 +121,49 @@
 			EM.open_datagram_socket "0.0.0.0", 0, DatagramServer
 			
 			#
 			# Load the system based on the database
 			#
-			ControlSystem.all.each do |controller|
+			ControlSystem.update_all(:active => false)
+			ControlSystem.find_each do |controller|
 				EM.defer do
 					begin
 						System.logger.debug "Booting #{controller.name}"
-						System.new_system(controller, Rails.configuration.automate.log_level)
+						result = System.start(controller, Rails.configuration.automate.log_level)
+						if result == false
+							#
+							# TODO:: we need a class for handling failed starts
+							#
+							AutomateEm.print_error(AutomateEm::System.logger, e, {
+								:message => "System #{controller.name} failed to start (gracefully). It is now offline",
+								:level => Logger::WARN
+							})
+							controller.active = false
+							controller.save
+							@@scheduler.in '5m' do
+								System.start(controller, Rails.configuration.automate.log_level)
+							end
+						end
 					rescue => e
 						AutomateEm.print_error(AutomateEm::System.logger, e, {
-							:message => "Error during boot",
-							:level => Logger::FATAL
+							:message => "System #{controller.name} threw an error whilst starting. It is now offline",
+							:level => Logger::WARN
 						})
-						EventMachine::stop_event_loop
+						#
+						# Mark as offline, do not retry and email
+						#
+						begin
+							controller.active = false
+							controller.save
+							#
+							# TODO:: email admin about failure
+							#
+						rescue => e
+							AutomateEm.print_error(AutomateEm::System.logger, e, {
+								:message => "Error marking system as offline",
+								:level => Logger::ERROR
+							})
+						end
 					end
 				end
 			end
 			
 			#