class CodeRunner # Every code module defines a custom child class of CodeRunner::Run. This class will also include a module which customizes it to work on the current system. The result is a class which knows how to run and analyses the results of the given code on the given system. # # Every simulation that is carried out has an instance of this custom class created for it, and this object, known as a run, contains both the results and the input parameters pertaining to that simulation. All these runs are then stored in the variable run_list in a runner (an instance of the CodeRunner class). The result is that every run has a runner that it can talk to, and that runner has a set of runs that it can talk to. # # Every run has its own directory, where all the input and output files from the simulation are stored. CodeRunner data for the run is stored in this folder in the files code_runner_info.rb and code_runner_results.rb. # # As soon as the simulation is complete, CodeRunner call Run#process_directory to carry out any anlysis, and the results are stored in code_runner_results.rb. For speed CodeRunner also caches the data in the file .code_runner_run_data in binary format. This cache will be used after the initial analysis unless CodeRunner is specifically told to reanalyse this run. # # All input parameters and results are available during run time as instance variables of the run class. # # The class CodeRunner::Run itself defines a base set of methods which are added to by the code module. class Run include Log # # include CodeRunner::HeuristicRunMethods # class_vars = [:conditions, :sort, :sys, :debug, :script_folder, :necessary_system_runner_methods, :necessary_code_runner_methods, :recalc_all, :parallel] # @@ruby_command = "ruby" #redefine if necessary # @@input_file_extension = nil # @@use_file_name_as_run_name = nil # @@successful_trial_system = nil # # @@readout_real_run_list = true # @@print_out_real_run_list = true # @@readout_phantom_run_list = false # @@print_out_phantom_run_list = false # class_accessor :readout_real_run_list, :print_out_real_run_list, :readout_phantom_run_list, :print_out_phantom_run_list, class_accessor :current_status, :runner # Use the instance method #queue_status (defined in the system module) to put a list of current jobs obtained from the system into the class variable @@current_status def self.update_status(runner) if runner.no_run @@current_status = "" else #eputs 'Getting queue status...' @@current_status = new(runner).queue_status end # puts @@current_status end def gets #No reading from the command line thank you very much! $stdin.gets end def self.gets $stdin.gets end (SUBMIT_OPTIONS + [:code, :version, :readout_list, :naming_pars, :other_pars, :all, :ruby_command, :run_sys_name, :modlet, :executable]).each do |variable| #define accessors for class options and instance options # class_accessor(variable) attr_accessor variable # set(variable, nil) unless class_variables.include? ("@@" + variable).to_sym end class_accessor :run_sys_name @@run_sys_name = nil # @@necessary_class_variables.keys.each{|v| send(:attr_accessor, v)} # @runnmaxes = {} attr_accessor :maxes, :max_complete, :version, :code, :nprocs, :executable_name, :runner, :sys, :naming_pars, :code_runner_version, :real_id # Purely for testing purposes; see the test suite attr_accessor :run_test_flags # Access to a hash which is stored in the runner (not the run). The hash will persist while the runner is in the object space. # # E.g. # cache[:my_stuff_to_store] = something_to_store def cache @runner.cache[:runs] ||= {} @runner.cache[:runs][@id] ||= {} @runner.cache[:runs][@id] end # The hard cache persists after the current program ceases because # it is written in the .code_runner_run_data file. # It will disappear if the -a or -A flags are specified at any point # If you edit the hard cache you ''must'' call save afterwards # or your changes may not be kept def hard_cache @hard_cache ||={} @hard_cache end #def save_hard_cache #Dir.chdir(@directory) class RunClassPropertyFetcher def initialize(the_class) @my_class = the_class end def method_missing(method, value=nil) if method.to_s =~ /=$/ raise 'rcps should not be set outside class methods' @my_class.instance_variable_set("@"+method.to_s.sub(/=/, ''), value) else the_class = @my_class loop do # p the_class, method # p the_class.instance_variables # p the_class.instance_variables.map{|v| the_class.instance_variable_get(v)} return the_class.instance_variable_get("@"+method.to_s) if the_class.instance_variables.include?(("@"+method.to_s).to_sym) the_class = the_class.superclass return nil unless the_class end end end def [](prop) send(prop.to_sym) end # def []=(prop, value) # set(prop.to_sym, value) # end # end end # Access properties of the run class. These properties are stored as instance variables of the run class object. (Remember, in Ruby, every Class is an Object (and Object is a Class!)). # # E.g. # puts rcp.variables def self.rcp @rcp ||= RunClassPropertyFetcher.new(self) end # Calls Run.rcp def rcp self.class.rcp end # Create a new run. runner should be an instance of the class CodeRunner. def initialize(runner) logf('initialize') # raise "runner must be either a CodeRunner or a RemoteCoderunner: it is a #{runner.class}" unless [CodeRunner, RemoteCodeRunner].include? runner.class @runner = runner # raise CRFatal.new("Code not defined: #{CODE}") unless @@code and @@code.class == String and @@code =~ /\S/ @sys, @code = rcp.sys, rcp.code @naming_pars = rcp.naming_pars.dup raise CRFatal.new("@modlet not specified for #{self.class}") if rcp.modlet_required and not rcp.modlet # @modlet = @@modlet; @modlet_location = @@modlet_location # # @executable_location = executable_location # @@necessary_run_variables.each{|v,clas| instance_eval("@#{v} = nil")} # initialize_code_specific # @script_folder = @@script_folder # @recalc_all = @@recalc_all @wall_mins = @runner.wall_mins if @runner @smaxes = {}; @csmaxes = {}; @max_complete = {}; end # Here we redefine the inspect function p to raise an error if anything is written to standard out # while in server mode. This is because the server mode uses standard out to communicate # and writing to standard out can break it. All messages, debug information and so on, should always # be written to standard error. def p(*args) if @runner and @runner.server raise "Writing to stdout in server mode will break things!" else super(*args) end end # Here we redefine the function puts to raise an error if anything is written to standard out # while in server mode. This is because the server mode uses standard out to communicate # and writing to standard out can break it. All messages, debug information and so on, should always # be written to standard error. def puts(*args) if @runner and @runner.server raise "Writing to stdout in server mode will break things!" else super(*args) end end # Here we redefine the function print to raise an error if anything is written to standard out # while in server mode. This is because the server mode uses standard out to communicate # and writing to standard out can break it. All messages, debug information and so on, should always # be written to standard error. def print(*args) if @runner and @runner.server raise "Writing to stdout in server mode will break things!" else super(*args) end end # Analyse the directory of the run. This should be called from the directory where the files of the run are located. This method reads in the CodeRunner data already available in code_runner_info.rb and code_runner_results.rb, and then calls process_directory_code_specific which is defined in the code module. def process_directory # Clear the cache @runner.cache[:runs]||={} @runner.cache[:runs][@id] = {} logf(:process_directory) raise CRFatal.new("Something has gone horribly wrong: runner.class is #{@runner.class} instead of CodeRunner") unless @runner.class.to_s == "CodeRunner" begin @code_runner_version = Version.new(File.read('.code_runner_version.txt')) File.read('code_runner_info.rb') rescue Errno::ENOENT # version may be less than 0.5.1 when .code_runner_version.txt was introduced @code_runner_version = Version.new('0.5.0') end @directory = Dir.pwd @relative_directory = File.expand_path(Dir.pwd).sub(File.expand_path(@runner.root_folder) + '/', '') # p @directory @readme = nil if @code_runner_version < Version.new('0.5.1') begin update_from_version_0_5_0_and_lower rescue Errno::ENOENT => err # No code runner files were found unless @runner.heuristic_analysis puts err raise CRFatal.new("No code runner files found: suggest using heuristic analysis (flag -H if you are using the code_runner script)") end unless @runner.current_request == :traverse_directories @runner.requests.push :traverse_directories unless @runner.requests.include? :traverse_directories raise CRMild.new("can't begin heuristic analysis until there has been a sweep over all directories") # this will get rescued end @runner.increment_max_id @id = @runner.max_id @job_no = -1 run_heuristic_analysis end end read_info begin read_results if FileTest.exist? 'code_runner_results.rb' rescue NoMethodError, SyntaxError => err puts err puts 'Results file possibly corrupted for ' + @run_name end if methods.include? :get_run_status @status = get_run_status(@job_no, @@current_status) rescue :Unknown else @status ||= :Unknown end @running = (@@current_status =~ Regexp.new(@job_no.to_s)) ? true : false #logi '@@current_status', @@current_status, '@job_no', @job_no #logi '@running', @running process_directory_code_specific raise CRFatal.new("status must be one of #{PERMITTED_STATI.inspect}") unless PERMITTED_STATI.include? @status @max = {} write_results generate_phantom_runs save return self end # Read input parameters from the file code_runner_info.rb def read_info eval(File.read('code_runner_info.rb')).each do |key, value| set(key, value) end end # Read results from the file code_runner_results.rb def read_results return if @runner.recalc_all eval(File.read('code_runner_results.rb')).each do |key, value| set(key, value) end end # Write results to the file code_runner_results.rb def write_results logf(:write_results) Dir.chdir(@directory){File.open("code_runner_results.rb", 'w'){|file| file.puts results_file}} end # Return the text of the results file. def results_file logf(:results_file) return <>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> # # This is a syntactically correct Ruby file, which is used by CodeRunner. Please do not edit unless you know what you are doing. # Directory: #{@directory} # Runname: #{@run_name} # ID: #{@id} # Results: #{(rcp.results+rcp.run_info).inject({}){|hash, (var,type_co)| hash[var] = send(var); hash}.pretty_inspect} # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< EOF end # Return a line of data for printing to file in CodeRunner#readout, organised according to the run class property rcp.readout_list def data_string rcp.readout_list.inject(""){|str,var| str+"#{send(var)}code_runner_spacer"}.gsub(/\s/, '_').gsub(/code_runner_spacer/, "\t") + "\n" end # Used for the status with comments command. def comment_line "#{id}: #{@comment}" sprintf("%2d:%d %1s:%2.1f(%s) %3s%1s %s", @id, @job_no, @status.to_s[0,1], @run_time.to_f / 60.0, @nprocs.to_s, percent_complete, "%", @comment) end # Cache the run object in the file .code_runner_run_data def save logf(:save) raise CRFatal.new("Something has gone horribly wrong: runner.class is #{@runner.class} instead of CodeRunner") unless @runner.class.to_s == "CodeRunner" runner, @runner = @runner, nil @system_triers, old_triers = nil, @system_triers @phantom_runs.each{|run| run.runner = nil} if @phantom_runs # logi(self) Dir.chdir(@directory){File.open(".code_runner_run_data", 'w'){|file| file.puts Marshal.dump(self)}} @runner = runner @phantom_runs.each{|run| run.runner = runner} if @phantom_runs @system_triers = old_triers end # Load the run object from the file .code_runner_run_data def self.load(dir, runner) raise CRFatal.new("runner supplied in Run.load was not an instance of code runner; runner.class = #{runner.class}") unless runner.class.to_s == "CodeRunner" begin raise CRMild.new("No saved run data") unless FileTest.exist? (dir+"/.code_runner_run_data") run = Marshal.load(File.read(dir+"/.code_runner_run_data")) rescue ArgumentError => err raise err unless err.message =~ /undefined class/ #NB this means that all code_names have to contain only lowercase letters: # code, modlet = err.message.scan(/CodeRunner\:\:([A-Z][a-z0-9]+)((?:[A-Z]\w+)+)?Run/)[0] # code, modlet = err.message.scan(/CodeRunner\:\:([A-Z][a-z0-9_]+)(?:::([A-Z]\w+))?/)[0] # ep code, modlet; exit # code.gsub!(/([a-z0-9])([A-Z])/, '\1_\2') # modlet.gsub!(/([a-z0-9])([A-Z])/, '\1_\2') # ep err, modlet, code # runner.setup_run_class(code.downcase, modlet: modlet.downcase) # retry CodeRunner.repair_marshal_run_class_not_found_error(err) run = Marshal.load(File.read(dir+"/.code_runner_run_data")) end run.runner = runner raise CRFatal.new("Something has gone horribly wrong: runner.class is #{run.runner.class} instead of CodeRunner") unless run.runner.class.to_s == "CodeRunner" run.directory = dir run.phantom_runs.each{|r| runner.add_phantom_run(r)} if run.phantom_runs #@phantom_runs = [] return run end # Return the name of the defaults file currently in use. def defaults_file_name if @runner.defaults_file return "#{@runner.defaults_file}_defaults.rb" else return "#{rcp.code}_defaults.rb" end end # Return the folder where the default defaults file is located. def defaults_location if @runner.defaults_file location = ["#{SCRIPT_FOLDER}/code_modules/#@code/my_defaults_files", "#{SCRIPT_FOLDER}/code_modules/#@code/defaults_files"].find{|folder| FileTest.exist? folder and Dir.entries(folder).include? defaults_file_name} raise "Defaults file: #{defaults_file_name} not found" unless location return location else return "#{SCRIPT_FOLDER}/code_modules/#@code" end end # Return true if the run is completed, false otherwise def is_complete @status == :Complete end alias :ctd :is_complete # This function is a hook which is used by system modules. For runs it is defined as the id. def job_identifier id end class NoRunnerError < StandardError def new(mess) super("No Runner: a runner was needed for this Run method call "+mess) end end # Update instance variables using the given defaults file. Give warnings if the defaults file contains variables which are not simulation input parameters def evaluate_defaults_file(filename) text = File.read(filename) text.scan(/^\s*@(\w+)/) do var_name = $~[1].to_sym next if var_name == :defaults_file_description unless rcp.variables.include? var_name warning("---#{var_name}---, specified in #{File.expand_path(filename)}, is not a variable. This could be an error") end end instance_eval(text) end # This function set the input parameters of the run using the following sources in ascending order of priority: main defaults file (in the code module folder), local defaults file (in the local Directory), parameters (an inspected hash usually specified on the command line). def update_submission_parameters(parameters, start_from_defaults=true) logf(:update_submission_parameters) if start_from_defaults upgrade_defaults_from_0_5_0 if self.class.constants.include? :DEFAULTS_FILE_NAME_0_5_0 main_defaults_file = "#{defaults_location}/#{defaults_file_name}" main_defaults_file_text = File.read(main_defaults_file) evaluate_defaults_file(main_defaults_file) unless FileTest.exist?(defaults_file_name) main_defaults_file_text.gsub!(/^/, "#") header = <>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> # Code: #{rcp.code} # System: #{@sys} # Version: #{@version} # Nprocs: #{@nprocs} # Directory: #{Dir.pwd} # Runname: #{@run_name} # ID: #{@id} # #{rcp.modlet_required ? "Modlet:\t#{rcp.modlet}" : ""} # Classname: #{self.class.to_s} # #{@job_no ? "Job_No: #{@job_no}" : ""} # Parameters: #{(rcp.variables + rcp.run_info + [:version, :code, :modlet, :sys]).inject({}){|hash, var| hash[var] = send(var) unless (!send(var) and send(var) == nil); hash}.pretty_inspect} # Actual Command: # #{run_command} # <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< EOF end def set_modlet(modlet, folder=nil) logf(:set_modlet) rcp.modlet.sub!(/.*/, modlet) self.class.set_modlet(modlet, folder) end private :set_modlet def self.set_modlet(modlet, folder = nil) raise 'old method -- needs to be debugged' Log.log("self.set_modlet", self.to_s) class_eval(File.read("#{folder ? folder + "/" : ""}#{modlet}")) check_and_update end # aliold :inspect def inspect old, @runner = @runner, nil str = super @runner = old str end # aliold :pretty_inspect def pretty_print(q) old, @runner = @runner, nil str = q.pp_object(self) @runner = old str end # def pretty_print(q) # if /\(Kernel\)#/ !~ Kernel.instance_method(:method).bind(self).call(:inspect).inspect # q.text self.inspect # elsif /\(Kernel\)#/ !~ Kernel.instance_method(:method).bind(self).call(:to_s).inspect && instance_variables.empty? # q.text self.to_s # else # q.pp_object(self) # end # end # A hook for module developers def self.finish_setting_up_class end # ALL = [] # READOUT_LIST = [] # CODE = nil def self.check_and_update Log.logf(:check_and_update) # finish_setting_up_class # ep self.to_s, constants, self.ancestors # raise CRFatal.new("Code not defined (#{CODE.inspect}) in class #{self.to_s}") unless CODE and CODE.class == String and CODE =~ /\S/ NECESSARY_RUN_SYSTEM_METHODS.each do |method| raise CRFatal.new("#{method} not defined in #{SYS}_system_runner.rb") unless instance_methods.include?(method) end NECESSARY_RUN_CODE_METHODS.each do |method| raise CRFatal.new("#{method} not defined in #{self.class}_code_runner.rb") unless instance_methods.include?(method) end NECESSARY_RUN_CLASS_PROPERTIES.each do |v,class_list| # raise CRFatal.new("#{v} not defined") unless rcp[v] raise CRFatal.new("#{v} not defined correctly: class is #{rcp[v].class} instead of one of #{class_list.to_s}") unless class_list.include? rcp[v].class end @readout_list = (rcp.variables+rcp.results) unless rcp.readout_list # (variables+results).each{|v| const_get(:READOUT_LIST).push v} unless READOUT_LIST.size > 0 if rcp.variables_0_5_0 rcp.variables_0_5_0.dup.each do |par, info| #for backwards compatibility only rcp.variables_0_5_0[par] = info[0] if info.class == Array end end # Log.log(:@@variables0, @@variables[0]) @run_info = rcp.run_info || [] # Run info can optionally be defined in the code module. # ep @run_info @run_info = rcp.run_info + ([:job_no, :running, :id, :status, :sys, :is_phantom, :naming_pars, :run_name, :resubmit_id, :real_id, :phantom_runs, :parameter_hash, :output_file, :error_file] + SUBMIT_OPTIONS) #.each{|v| RUN_INFO.push v} unless RUN_INFO.include? :job_no @all = (rcp.variables + rcp.results + rcp.run_info) #.each{|v| ALL.push v} # ep "GOT HERE" (@all + [:directory, :run_name, :modlet, :relative_directory]).each{|var| send(:attr_accessor, var)} define_method(:output_file) do return @output_file if @output_file @output_file = super() end define_method(:error_file) do return @error_file if @error_file @error_file = super() end Dir.chdir(SCRIPT_FOLDER + "/system_modules") do @system_run_classes ||= Dir.entries(Dir.pwd).find_all{|file| file =~ /^[^\.].+\.rb$/}.inject([]) do |arr, file| #p Dir.pwd #p 'required', file, Dir.pwd require Dir.pwd + '/' + file # p CodeRunner.constants sys = file.sub(/\.rb$/, '') arr.push(add_a_child_class("#{sys.variable_to_class_name}Run")) arr[-1].send(:include, CodeRunner.const_get(sys.variable_to_class_name)) arr end end end def dup return self.class.new(@runner).learn_from(self) end def create_phantom @phantom_runs ||= [] new_run = dup new_run.is_phantom = true new_run.real_id = @id @runner.add_phantom_run(new_run) @phantom_runs.push new_run new_run end def learn_from(run) run.instance_variables.each do |var| # puts var # puts run.instance_variable_get(var) instance_variable_set(var,run.instance_variable_get(var)) # puts instance_variable_get(var) # rescue NoMethodError # next # end end self end def logiv instance_variables.each do |var| unless var == :@runner or var == :@system_triers log(var); logi(instance_variable_get(var)) end end end def recheck logf(:recheck) Dir.chdir(@directory) do # puts 'ackack' puts "Rechecking #@run_name" runner = @runner instance_variables.each{|var| instance_variable_set(var, nil) unless var == :@runner} begin File.delete("CODE_RUNNER_RUN_DATA") rescue Errno::ENOENT end begin File.delete("CODE_RUNNER_RESULTS") rescue Errno::ENOENT end process_directory save end end def generate_phantom_runs end def generate_combined_ids(type) raise CRFatal.new("Can't call generate_combined_ids from a run") end # @@maxes = {} # @@cmaxes = {} def max(variable, complete=false) #does this run have the maximum value of this variable raise ArgumentError.new("complete must be true or false") unless [TrueClass, FalseClass].include? complete.class @runner.generate_combined_ids ids = @runner.combined_ids if complete ids = ids.find_all{|id| @runner.combined_run_list[id].status == :Complete} max_id = @runner.cmaxes[variable] ||= ids.sort_by{|id| @runner.combined_run_list[id].send(variable)}[-1] else max_id = @runner.maxes[variable] ||= ids.sort_by{|id| @runner.combined_run_list[id].send(variable)}[-1] end return @runner.combined_run_list[max_id].send(variable) == send(variable) end # # @@mins = {} # @@cmins = {} def min(variable, complete=false) #does this run have the minimum value of this variable raise ArgumentError.new("complete must be true or false") unless [TrueClass, FalseClass].include? complete.class @runner.generate_combined_ids ids = @runner.combined_ids if complete ids = ids.find_all{|id| @runner.combined_run_list[id].status == :Complete} min_id = @runner.cmins[variable] ||= ids.sort_by{|id| @runner.combined_run_list[id].send(variable)}[0] else min_id = @runner.mins[variable] ||= ids.sort_by{|id| @runner.combined_run_list[id].send(variable)}[0] end return @runner.combined_run_list[min_id].send(variable) == send(variable) end # @@fmaxes = {} # @@cfmaxes = {} # # Does this run have the maximum value of this variable to be found amoung the filtered runs? # # def fmax(variable, complete = false) # raise ArgumentError.new("complete must be true or false") unless [TrueClass, FalseClass].include complete.class # @runner.generate_combined_ids # ids = @runner.filtered_ids # o^o-¬ # if complete # ids = ids.find_all{|id| @runner.combined_run_list[id].status == :Complete} # max_id = @@cfmaxes[variable] ||= ids.sort_by{|id| @runner.combined_run_list[id].send(variable)}[-1] # else # max_id = @@fmaxes[variable] ||= ids.sort_by{|id| @runner.combined_run_list[id].send(variable)}[-1] # end # return @runner.combined_run_list[max_id].send(variable) == send(variable) # end # def smax(variable,sweep=nil, complete=nil) logf(:max) sweep ||= variable if complete @csmaxes[variable] ||= {} max_id = @csmaxes[variable][sweep] = @runner.get_max(self, variable,sweep, complete) else @smaxes[variable] ||= {} max_id = @smaxes[variable][sweep] = @runner.get_max(self, variable,sweep, complete) end return @runner.combined_run_list[max_id].send(variable) == send(variable) end def smin(variable,sweep=nil, complete=nil) logf(:min) sweep ||= variable @smins ||= {} @csmins ||= {} if complete @csmins[variable] ||= {} min_id = @csmins[variable][sweep] = @runner.get_min(self, variable,sweep, complete) else @smins[variable] ||= {} min_id = @smins[variable][sweep] = @runner.get_min(self, variable,sweep, complete) end return @runner.combined_run_list[min_id].send(variable) == send(variable) end # aliold :_dump # def _dump(*args) # @runner, runner = nil, @runner # ans = super(*args) # @runner = runner # return ans # end def executable_name File.basename(@executable||=@runner.executable) end def executable_location File.dirname(@executable||=@runner.executable) end def update_in_queue unless @status == :Queueing raise 'Can only updated runs whose status is :Queueing' end unless methods.include? :batch_script_file raise 'Can only update runs which have been submitted using a batch script file' end old_run_name = @run_name generate_run_name new_run_name = @run_name #@run_name = old_run_name unless FileTest.exist?(filename = @directory + '/' + batch_script_file) or FileTest.exist?(filename = @runner.root_folder + '/' + batch_script_file) raise 'Could not find batch_script_file' end old_batch_script=File.read(filename) eputs old_batch_script eputs old_batch_script.gsub(Regexp.new(Regexp.escape(old_run_name)), new_run_name) ep Regexp.new(Regexp.escape(old_run_name)) File.open(filename, 'w'){|file| file.puts old_batch_script.gsub(Regexp.new(Regexp.escape(old_run_name)), new_run_name)} generate_input_file #throw(:done) write_info end # A hook... default is to do nothing def self.modify_job_script(runner, runs, script) return script end # A hook... a string which gets put into the job # script. Used to load modules, configure the run # time environment for a given code. Default # is to return an empty string. def code_run_environment "" end end end