$stderr.puts "CodeRunner (c) 2009 Edmund Highcock. Loading..." unless $has_put_startup_message_for_code_runner if RUBY_VERSION.to_f < 1.9 raise "Ruby version 1.9 or greater required (current version is #{RUBY_VERSION})" end # Basic class setup require 'rubygems' require "rubyhacks" #Create the coderunner config directory if it doesn't exist FileUtils.makedirs(ENV['HOME'] + "/.coderunner") class CodeRunner COMMAND_FOLDER = Dir.pwd SCRIPT_FOLDER = File.dirname(File.expand_path(__FILE__)) + '/coderunner' #i.e. where this script is if ENV['CODE_RUNNER_OPTIONS'] GLOBAL_OPTIONS = eval(ENV['CODE_RUNNER_OPTIONS']) # global options are set by the environment but some can be changed. else GLOBAL_OPTIONS = {} end SYS = (GLOBAL_OPTIONS[:system] or ENV['CODE_RUNNER_SYSTEM'] or ENV['SYSTEM'] or "generic_linux") require SCRIPT_FOLDER + "/system_modules/#{SYS}.rb" SYSTEM_MODULE = const_get(SYS.variable_to_class_name) include SYSTEM_MODULE class << self include SYSTEM_MODULE end # begin # SYSTEM_MODULE.configure_environment # rescue NoMethodError # end @@sys = SYS def gets #No reading from the command line thank you very much! $stdin.gets end def self.gets $stdin.gets end end # CodeRunner::SCRIPT_FOLDER = File.dirname(File.expand_path(__FILE__)) #i.e. where this script is $JCODE = 'U' ######################################### # Load Libraries # ######################################### $stderr.print 'Loading libraries...' unless $has_put_startup_message_for_code_runner ################################# #read this if you are puzzled by # some non-standard use of ruby ################################ #require CodeRunner::SCRIPT_FOLDER + "/box_of_tricks.rb" ################################ require "getoptlong" require "thread" require "fileutils" require "drb" require "test/unit/assertions" require 'parallelpipes' require 'find' begin require 'hostmanager' rescue LoadError eprint 'no hostmanager.' end begin require "rubygems" require "rbgsl" require "gsl_extras" #require CodeRunner::SCRIPT_FOLDER + "/gsl_tools.rb" rescue LoadError $stderr.puts "Warning: could not load rbgsl; limited functionality" end #require CodeRunner::SCRIPT_FOLDER + "/gnuplot.rb" require "graphkit" CodeRunner::GraphKit = GraphKit # Backwards compatibility # load 'gnuplot' load CodeRunner::SCRIPT_FOLDER + "/feedback.rb" eprint '.' unless $has_put_startup_message_for_code_runner load CodeRunner::SCRIPT_FOLDER + "/test.rb" eprint '.' unless $has_put_startup_message_for_code_runner #load CodeRunner::SCRIPT_FOLDER + "/input_file_generator.rb" #eprint '.' unless $has_put_startup_message_for_code_runner load CodeRunner::SCRIPT_FOLDER + "/long_regexen.rb" eprint '.' unless $has_put_startup_message_for_code_runner load CodeRunner::SCRIPT_FOLDER + "/version.rb" eprint '.' unless $has_put_startup_message_for_code_runner load CodeRunner::SCRIPT_FOLDER + "/heuristic_run_methods.rb" eprint '.' unless $has_put_startup_message_for_code_runner #load CodeRunner::SCRIPT_FOLDER + "/code_runner_version.rb" #eprint '.' unless $has_put_startup_message_for_code_runner load CodeRunner::SCRIPT_FOLDER + "/fortran_namelist.rb" eprint '.' unless $has_put_startup_message_for_code_runner load CodeRunner::SCRIPT_FOLDER + "/fortran_namelist_c.rb" eprint '.' unless $has_put_startup_message_for_code_runner CodeRunner::CODE_RUNNER_VERSION = Version.new(Gem.loaded_specs['coderunner'].version.to_s) rescue "test" CodeRunner::GLOBAL_BINDING = binding Log.log_file = nil class CodeRunner #################################### # Commmand line processing # #################################### # Here are all the methods that map the command line invocation into the correct class method call COMMAND_LINE_FLAGS_WITH_HELP = [ ["--recalc-all", "-A", GetoptLong::NO_ARGUMENT, %[Causes each directory to be reprocessed, rather than reading the cache of data. Its exact effect depends on the code module being used. By convention it implies that ALL data analysis will be redone.]], ["--reprocess-all", "-a", GetoptLong::NO_ARGUMENT, %[Causes each directory to be reprocessed, rather than reading the cache of data. Its exact effect depends on the code module being used. By convention it implies that VERY LITTLE data analysis will be redone.]], ["--code", "-C", GetoptLong::REQUIRED_ARGUMENT, %[The code that is being used for simulations in this folder. This string must correspond to a code module supplied to CodeRunner. It usually only needs to be specified once as it will be stored as a default in the folder.]], ["--comment", "-c", GetoptLong::REQUIRED_ARGUMENT, %[A comment about the submitted run.]], ["--debug", "-d", GetoptLong::NO_ARGUMENT, %[Submit the simulation to the debug queue. This usually only has meaning on HPC systems. It does not mean debug CodeRunner!]], ["--defaults-file", "-D", GetoptLong::REQUIRED_ARGUMENT, %[Specify a defaults file to be used when submitting runs. The name should correspond to file named something like '_defaults.rb' in the correct folders within the code module. Every time a different defaults file is specified, a local copy of that defaults file is stored in the root folder. This local copy can be edited and all runs will use the local copy to get defaults. CodeRunner will never overwrite the local copy.]], ["--film-options", "-F", GetoptLong::OPTIONAL_ARGUMENT, %[Specify a hash of options when making films. The most important one is fa (frame array). For example -F '{fa: [0, 200]}'. For all possible options see the CodeRunner method make_film_from_lists.]], ["--conditions", "-f", GetoptLong::REQUIRED_ARGUMENT, %[A string specifying conditions used to filter runs. This filter is used in a variety of circumstances, for example when printing out the status, plotting graphs etc. Example: '@height == 10 and @width = 2.2 and @status==:Complete'.]], ["--run-graph", "-g", GetoptLong::REQUIRED_ARGUMENT, %[Specify a run_graphkit to plot. A run_graphkit is one that is plotted for an individual run. The run graphkits available depend on the code module. The syntax is graphkit shorthand:\n -g '[ ; [ ; [ ; ] ] ]'\n where conditions (i.e. filter) and sort will override the -f and -O flags respectively. The -g flag can be specified multiple times, which will plot multiple graphs on the same page.]], ["--graph", "-G", GetoptLong::REQUIRED_ARGUMENT, %[Specify a graphkit to plot. A graphkit combines data for every filtered run. The syntax is graphkit shorthand:\n -G '[ : [ : ] ] ] [ ; [ ; [ ; ] ] ]'\n where conditions (i.e. filter) and sort will override the -f and -O flags respectively. etc are strings which can be evaluated by the runs. The -G flag can be specified multiple times, which will plot multiple graphs on the same page. For example\n -G 'width : 2*height ; {} ; depth == 2 ; width'\n will plot twice the height against the width for every run where the depth is equal to 2, and will order the data points by width.]], ["--heuristic-analysis", "-H", GetoptLong::NO_ARGUMENT, %[Should be specified whenever CodeRunner is being used to analyse simulations which did not originally submit (and which will therefore not have the usual CodeRunner meta data stored with them).] ], ["--use-phantom", "-h", GetoptLong::OPTIONAL_ARGUMENT, %[Specify whether to use real or phantom runs]], ["--just", "-j", GetoptLong::REQUIRED_ARGUMENT, %[Specify individual run ids. For example -j 45,63,128 is shorthand for -f 'id==45 or id==63 or id==128']], ["--job_chain", "-J", GetoptLong::NO_ARGUMENT, %[Chain multiple simulations into one batch/submission job. Most useful for HPC systems.]], ["--skip-similar-jobs-off", "-k", GetoptLong::NO_ARGUMENT, %[Normally CodeRunner will not submit a run whose input parameters identical to a previous run (to avoid wasting computer time). Specifying the flag will override that behaviour and force submission of an identical run.]], ["--loop", "-l", GetoptLong::NO_ARGUMENT, %[Used with the status command. Keep continually printing out live status information.]], ["--multiple-processes", "-M", GetoptLong::REQUIRED_ARGUMENT], ["--modlet", "-m", GetoptLong::REQUIRED_ARGUMENT, %[Specify the modlet to be used in the current folder. Only needs to be specified once as it will be stored as a default.]], ["--no-run", "-N", GetoptLong::NO_ARGUMENT, %[On some machines getting a list of currently running jobs takes a long time. Specifying this flag tells CodeRunner that you definitely know that no runs in the folder are still queueing or running. Do not specify it if there are still running jobs as it will cause their statuses to be updated incorrectly.]], ["--nprocs", "-n", GetoptLong::REQUIRED_ARGUMENT, %[A string specifying the processor layout for the simulation. For example -n 46x4 means use 46 nodes with four processors per node. In the case of a personal computer something like -n 2 is more likely. The default is 1]], ["--sort", "-O", GetoptLong::REQUIRED_ARGUMENT, %[Specify the sort order for the runs. Used for a variety of commands, for example status. It is a string of semicolon separated sort keys: for example -O height;width will sort the runs by height and then width.]], ["--project", "-P", GetoptLong::REQUIRED_ARGUMENT, %[Specify the project to be used for billing purposes. Only necessary on some systems.]], ["--parameters", "-p", GetoptLong::REQUIRED_ARGUMENT, %[A hash of parameters for the simulation. For example -p '{height: 20, width: 2.3}'. These parameters will override the defaults in the local defaults file.]], ["--no-auto-create-runner", "-q", GetoptLong::NO_ARGUMENT, %[Used for interactive mode when you don't want CodeRunner to analyse the current directory.]], ["--terminal-size", "-t", GetoptLong::REQUIRED_ARGUMENT, %[Specify the terminal size for situations where CodeRunner cannot work it out: -t '[rows, cols]' (square brackets are part of the syntax)]], ["--test-submission", "-T", GetoptLong::NO_ARGUMENT, %[Don't actually submit the run, but exit after printing out the run parameters and generating any input files necessary.]], ["--use-large-cache-but-recheck-incomplete", "-u", GetoptLong::NO_ARGUMENT, %[Use the large cache for speed, but check any runs whose status is not :Complete or :Failed.]], ["--use-large-cache", "-U", GetoptLong::NO_ARGUMENT, %[Use the large cache for speed. No run data will be updated.]], ["--version", "-v", GetoptLong::REQUIRED_ARGUMENT, %[Specify the version of the simulation code being used. Only has an effect for certain code modules.]], ["--wall-mins", "-W", GetoptLong::REQUIRED_ARGUMENT, %[Specify the wall clock limit in minutes.]], ["--write-options", "-w", GetoptLong::REQUIRED_ARGUMENT, %[Use when plotting graphs. A hash of custom options which are applied to the graphkit just before plotting it; for example: -w '{xlabel: 'X Axis Quantity, log_axis: 'y'}']], ["--executable", "-X", GetoptLong::REQUIRED_ARGUMENT, %[Specify the location of the executable of the simulation code. It only needs to be specified once in any folder, unless it needs to be changed.]], ["--other-folder", "-Y", GetoptLong::REQUIRED_ARGUMENT, %[Run CodeRunner in a different folder. On a local machine 'coderunner st -Y some/other/folder' is identical to 'cd some/other/folder; coderunner st -Y'. However, this flag can also be used for remote folders using RemoteCodeRunner (as long as CodeRunner is installed on the remote machine). e.g. -Y username@machine.location:path/to/folder.]], ["--supplementary-options", "-y", GetoptLong::REQUIRED_ARGUMENT], ["--server", "-Z", GetoptLong::REQUIRED_ARGUMENT, %[Technical use only]], ["--log", "-z", GetoptLong::NO_ARGUMENT, %[Switch logging on (currently not working very well (05/2010)).]] # :nodoc: ] CLF = COMMAND_LINE_FLAGS = COMMAND_LINE_FLAGS_WITH_HELP.map{|arr| arr.slice(0..2)} CODE_COMMAND_OPTIONS = [] # NEEDS FIXING!!!! #(Dir.entries(SCRIPT_FOLDER + "/code_modules/") - [".", "..", ".svn"]).map do |d| #["--#{d}-options", "", GetoptLong::REQUIRED_ARGUMENT, %[A hash of options for the #{d} code module]] #end LONG_COMMAND_LINE_OPTIONS = [ ["--replace-existing", "", GetoptLong::NO_ARGUMENT, %[Use with resubmit: causes each resubmitted run to replace the run being resubmitted.]], ["--smart-resubmit-name", "", GetoptLong::NO_ARGUMENT, %[Use with resubmit: causes each resubmitted run to only contain its original id and changed parameters in its run name.]], ] + CODE_COMMAND_OPTIONS LONG_COMMAND_LINE_FLAGS = LONG_COMMAND_LINE_OPTIONS.map{|arr| [arr[0], arr[2]]} rihelp = < err eputs "\nOption #{opt} must be a hash\n\n" raise err end elsif CLF_TO_LONG[opt] copts[CLF_TO_LONG[opt]] = arg else raise "Unknown command line argument: #{opt}" end end copts end # Default command options; they are usually determined by the command line flags, but can be set independently DEFAULT_COMMAND_OPTIONS = {} def self.set_default_command_options_from_command_line #some defaults # DEFAULT_COMMAND_OPTIONS[:p] ||= {} DEFAULT_COMMAND_OPTIONS[:v] ||= "" #DEFAULT_COMMAND_OPTIONS[:n] ||= "1" DEFAULT_COMMAND_OPTIONS[:G] = [] DEFAULT_COMMAND_OPTIONS[:g] = [] DEFAULT_COMMAND_OPTIONS[:k] = true DEFAULT_COMMAND_OPTIONS[:h] ||= :real DEFAULT_COMMAND_OPTIONS[:p] = [] # ep COMMAND_LINE_FLAGS opts = GetoptLong.new(*(COMMAND_LINE_FLAGS + LONG_COMMAND_LINE_FLAGS)) opts.each do |opt, arg| process_command_line_option(opt, arg, DEFAULT_COMMAND_OPTIONS) end #raise "\n\nCannot use large cache ('-U' or '-u' ) if submitting runs" if DEFAULT_COMMAND_OPTIONS[:U] and (DEFAULT_COMMAND_OPTIONS[:s] or DEFAULT_COMMAND_OPTIONS[:P]) if DEFAULT_COMMAND_OPTIONS[:z] Log.log_file = Dir.pwd + '/.cr_logfile.txt' Log.clean_up else Log.log_file = nil # puts Log.log_file end end def self.run_script if DEFAULT_COMMAND_OPTIONS[:Z] DEFAULT_COMMAND_OPTIONS.absorb(eval(DEFAULT_COMMAND_OPTIONS[:Z])) puts "Begin Output" end command = COMMANDS.find{|com| com.slice(0..1).include? ARGV[0]} raise "\n-------------------\nCommand #{ARGV[0].inspect} not found: try 'coderunner man' for help\n-------------------\n" unless command send(command[0].to_sym, *ARGV.values_at(*(1...(1+command[2])).to_a)) end end load CodeRunner::SCRIPT_FOLDER + "/class_methods.rb" load CodeRunner::SCRIPT_FOLDER + "/instance_methods.rb" load CodeRunner::SCRIPT_FOLDER + "/interactive_methods.rb" $stderr.puts unless $has_put_startup_message_for_code_runner $has_put_startup_message_for_code_runner = true CodeRunner.set_default_command_options_from_command_line do_profile = (ENV['CODE_RUNNER_PROFILE'] and ENV['CODE_RUNNER_PROFILE'].size > 0) ? ENV['CODE_RUNNER_PROFILE'] : false if do_profile begin require 'ruby-prof' rescue LoadError eputs "Please install ruby-prof using ' $ gem install ruby-prof'" exit end # Profile the code RubyProf.start end #################### CodeRunner.run_script if $0 == __FILE__ ################### if do_profile result = RubyProf.stop # Print a flat profile to text case ENV['CODE_RUNNER_PROFILE'] when /html/i printer = RubyProf::GraphHtmlPrinter.new(result) when /graph/i printer = RubyProf::GraphPrinter.new(result) when /txt/i printer = RubyProf::FlatPrinter.new(result) else raise "CODE_RUNNER_PROFILE should be 'html', 'graph' or 'txt'" end printer.print($stdout, {}) end