require 'thor' require 'yaml' require 'erb' require 'rubygems' require 'zip' require 'digest' require 'json' require 'rbconfig' require 'tmpdir' require 'fileutils' require 'retriable' require 'xamarin-test-cloud/version' require 'xamarin-test-cloud/retriable_options' require 'xamarin-test-cloud/http/request' require 'xamarin-test-cloud/http/retriable_client' require 'securerandom' require 'open3' trap "SIGINT" do puts "Exiting" exit 10 end module XamarinTestCloud class ValidationError < Thor::InvocationError end class CLI < Thor include Thor::Actions attr_accessor :app, :api_key, :appname, :test_parameters, :user, :workspace, :config, :profile, :skip_config_check, :dry_run, :device_selection, :pretty, :async, :async_json, :priority, :endpoint_path, :locale, :series, :dsym, :session_id def self.exit_on_failure? true end def initialize(*args) self.session_id = SecureRandom.hex begin r = JSON.parse(http_post("check_version", {args: ARGV})) if r["error_message"] puts r["error_message"] exit 1 end rescue end super(*args) end FILE_UPLOAD_ENDPOINT = 'upload2' FORM_URL_ENCODED_ENDPOINT = 'upload' def self.source_root File.join(File.dirname(__FILE__), '..') end desc 'version', 'Prints version of the xamarin-test-cloud gem' def version puts XamarinTestCloud::VERSION end desc 'submit ', 'Submits your app and test suite to Xamarin Test Cloud' method_option 'app-name', :desc => 'App name to create or add test to', :aliases => '-a', :required => false, :type => :string method_option 'devices', :desc => 'Device selection', :aliases => '-d', :required => true, :type => :string method_option 'test-parameters', :desc => 'Test parameters (e.g., -params username:nat@xamarin.com password:xamarin)', :aliases => '-params', :type => :hash method_option :workspace, :desc => 'Path to your Calabash workspace (containing your features folder)', :aliases => '-w', :type => :string method_option :config, :desc => 'Cucumber configuration file (cucumber.yml)', :aliases => '-c', :type => :string method_option 'skip-config-check', :desc => "Force running without Cucumber profile (cucumber.yml)", :type => :boolean, :default => false method_option :profile, :desc => 'Profile to run (profile from cucumber.yml)', :aliases => '-p', :type => :string method_option :pretty, :desc => 'Pretty print output.', :type => :boolean, :default => false method_option :async, :desc => "Don't block waiting for test results.", :type => :boolean, :default => false method_option 'async-json', :desc => "Don't block waiting for test results. Output results in json format.", :type => :boolean, :default => false method_option :priority, :desc => "Run as priority test execution. Please note: This is only available for some tiers, and priority test executions cost double.", :type => :boolean, :default => false method_option 'dry-run', :desc => "Sanity check only, don't upload", :aliases => '-s', :type => :boolean, :default => false #do upload by default method_option 'locale', :desc => "System language", :type => :string method_option 'series', :desc => "Test series", :type => :string method_option 'dsym-file', :desc => 'Optional dSym file for iOS Crash symbolication', :aliases => '-y', :required => false, :type => :string method_option 'user', :desc => 'Email address of the user uploading', :aliases => '-u', :required => false, :type => :string def submit(app, api_key) self.pretty = options[:pretty] self.async_json = options['async-json'] self.async = options[:async] || self.async_json # Async mode wraps all console output in a json object # So we need to intercept all writes to $stdout if self.async_json @async_log = StringIO.new @async_result = { test_run_id: nil, error_messages: [], log: [] } $stdout = @async_log end app_path = File.expand_path(app) unless File.exist?(app_path) raise ValidationError, "App is not a file: #{app_path}" end if shared_runtime?(app_path) puts "Xamarin Test Cloud doesn't yet support shared runtime apps." puts "To test your app it needs to be compiled for release." puts "You can learn how to compile you app for release here:" puts "http://docs.xamarin.com/guides/android/deployment%2C_testing%2C_and_metrics/publishing_an_application/part_1_-_preparing_an_application_for_release" raise ValidationError, "Aborting" end app_extension = File.extname(app_path) unless /ipa/i.match(app_extension) || /apk/i.match(app_extension) raise ValidationError, "App #{app_path} must be an .ipa or .apk file" end self.app = app_path self.user = options['user'] self.dry_run = options['dry-run'] self.api_key = api_key self.test_parameters = options['test-parameters'] || {} self.appname = options['app-name'] self.device_selection = options['devices'] device_selection_data = validate_device_selection self.locale = options['locale'] self.series = options['series'] self.priority = options['priority'] self.skip_config_check = options['skip-config-check'] self.dsym= options['dsym-file'] if dsym dsym_extension = File.extname(self.dsym) unless /dsym/i.match(dsym_extension) && File.directory?(dsym) raise ValidationError, "dsym-file must be a directory and have dSYM extension: #{dsym}" end end workspace_path = options[:workspace] || File.expand_path('.') unless File.directory?(workspace_path) raise ValidationError, "Provided workspace: #{workspace_path} is not a directory." end workspace_basename = File.basename(workspace_path) if workspace_basename.downcase == 'features' self.workspace = File.expand_path(File.join(workspace_path, '..')) puts "Deriving workspace #{self.workspace} from features folder #{workspace_basename}" else self.workspace = File.expand_path(workspace_path) end self.workspace = File.join(self.workspace, File::Separator) unless File.directory?(File.join(self.workspace, 'features')) log_header "Did not find features folder in workspace #{self.workspace}" puts "Either run the test-cloud command from the directory containing your features" puts "or use the --workspace option to refer to this directory" puts "See also test-cloud help submit" raise ValidationError, "Unable to find features folder in #{self.workspace}" end parse_and_set_config_and_profile unless self.skip_config_check default_config = File.join(self.workspace, 'config', 'cucumber.yml') if File.exist?(default_config) && self.config.nil? log_header 'Warning: Detected cucumber.yml config file, but no --config specified' puts "Please specify --config #{default_config}" puts 'and specify a profile via --profile' puts 'If you know what you are doing you can skip this check with' puts '--skip-config-check' raise ValidationError, "#{default_config} detected but no profile selected." end end if debug? puts "Host = #{self.host}" puts "User = #{self.user}" puts "App = #{self.app}" puts "App Name = #{self.app}" puts "TestParams = #{self.test_parameters}" puts "API Key = #{self.api_key}" puts "Device Selection = #{self.device_selection}" puts "Workspace = #{self.workspace}" puts "Config = #{self.config}" puts "Profile = #{self.profile}" puts "dSym = #{self.dsym}" end #Argument parsing done test_jon_data = submit_test_job(device_selection_data) if self.dry_run return end json = test_jon_data[:body] if debug? p json end log_header('Test enqueued') puts "User: #{json['user_email']}" puts "Team: #{json['team']}" if json['team'] rejected_devices = json['rejected_devices'] if rejected_devices && rejected_devices.size > 0 puts 'Skipping devices (you can update your selections via https://testcloud.xamarin.com)' rejected_devices.each { |d| puts d } end puts '' puts 'Running on Devices:' json['devices'].each do |device| puts device end puts '' unless self.async wait_for_job(json['id']) else log 'Async mode: not awaiting test results' @async_result[:test_run_id] = json['id'] if self.async_json end rescue XamarinTestCloud::ValidationError => e if self.async_json @async_result[:error_messages] << e.message else raise end ensure $stdout = STDOUT if self.async_json process_async_log puts @async_result.to_json end end default_task :submit no_tasks do def debug? ENV['DEBUG'] == '1' end def process_async_log @async_result[:log] = @async_log.string .split(/\n/).map { |string| string.gsub(/\e\[(\d+)m/, '').strip } .select { |string| string.length > 0 } end def exit_on_failure? true end def wait_for_job(id) retry_opts = XamarinTestCloud::RetriableOptions.tries_and_interval(60, 10) while(true) status_json = Retriable.retriable(retry_opts) do JSON.parse(http_post("status_v3", {'id' => id, 'api_key' => api_key, 'user' => user})) end if debug? log "Status JSON result:" puts status_json end wait_time = (Integer status_json["wait_time"] rescue nil) || 10 wait_time = 1 if debug? log status_json["message"] if status_json["exit_code"] exit Integer status_json["exit_code"] end sleep wait_time end end def validate_device_selection return device_selection if device_selection == device_selection.upcase #Allow for special device selections to be passed to the server. Needs to been in all caps. unless /^[0-9a-fA-F]{8,12}$/ =~ device_selection raise ValidationError, 'Device selection is not in the proper format. Please generate a new one on the Xamarin Test Cloud website.' end device_selection end def workspace_gemfile File.join(self.workspace, 'Gemfile') end def workspace_gemfile_lock File.join(self.workspace, 'Gemfile.lock') end def submit_test_job(device_selection_data) tmpdir = Dir.mktmpdir if debug? log "Packaging gems in: #{tmpdir}" end server = verify_app_and_extract_test_server log_header('Checking for Gemfile') if File.exist?(workspace_gemfile) FileUtils.cp workspace_gemfile, tmpdir FileUtils.cp workspace_gemfile_lock, tmpdir if File.exist?(workspace_gemfile_lock) else copy_default_gemfile(File.join(tmpdir, "Gemfile"), server) end log_header('Packaging') ENV['BUNDLE_GEMFILE'] = File.join(tmpdir, "Gemfile") FileUtils.cd(self.workspace) do if self.async_json bundle_log, status = Open3.capture2e('bundle package --all') puts bundle_log else system('bundle package --all') status = $? end if status != 0 log_and_abort 'Bundler failed. Please check command: bundle package' end end log_header('Verifying dependencies') verify_dependencies(tmpdir) if dry_run log_header('Dry run only') log('Dry run completed OK') return end app_file, dsym_zip, files, paths = gather_files_and_paths_to_upload(all_files(tmpdir), tmpdir) log_header('Uploading negotiated files') upload_data = {'files' => files, 'paths' => paths, 'user' => self.user, 'client_version' => XamarinTestCloud::VERSION, 'app_file' => app_file, 'device_selection' => device_selection_data, 'app' => self.appname, 'test_parameters' => self.test_parameters, 'locale' => self.locale, 'series' => self.series, 'api_key' => api_key, 'dsym_file' => dsym_zip, 'dsym_filename' => dsym_file_name, 'app_filename' => File.basename(app), 'priority' => self.priority} if profile #only if config and profile upload_data['profile'] = profile end if debug? puts JSON.pretty_generate(upload_data) end contains_file = files.find { |f| f.is_a?(File) } contains_file = contains_file || app_file.is_a?(File) if contains_file self.endpoint_path = FILE_UPLOAD_ENDPOINT #nginx receives upload else self.endpoint_path = FORM_URL_ENCODED_ENDPOINT #ruby receives upload end if debug? puts "Will upload to file path: #{self.endpoint_path}" end response = http_post(endpoint_path, upload_data) do |response, request| if debug? puts "Request url: #{request.url}" puts "Response code: #{response.code}" puts "Response body: #{response.body}" end case response.status_code when 200..202 response when 400 error_message = JSON.parse(response.body)['error_message'] rescue 'Bad request' log_and_abort(error_message) when 403 error_message = JSON.parse(response.body)['message'] rescue 'Forbidden' log_and_abort(error_message) when 413 error_message = 'Files are too large' log_and_abort(error_message) else error_message = 'Unexpected Error. Please contact support at testcloud@xamarin.com.' log_and_abort(error_message) end end return :status => response.status_code, :body => JSON.parse(response.body) end def copy_default_gemfile(gemfile_path, server) log('') log('Gemfile missing.') log('You must provide a Gemfile in your workspace.') log('A Gemfile must describe your dependencies and their versions') log('See: http://gembundler.com/v1.3/gemfile.html') log('') log('Warning proceeding with default Gemfile.') log('It is strongly recommended that you create a custom Gemfile.') File.open(gemfile_path, "w") do |f| f.puts "source 'http://rubygems.org'" if is_android? f.puts "gem 'calabash-android', '#{calabash_android_version}'" elsif is_ios? f.puts "gem 'calabash-cucumber', '#{calabash_ios_version}'" else raise ValidationError, 'Your app must be an ipa or apk file.' end end log("Proceeding with Gemfile: #{gemfile_path}") puts(File.read(gemfile_path)) log('') end def gather_files_and_paths_to_upload(collected_files, tmpdir) log_header('Calculating digests') file_paths = collected_files[:files] feature_prefix = collected_files[:feature_prefix] workspace_prefix = collected_files[:workspace_prefix] hashes = file_paths.collect { |f| digest(f) } if hashes.nil? || hashes.size == 0 hashes << '0222222222222222222222222222222222222222222222222222222222222222' end log_header('Negotiating upload') app_digest = digest(app) dsym_digest= nil if dsym FileUtils.cp_r(dsym, tmpdir) files_in_dwarf = Dir.glob(File.join(tmpdir, File.basename(dsym), 'Contents', 'Resources', 'DWARF', '*')) unless files_in_dwarf.count == 1 raise ValidationError, "dSym #{dsym} contains more than one file in Contents/Resources/DWARF: #{files_in_dwarf}" end dsym_abs_path= files_in_dwarf.first dsym_digest = digest(dsym_abs_path) end out = {'hashes' => hashes, 'app_hash' => app_digest, 'dsym_hash' => dsym_digest} response = http_post('check_hash', out) cache_status = JSON.parse(response) log_header('Gathering files based on negotiation') files = [] paths = [] file_paths.each do |file| if cache_status[digest(file)] #Server already knows about this file. No need to upload it. files << digest(file) else #Upload file files << File.new(file) end if file.start_with?(feature_prefix) prefix = feature_prefix else prefix = workspace_prefix end paths << file.sub(prefix, '').sub("#{tmpdir}/", '') end if config files << File.new(config) paths << 'config/cucumber.yml' end app_file = cache_status[app_digest] ? app_digest : File.new(app) if dsym_digest dsym_file = cache_status[dsym_digest] ? dsym_digest : File.new(dsym_abs_path) end return app_file, dsym_file, files, paths end def digest(file) Digest::SHA256.file(file).hexdigest end def unzip_file (file, destination) Zip::File.open(file) { |zip_file| zip_file.each { |f| f_path=File.join(destination, f.name) FileUtils.mkdir_p(File.dirname(f_path)) zip_file.extract(f, f_path) unless File.exist?(f_path) } } end def is_android? app.end_with? '.apk' end def is_ios? app.end_with? '.ipa' end def is_calabash_2? detect_calabash_2 end def get_calabash_version_from_bundler cmd = "bundle exec ruby -e \"begin; require 'calabash/version';puts Calabash::VERSION;rescue LoadError => e; puts '';end\"" `#{cmd}`.strip end def detect_calabash_2 @detected_calabash_2 ||= lambda do log "Auto-detecting if Calabash 2.0 testsuite" if debug? # Implicitly detect if File.exist?(workspace_gemfile) FileUtils.cd(self.workspace) do version = get_calabash_version_from_bundler if version.split('.').first.to_i >= 2 log "It is a Calabash 2.0 testsuite" if debug? return true end end end false end.call end def calabash_2_version version = nil if File.exist?(workspace_gemfile) FileUtils.cd(self.workspace) do version = get_calabash_version_from_bundler end end unless version require 'calabash' version = Calabash::VERSION end version.strip end def calabash_android_version if is_calabash_2? return calabash_2_version end version = nil if File.exist?(workspace_gemfile) FileUtils.cd(self.workspace) do version = `bundle exec ruby -e "require 'calabash-android/version'; puts Calabash::Android::VERSION"` version = version && version.strip end end unless version require 'calabash-android' version = Calabash::Android::VERSION end version.strip end def calabash_ios_version if is_calabash_2? return calabash_2_version end version = nil if File.exist?(workspace_gemfile) FileUtils.cd(self.workspace) do version = `bundle exec ruby -e "require 'calabash-cucumber/version'; puts Calabash::Cucumber::VERSION"` version = version && version.strip end end unless version require 'calabash-cucumber' version = Calabash::Cucumber::VERSION end version.strip end def test_server_path require 'digest/md5' digest = Digest::MD5.file(app).hexdigest File.join(self.workspace, 'test_servers', "#{digest}_#{calabash_android_version}.apk") end def all_files(tmpdir) dir = workspace files = Dir.glob(File.join("#{dir}features", '**', '*')) if File.directory?("#{dir}playback") files += Dir.glob(File.join("#{dir}playback", '*')) end if config files << config end files += Dir.glob(File.join(tmpdir, "vendor", 'cache', '*')) if workspace and workspace.strip != '' files += Dir.glob("#{workspace}Gemfile") files += Dir.glob("#{workspace}Gemfile.lock") end if is_android? files << test_server_path end {:feature_prefix => dir, :workspace_prefix => workspace, :files => files.find_all { |file_or_dir| File.file? file_or_dir }} end def dsym_file_name if dsym "#{File.basename(self.app)}_dSym" end end def host ENV['XTC_ENDPOINT'] || 'https://testcloud.xamarin.com/ci' end def http_post(address, args = {}, &block) args['uploader_version'] = XamarinTestCloud::VERSION args['session_id'] = session_id request = XamarinTestCloud::HTTP::Request.new(address, args) exec_options = {:header => { 'Content-Type' => 'application/x-www-form-urlencoded' }} if ENV['XTC_USERNAME'] && ENV['XTC_PASSWORD'] exec_options[:user] = ENV['XTC_USERNAME'] exec_options[:password] = ENV['XTC_PASSWORD'] end client = XamarinTestCloud::HTTP::RetriableClient.new("#{host}/") if block_given? exec_options[:header] = {'Content-Type' => 'multipart/form-data'} exec_options[:timeout] = 90000000 response = client.post(request, exec_options) block.call(response, request) response else response = client.post(request, exec_options) response.body end end def is_windows? (RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/) end def is_macosx? (RbConfig::CONFIG['host_os'] =~ /darwin/) end def validate_ipa(ipa) result = false dir = Dir.mktmpdir #do |dir| unzip_file(ipa, dir) unless File.directory?("#{dir}/Payload") #macos only raise ValidationError, "Unzipping #{ipa} to #{dir} failed: Did not find a Payload directory (invalid .ipa)." end app_dir = Dir.foreach("#{dir}/Payload").find { |d| /\.app$/.match(d) } res = `otool "#{File.expand_path(dir)}/Payload/#{app_dir}/"* -o 2> /dev/null | grep CalabashServer` if /CalabashServer/.match(res) puts "ipa: #{ipa} *contains* calabash.framework" result = :calabash end unless result res = `otool "#{File.expand_path(dir)}/Payload/#{app_dir}/"* -o 2> /dev/null | grep FrankServer` if /FrankServer/.match(res) puts "ipa: #{ipa} *contains* FrankServer" raise ValidationError, 'Frank not supported just yet' else puts "ipa: #{ipa} *does not contain* calabash.framework" result = false end end result end def log(message) if message.is_a? Array message.each { |m| log(m) } else puts "#{Time.now } #{message}" $stdout.flush end end def log_header(message) if is_windows? puts "\n### #{message} ###" else puts "\n\e[#{35}m### #{message} ###\e[0m" end end def verify_app_and_extract_test_server server = nil unless File.exist?(app) raise ValidationError, "No such file: #{app}" end unless (/\.(apk|ipa)$/ =~ app) raise ValidationError, ' should be either an ipa or apk file.' end if is_ios? and is_macosx? log_header('Checking ipa for linking with Calabash') server = validate_ipa(app) abort_unless(server) do puts 'The .ipa file does not seem to be linked with Calabash.' puts 'Verify that your app is linked correctly.' end end server end def abort(&block) yield block exit 1 end def abort_unless(condition, &block) unless condition yield block exit 1 end end def log_and_abort(message) raise XamarinTestCloud::ValidationError.new(message) if self.async_json abort do print 'Error: ' puts message end end def shared_runtime?(app_path) f = files(app_path) f.any? do |file| filename = file[:filename] if filename.end_with?("libmonodroid.so") file[:size] < 120 * 1024 && f.none? { |x| x[:filename] == filename.sub("libmonodroid.so", "libmonosgen-2.0.so") } end end end def files(app) Zip::File.open(app) do |zip_file| zip_file.collect do |entry| {:filename => entry.to_s, :size => entry.size} end end end def verify_dependencies(path) if is_android? abort_unless(File.exist?(test_server_path)) do puts "No test server '#{test_server_path}' found. Please run:" if is_calabash_2? puts " calabash build #{app}" else puts " calabash-android build #{app}" end end if is_calabash_2? calabash_gem = Dir.glob("#{path}/vendor/cache/calabash-*").first abort_unless(calabash_gem) do puts 'calabash was not packaged correct.' puts 'Please tell testcloud@xamarin.com about this bug.' end else calabash_gem = Dir.glob("#{path}/vendor/cache/calabash-android-*").first abort_unless(calabash_gem) do puts 'calabash-android was not packaged correct.' puts 'Please tell testcloud@xamarin.com about this bug.' end end end end def parse_and_set_config_and_profile config_path = options[:config] if config_path config_path = File.expand_path(config_path) unless File.exist?(config_path) raise ValidationError, "Config file does not exist #{config_path}" end begin config_yml = YAML.load(ERB.new(File.read(config_path)).result) rescue Exception => e puts "Unable to parse #{config_path} as yml. Is this your Cucumber.yml file?" raise ValidationError, e end if debug? puts 'Parsed Cucumber config as:' puts config_yml.inspect end profile = options[:profile] unless profile raise ValidationError, 'Config file provided but no profile selected.' else unless config_yml[profile] raise ValidationError, "Config file provided did not contain profile #{profile}." else puts "Using profile #{profile}..." self.profile = profile end end else if options[:profile] raise ValidationError, 'Profile selected but no config file provided.' end end self.config = config_path end end end end