modules/mu/groomers/chef.rb in cloud-mu-3.1.3 vs modules/mu/groomers/chef.rb in cloud-mu-3.1.4

- old
+ new

@@ -182,17 +182,17 @@ loaded = nil if !item.nil? begin loaded = ::ChefVault::Item.load(vault, item) - rescue ::ChefVault::Exceptions::KeysNotFound => e + rescue ::ChefVault::Exceptions::KeysNotFound raise MuNoSuchSecret, "Can't load the Chef Vault #{vault}:#{item}. Does it exist? Chef user: #{MU.chef_user}" end else # If we didn't ask for a particular item, list what we have. begin - loaded = ::Chef::DataBag.load(vault).keys.select { |k, v| !k.match(/_keys$/) } + loaded = ::Chef::DataBag.load(vault).keys.select { |k| !k.match(/_keys$/) } rescue Net::HTTPServerException raise MuNoSuchSecret, "Failed to retrieve Vault #{vault}" end end @@ -256,11 +256,10 @@ self.class.loadChefLib if update_runlist and !@config['run_list'].nil? knifeAddToRunList(multiple: @config['run_list']) end - pending_reboot_count = 0 chef_node = ::Chef::Node.load(@server.mu_name) if !@config['application_attributes'].nil? MU.log "Setting node:#{@server.mu_name} application_attributes", MU::DEBUG, details: @config['application_attributes'] chef_node.normal['application_attributes'] = @config['application_attributes'] chef_node.save @@ -298,11 +297,11 @@ else upgrade_cmd = try_upgrade ? "curl -L https://chef.io/chef/install.sh | version=#{MU.chefVersion} sh &&" : "" cmd = "#{upgrade_cmd} chef-client --color || echo #{error_signal}" end Timeout::timeout(timeout) { - retval = ssh.exec!(cmd) { |ch, stream, data| + ssh.exec!(cmd) { |_ch, _stream, data| extra_logfile = if Dir.exist?(@server.deploy.deploy_dir) File.open(@server.deploy.deploy_dir+"/log", "a") end puts data output_lines << data @@ -378,20 +377,22 @@ if MU::Cloud::AWS.isGovCloud?(@config['region']) @server.reboot(true) sleep 30 end retry - rescue RuntimeError, SystemCallError, Timeout::Error, SocketError, Errno::ECONNRESET, IOError, Net::SSH::Exception, MU::Groomer::RunError, WinRM::WinRMError, MU::MuError => e + rescue SystemExit, Timeout::Error, MU::Cloud::BootstrapTempFail, Net::HTTPServerException, HTTPClient::ConnectTimeoutError, WinRM::WinRMError, Net::SSH::AuthenticationFailed, Net::SSH::Disconnect, Net::SSH::ConnectionTimeout, Net::SSH::Proxy::ConnectError, Net::SSH::Exception, Errno::ECONNRESET, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, Errno::EPIPE, SocketError, IOError => e begin ssh.close if !ssh.nil? rescue Net::SSH::Exception, IOError => e if @server.windows? MU.log "Windows has probably closed the ssh session before we could. Waiting before trying again", MU::DEBUG else MU.log "ssh session to #{@server.mu_name} was closed unexpectedly, waiting before trying again", MU::NOTICE end sleep 10 + rescue StandardError => e + MU.log "Error I don't recognize closing ssh tunnel", MU::WARN, details: e.inspect end if e.instance_of?(MU::Groomer::RunError) and retries == 0 and max_retries > 1 and purpose != "Base Windows configuration" MU.log "Got a run error, will attempt to install/update Chef Client on next attempt", MU::NOTICE try_upgrade = true else @@ -402,11 +403,11 @@ if reboot_first_fail try_upgrade = true begin preClean(true) # drop any Chef install that's not ours @server.reboot # try gently rebooting the thing - rescue Exception => e # it's ok to fail here (and to ignore failure) + rescue StandardError => e # it's ok to fail here (and to ignore failure) MU.log "preclean err #{e.inspect}", MU::ERR end reboot_first_fail = false end end @@ -427,11 +428,11 @@ retry else @server.deploy.sendAdminSlack("Chef run '#{purpose}' failed on `#{@server.mu_name}` :crying_cat_face:", msg: e.message) raise MU::Groomer::RunError, "#{@server.mu_name}: Chef run '#{purpose}' failed #{max_retries} times, last error was: #{e.message}" end - rescue Exception => e + rescue StandardError => e @server.deploy.sendAdminSlack("Chef run '#{purpose}' failed on `#{@server.mu_name}` :crying_cat_face:", msg: e.inspect) raise MU::Groomer::RunError, "Caught unexpected #{e.inspect} on #{@server.mu_name} in @groomer.run at #{e.backtrace[0]}" end @@ -441,12 +442,12 @@ # Make sure we've got a Splunk admin vault for any mu-splunk-servers to # use, and set it up if we don't. def splunkVaultInit self.class.loadChefLib begin - loaded = ::ChefVault::Item.load("splunk", "admin_user") - rescue ::ChefVault::Exceptions::KeysNotFound => e + ::ChefVault::Item.load("splunk", "admin_user") + rescue ::ChefVault::Exceptions::KeysNotFound pw = Password.pronounceable(12..14) creds = { "username" => "admin", "password" => pw, "auth" => "admin:#{pw}" @@ -548,19 +549,19 @@ begin MU.log "Attempting Chef upgrade via WinRM on #{@server.mu_name}", MU::NOTICE, details: cmd winrm = @server.getWinRMSession(1, 30, winrm_retries: 2) pp winrm.run(cmd) return - rescue Net::SSH::Disconnect, SystemCallError, Timeout::Error, Errno::ECONNRESET, Errno::EHOSTUNREACH, Net::SSH::Proxy::ConnectError, SocketError, Net::SSH::Disconnect, Net::SSH::AuthenticationFailed, IOError, Net::HTTPServerException, SystemExit, Errno::ECONNREFUSED, Errno::EPIPE, WinRM::WinRMError, HTTPClient::ConnectTimeoutError, RuntimeError, MU::Cloud::BootstrapTempFail, MU::MuError => e + rescue SystemExit, Timeout::Error, MU::Cloud::BootstrapTempFail, MU::MuError, Net::HTTPServerException, HTTPClient::ConnectTimeoutError, WinRM::WinRMError, Net::SSH::AuthenticationFailed, Net::SSH::Disconnect, Net::SSH::ConnectionTimeout, Net::SSH::Proxy::ConnectError, Net::SSH::Exception, Errno::ECONNRESET, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, Errno::EPIPE, SocketError, IOError MU.log "WinRM failure attempting Chef upgrade on #{@server.mu_name}, will fall back to ssh", MU::WARN cmd = %Q{powershell.exe -inputformat none -noprofile "#{cmd}"} end end MU.log "Attempting Chef upgrade via ssh on #{@server.mu_name}", MU::NOTICE, details: cmd ssh = @server.getSSHSession(1) - retval = ssh.exec!(cmd) { |ch, stream, data| + ssh.exec!(cmd) { |_ch, _stream, data| puts data } end # Bootstrap our server with Chef @@ -578,11 +579,11 @@ retry end @config['cleaned_chef'] = true end - nat_ssh_key, nat_ssh_user, nat_ssh_host, canonical_addr, ssh_user, ssh_key_name = @server.getSSHConfig + _nat_ssh_key, _nat_ssh_user, _nat_ssh_host, canonical_addr, ssh_user, ssh_key_name = @server.getSSHConfig MU.log "Bootstrapping #{@server.mu_name} (#{canonical_addr}) with knife" run_list = ["recipe[mu-tools::newclient]"] run_list << "mu-tools::gcloud" if @server.cloud == "Google" or @server.config['cloud'] == "Google" @@ -591,15 +592,16 @@ if !@config['application_attributes'].nil? json_attribs['application_attributes'] = @config['application_attributes'] json_attribs['skipinitialupdates'] = @config['skipinitialupdates'] end - if !@config['vault_access'].nil? - vault_access = @config['vault_access'] - else - vault_access = [] - end +# XXX this seems to break Knife Bootstrap +# vault_access = if !@config['vault_access'].nil? +# @config['vault_access'] +# else +# [] +# end @server.windows? ? max_retries = 25 : max_retries = 10 @server.windows? ? timeout = 1800 : timeout = 300 retries = 0 begin @@ -656,11 +658,11 @@ MU::Cloud.handleNetSSHExceptions kb.run } # throws Net::HTTPServerException if we haven't really bootstrapped ::Chef::Node.load(@server.mu_name) - rescue Net::SSH::Disconnect, SystemCallError, Timeout::Error, Errno::ECONNRESET, Errno::EHOSTUNREACH, Net::SSH::Proxy::ConnectError, SocketError, Net::SSH::Disconnect, Net::SSH::AuthenticationFailed, IOError, Net::HTTPServerException, SystemExit, Errno::ECONNREFUSED, Errno::EPIPE, WinRM::WinRMError, HTTPClient::ConnectTimeoutError, RuntimeError, MU::Cloud::BootstrapTempFail, Net::SSH::Exception, Net::SSH::ConnectionTimeout => e + rescue SystemExit, Timeout::Error, MU::Cloud::BootstrapTempFail, Net::HTTPServerException, HTTPClient::ConnectTimeoutError, WinRM::WinRMError, Net::SSH::AuthenticationFailed, Net::SSH::Disconnect, Net::SSH::ConnectionTimeout, Net::SSH::Proxy::ConnectError, Net::SSH::Exception, Errno::ECONNRESET, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, Errno::EPIPE, SocketError, IOError => e if retries < max_retries retries += 1 # Bad Chef installs are possible culprits of bootstrap failures, so # try scrubbing them when that happens. # On Windows, even a fresh install comes up screwy disturbingly @@ -669,11 +671,11 @@ if !e.is_a?(MU::Cloud::BootstrapTempFail) and !(e.is_a?(WinRM::WinRMError) and @config['forced_preclean']) and !@config['forced_preclean'] begin preClean(false) # it's ok for this to fail - rescue Exception => e + rescue StandardError => e end MU::Groomer::Chef.cleanup(@server.mu_name, nodeonly: true) @config['forced_preclean'] = true @server.reboot if @server.windows? # *sigh* end @@ -681,11 +683,11 @@ sleep 10*retries retry else raise MuError, "#{@server.mu_name}: Knife Bootstrap failed too many times with #{e.inspect}" end - rescue Exception => e + rescue StandardError => e MU.log e.inspect, MU::ERR, details: e.backtrace sleep 10*retries retry end @@ -698,11 +700,11 @@ rescue SystemExit => e MU.log "#{@server.mu_name}: Run list removal of recipe[#{recipe}] failed with #{e.inspect}", MU::WARN end } knifeAddToRunList("role[mu-node]") - knifeAddToRunList("mu-tools::selinux") + knifeAddToRunList("recipe[mu-tools::selinux]") grantSecretAccess(@server.mu_name, "windows_credentials") if @server.windows? grantSecretAccess(@server.mu_name, "ssl_cert") saveChefMetadata @@ -746,11 +748,11 @@ if !haveBootstrapped? MU.log "saveDeployData invoked on #{@server.to_s} before Chef has been bootstrapped!", MU::WARN, details: caller return end - @server.describe(update_cache: true) # Make sure we're fresh + @server.describe saveChefMetadata begin chef_node = ::Chef::Node.load(@server.mu_name) # Our deploydata gets corrupted often with server pools, in this case the the deploy data structure of some nodes is corrupt the hashes can become too nested and also invalid. @@ -783,11 +785,11 @@ MU.log "Updating node: #{@server.mu_name} deployment attributes", details: @server.deploy.deployment chef_node.normal['deployment'].merge!(@server.deploy.deployment) chef_node.save end return chef_node['deployment'] - rescue Net::HTTPServerException => e + rescue Net::HTTPServerException MU.log "Attempted to save deployment to Chef node #{@server.mu_name} before it was bootstrapped.", MU::DEBUG end end # Expunge Chef resources associated with a node. @@ -802,11 +804,11 @@ vaults_to_clean.each { |vault| MU::MommaCat.lock("vault-#{vault['vault']}", false, true) MU.log "knife vault remove #{vault['vault']} #{vault['item']} --search name:#{node}", MU::NOTICE begin ::Chef::Knife.run(['vault', 'remove', vault['vault'], vault['item'], "--search", "name:#{node}"]) if !noop - rescue Exception => e + rescue StandardError => e MU.log "Error removing vault access for #{node} from #{vault['vault']} #{vault['item']}", MU::ERR, details: e.inspect end MU::MommaCat.unlock("vault-#{vault['vault']}") } end @@ -856,11 +858,11 @@ ::Chef::Knife.run(['vault', 'rotate', 'keys', vault['vault'], vault['item'], "--clean-unknown-clients"]) ::Chef::Knife.run(['vault', 'update', vault['vault'], vault['item'], "--search", searchstr]) ::Chef::Knife.run(['vault', 'refresh', vault['vault'], vault['item']]) end end - rescue JSON::ParserError => e + rescue JSON::ParserError MU.log "Error parsing JSON from data bag #{vault['vault']} #{vault['item']}_keys, skipping vault client cleanse", MU::WARN end end MU::MommaCat.unlock("vault-#{vault['vault']}") } @@ -885,22 +887,38 @@ loadChefLib MU::MommaCat.lock("vault-#{vault}", false, true) MU.log "Granting #{host} access to #{vault} #{item}" begin ::Chef::Knife.run(['vault', 'update', vault, item, "--search", "name:#{host}"]) - rescue Exception => e + rescue StandardError => e MU.log e.inspect, MU::ERR, details: caller end MU::MommaCat.unlock("vault-#{vault}", true) end + # Execute a +knife+ command, and return its exit status and output + # @param cmd [String]: The knife subcommand to run, such as +vault list+ + # @param showoutput [String]: Print the results to stdout + # @return [Array<Integer,String>] + def self.knifeCmd(cmd, showoutput = false) + MU.log "knife #{cmd}", MU::NOTICE if showoutput + output = `#{MU::Groomer::Chef.knife} #{cmd}` + exitstatus = $?.exitstatus + + if showoutput + puts output + puts "Exit status: #{exitstatus}" + end + return [exitstatus, output] + end + private # Save common Mu attributes to this node's Chef node structure. def saveChefMetadata self.class.loadChefLib - nat_ssh_key, nat_ssh_user, nat_ssh_host, canonical_addr, ssh_user, ssh_key_name = @server.getSSHConfig + @server.getSSHConfig # why though MU.log "Saving #{@server.mu_name} Chef artifacts" begin chef_node = ::Chef::Node.load(@server.mu_name) rescue Net::HTTPServerException @@ -996,11 +1014,11 @@ # In some cases the cached getLitter response will not have all the resources in the deploy, so lets not use the cache. if @config.has_key?('dependencies') deploy = MU::MommaCat.getLitter(MU.deploy_id, use_cache: false) @config['dependencies'].each{ |dep| if dep['type'] == "database" && deploy.deployment.has_key?("databases") && deploy.deployment["databases"].has_key?(dep['name']) - deploy.deployment["databases"][dep['name']].each { |name, database| + deploy.deployment["databases"][dep['name']].values.each { |database| grantSecretAccess(database['vault_name'], database['vault_item']) if database.has_key?("vault_name") && database.has_key?("vault_item") } end } end @@ -1017,22 +1035,10 @@ return if @secrets_granted["#{vault}:#{item}"] == item self.class.grantSecretAccess(@server.mu_name, vault, item) @secrets_granted["#{vault}:#{item}"] = item end - def self.knifeCmd(cmd, showoutput = false) - MU.log "knife #{cmd}", MU::NOTICE if showoutput - output = `#{MU::Groomer::Chef.knife} #{cmd}` - exitstatus = $?.exitstatus - - if showoutput - puts output - puts "Exit status: #{exitstatus}" - end - return [exitstatus, output] - end - def knifeCmd(cmd, showoutput = false) self.class.knifeCmd(cmd, showoutput) end # Upload the certificate to a Chef Vault for this node @@ -1061,13 +1067,15 @@ self.class.loadChefLib return if rl_entry.nil? and multiple.size == 0 if multiple.size == 0 multiple = [rl_entry] end - multiple.each { |entry| + multiple.map! { |entry| if !entry.match(/^role|recipe\[/) - entry = "#{type}[#{entry}]" + "#{type}[#{entry}]" + else + entry end } if !ignore_missing role_list = nil @@ -1108,11 +1116,11 @@ begin query=%Q{#{MU::Groomer::Chef.knife} node run_list add #{@server.mu_name} "#{rl_string}"}; MU.log("Adding #{rl_string} to Chef run_list of #{@server.mu_name}") MU.log("Running #{query}", MU::DEBUG) output=%x{#{query}} - # XXX rescue Exception is bad style - rescue Exception => e + # XXX rescue StandardError is bad style + rescue StandardError => e raise MuError, "FAIL: #{MU::Groomer::Chef.knife} node run_list add #{@server.mu_name} \"#{rl_string}\": #{e.message} (output was #{output})" end end end # class Chef