#!/usr/bin/env ruby # frozen_string_literal: true require "open3" require "fileutils" $LOAD_PATH.unshift File.join(File.dirname(__FILE__), "..", "lib") require "helm_upgrade_logs" UPGRADE_LOG_FOLDER = "helm_upgrade_logs" @release_name = ARGV.find { |arg| !arg.start_with?("-") } @namespace = namespace_from_args(ARGV) @helm_pid = Process.spawn "helm upgrade #{ARGV.join(" ")}" pods_before_upgrade = read_pods puts "[INFO] Pods before upgrade #{pods_before_upgrade}" if pods_before_upgrade.size.positive? event_pid = Process.spawn(add_ns("kubectl get events --watch-only=true")) service_pid = Process.spawn(add_ns("kubectl get services --watch-only=true")) wait_for_pod_to_log @pods = [] @pod_pids = {} begin Process.waitpid(@helm_pid, Process::WNOHANG) rescue Errno::ECHILD Process.kill("KILL", event_pid) Process.kill("KILL", service_pid) raise HelmUpgradeLogs::Error, "Failed to find logs before helm finished" end FileUtils.mkdir_p(UPGRADE_LOG_FOLDER) if ENV["helm_upgrade_logs_error_msg"] while Process.waitpid(@helm_pid, Process::WNOHANG).nil? pods = read_pods - pods_before_upgrade if pods != @pods @pods = pods puts "[INFO] Pods: #{pods.join(",")}" # Could change this each to Parallel.each @pods.each do |pod| next unless @pod_pids[pod].nil? wait_for_specific_pod_to_log pod std_out_pid = Process.spawn( add_ns("kubectl logs #{pod} -f --all-containers --prefix --ignore-errors=true --timestamps=true") ) @pod_pids[pod] = std_out_pid next unless ENV["helm_upgrade_logs_error_msg"] logfile_pid = Process.spawn( add_ns("kubectl logs #{pod} -f --all-containers --prefix --ignore-errors=true --timestamps=true"), out: "#{UPGRADE_LOG_FOLDER}/#{pod.gsub("/", "_")}.log" ) @pod_pids["#{pod}_fileout"] = logfile_pid end end sleep 1 end puts "[INFO] Cleaning up loggers after helm upgrade" helm_status = $CHILD_STATUS.exitstatus Process.kill("KILL", event_pid) Process.wait event_pid Process.kill("KILL", service_pid) Process.wait service_pid if ENV["helm_upgrade_logs_debug"] == "true" puts "[INFO] Log processes" puts @pod_pids end @pod_pids.each do |_pod, pid| puts "Terminating #{_pod} #{pid}" if ENV["helm_upgrade_logs_debug"] == "true" Process.kill("KILL", pid) Process.wait pid end puts `ps` if ENV["helm_upgrade_logs_debug"] == "true" if ENV["helm_upgrade_logs_error_msg"] pod_logs_files = Dir.glob("#{UPGRADE_LOG_FOLDER}/*.log") puts "Reading logs from #{pod_logs_files.count} pod logs" if ENV["helm_upgrade_logs_debug"] == "true" pod_logs_files.each do |log_file| lines = File.readlines log_file lines.each_with_index do |line, index| next unless line.include?(ENV["helm_upgrade_logs_error_msg"]) msg = "Helm install error: #{line} from #{log_file} (Line #{index + 1})" if ENV["helm_upgrade_logs_ado_error"] == "true" puts "##vso[task.logissue type=error] #{msg}" else puts "[ERROR] #{msg}" end end end end exit helm_status