# frozen_string_literal: true module Beaker # Docker hypervisor for Beaker acceptance testing framework class Docker < Beaker::Hypervisor # Docker hypvervisor initializtion # Env variables supported: # DOCKER_REGISTRY: Docker registry URL # DOCKER_HOST: Remote docker host # DOCKER_BUILDARGS: Docker buildargs map # @param [Host, Array, String, Symbol] hosts One or more hosts to act upon, # or a role (String or Symbol) that identifies one or more hosts. # @param [Hash{Symbol=>String}] options Options to pass on to the hypervisor def initialize(hosts, options) super require 'docker' @options = options @logger = options[:logger] || Beaker::Logger.new @hosts = hosts # increase the http timeouts as provisioning images can be slow default_docker_options = { write_timeout: 300, read_timeout: 300 }.merge(::Docker.options || {}) # Merge docker options from the entry in hosts file ::Docker.options = default_docker_options.merge(@options[:docker_options] || {}) # Ensure that we can correctly communicate with the docker API begin @docker_version = ::Docker.version rescue Excon::Errors::SocketError => e raise <<~ERRMSG Docker instance not connectable Error was: #{e} * Check your DOCKER_HOST variable has been set * If you are on OSX or Windows, you might not have Docker Machine setup correctly: https://docs.docker.com/machine/ * If you are using rootless podman, you might need to set up your local socket and service ERRMSG end # Pass on all the logging from docker-api to the beaker logger instance ::Docker.logger = @logger # Find out what kind of remote instance we are talking against if @docker_version['Version'].include?('swarm') @docker_type = 'swarm' raise "Using Swarm with beaker requires a private registry. Please setup the private registry and set the 'DOCKER_REGISTRY' env var" unless ENV['DOCKER_REGISTRY'] @registry = ENV.fetch('DOCKER_REGISTRY', nil) elsif ::Docker.respond_to?(:podman?) && ::Docker.podman? @docker_type = 'podman' else @docker_type = 'docker' end end def install_and_run_ssh(host) def host.enable_root_login(host, _opts) logger.debug("Root login already enabled for #{host}") end # If the container is running ssh as its init process then this method # will cause issues. if Array(host[:docker_cmd]).first&.include?('sshd') def host.ssh_service_restart self[:docker_container].exec(%w[kill -1 1]) end end host['dockerfile'] || host['use_image_entry_point'] end def get_container_opts(host, image_name) container_opts = {} container_opts['ExposedPorts'] = { '22/tcp' => {} } if host['dockerfile'] container_opts.merge!({ 'Image' => image_name, 'Hostname' => host.name, 'HostConfig' => { 'PortBindings' => { '22/tcp' => [{ 'HostPort' => rand(1025..9999).to_s, 'HostIp' => '0.0.0.0' }], }, 'PublishAllPorts' => true, 'RestartPolicy' => { 'Name' => 'always', }, }, }) end def get_container_image(host) @logger.debug('Creating image') return ::Docker::Image.create('fromImage' => host['image']) if host['use_image_as_is'] dockerfile = host['dockerfile'] if dockerfile # assume that the dockerfile is in the repo and tests are running # from the root of the repo; maybe add support for external Dockerfiles # with external build dependencies later. raise "Unable to find dockerfile at #{dockerfile}" unless File.exist?(dockerfile) dir = File.expand_path(dockerfile).chomp(dockerfile) return ::Docker::Image.build_from_dir( dir, { 'dockerfile' => dockerfile, :rm => true, :buildargs => buildargs_for(host), }, ) elsif host['use_image_entry_point'] df = <<-DF FROM #{host['image']} EXPOSE 22 DF cmd = host['docker_cmd'] df += cmd if cmd return ::Docker::Image.build(df, { rm: true, buildargs: buildargs_for(host) }) end ::Docker::Image.build(dockerfile_for(host), { rm: true, buildargs: buildargs_for(host) }) end # Nested Docker scenarios def nested_docker? ENV['DOCKER_IN_DOCKER'] || ENV.fetch('WSLENV', nil) end # Find out where the ssh port is from the container # When running on swarm DOCKER_HOST points to the swarm manager so we have to get the # IP of the swarm slave via the container data # When we are talking to a normal docker instance DOCKER_HOST can point to a remote docker instance. def get_ssh_connection_info(container) ssh_connection_info = { ip: nil, port: nil, } container_json = container.json network_settings = container_json['NetworkSettings'] host_config = container_json['HostConfig'] ip = nil port = nil # Talking against a remote docker host which is a normal docker host if @docker_type == 'docker' && ENV.fetch('DOCKER_HOST', nil) && !ENV.fetch('DOCKER_HOST', '').include?(':///') && !nested_docker? ip = URI.parse(ENV.fetch('DOCKER_HOST', nil)).host elsif in_container? && !nested_docker? # Swarm or local docker host gw = network_settings['Gateway'] ip = gw unless gw.nil? || gw.empty? else # The many faces of container networking # Host to Container port22 = network_settings.dig('PortBindings', '22/tcp') port22 = network_settings.dig('Ports', '22/tcp') if port22.nil? && network_settings.key?('Ports') && !nested_docker? ip = port22[0]['HostIp'] if port22 port = port22[0]['HostPort'] if port22 # Container to container unless ip && port ip = network_settings['IPAddress'] port = (ip && !ip.empty?) ? 22 : nil end # Container through gateway unless ip && port ip = network_settings['Gateway'] if ip && !ip.empty? port22 = network_settings.dig('PortBindings', '22/tcp') port = port22[0]['HostPort'] if port22 else port = nil end end # Legacy fallback unless ip && port port22 = network_settings.dig('Ports', '22/tcp') ip = port22[0]['HostIp'] if port22 port = port22[0]['HostPort'] if port22 end end if host_config['NetworkMode'] != 'slirp4netns' && network_settings['IPAddress'] && !network_settings['IPAddress'].empty? ip = network_settings['IPAddress'] if ip.nil? else port22 = network_settings.dig('Ports', '22/tcp') port = port22[0]['HostPort'] if port22 end ssh_connection_info[:ip] = (ip == '0.0.0.0') ? '127.0.0.1' : ip ssh_connection_info[:port] = port || '22' ssh_connection_info end def provision @logger.notify 'Provisioning docker' @hosts.each do |host| @logger.notify "provisioning #{host.name}" image = get_container_image(host) image.tag({ repo: host['tag'] }) if host['tag'] if @docker_type == 'swarm' image_name = "#{@registry}/beaker/#{image.id}" ret = ::Docker::Image.search(term: image_name) if ret.first.nil? @logger.debug('Image does not exist on registry. Pushing.') image.tag({ repo: image_name, force: true }) image.push end else image_name = image.id end ### BEGIN CONTAINER OPTIONS MANGLING ### container_opts = get_container_opts(host, image_name) if host['dockeropts'] || @options[:dockeropts] dockeropts = host['dockeropts'] || @options[:dockeropts] dockeropts.each do |k, v| container_opts[k] = v end end container = find_container(host) # Provisioning - Only provision if the host's container can't be found # via its name or ID if container.nil? unless host['mount_folders'].nil? container_opts['HostConfig'] ||= {} container_opts['HostConfig']['Binds'] = host['mount_folders'].values.map do |mount| host_path = File.expand_path(mount['host_path']) # When using docker_toolbox and getting a "(Driveletter):/" path, convert windows path to VM mount host_path = "/#{host_path.gsub(/^.:/, host_path[/^(.)/].downcase)}" if ENV['DOCKER_TOOLBOX_INSTALL_PATH'] && host_path =~ %r{^.:/} a = [host_path, mount['container_path']] # TODO: rewrite this part if mount.key?('opts') a << mount['opts'] if mount.key?('opts') else a << mount['opts'] = 'z' end a.join(':') end end container_opts['Env'] = host['docker_env'] if host['docker_env'] # Fixup privileges # # If the user has specified CAPs, then we cannot be privileged # # If the user has not specified CAPs, we will default to privileged for # compatibility with worst practice if host['docker_cap_add'] container_opts['HostConfig']['CapAdd'] = host['docker_cap_add'] container_opts['HostConfig'].delete('Privileged') else container_opts['HostConfig']['Privileged'] = container_opts['HostConfig']['Privileged'].nil? ? true : container_opts['HostConfig']['Privileged'] end container_opts['name'] = (host['docker_container_name'] || ['beaker', host.name, SecureRandom.uuid.split('-').last].join('-')) if host['docker_port_bindings'] container_opts['ExposedPorts'] = {} if container_opts['ExposedPorts'].nil? host['docker_port_bindings'].each_pair do |port, bind| container_opts['ExposedPorts'][port.to_s] = {} container_opts['HostConfig']['PortBindings'][port.to_s] = bind end end ### END CONTAINER OPTIONS MANGLING ### @logger.debug("Creating container from image #{image_name}") ok = false retries = 0 while !ok && (retries < 5) container = ::Docker::Container.create(container_opts) ssh_info = get_ssh_connection_info(container) if ::Docker.rootless? && ssh_info[:ip] == '127.0.0.1' && (ssh_info[:port].to_i < 1024) @logger.debug("#{host} was given a port less than 1024 but you are connecting to a rootless instance, retrying") container.delete(force: true) container = nil retries += 1 next end ok = true end else host['use_existing_container'] = true end if container.nil? raise 'Cannot continue because no existing container ' \ 'could be found and provisioning is disabled.' end fix_ssh(container) if @options[:provision] == false @logger.debug("Starting container #{container.id}") container.start begin container.stats rescue StandardError => e container.delete(force: true) raise "Container '#{container.id}' in a bad state: #{e}" end # Preserve the ability to talk directly to the underlying API # # You can use any method defined by the docker-api gem on this object # https://github.com/swipely/docker-api host[:docker_container] = container if install_and_run_ssh(host) @logger.notify("Installing ssh components and starting ssh daemon in #{host} container") install_ssh_components(container, host) # run fixssh to configure and start the ssh service fix_ssh(container, host) end ssh_connection_info = get_ssh_connection_info(container) ip = ssh_connection_info[:ip] port = ssh_connection_info[:port] @logger.info("Using container connection at #{ip}:#{port}") forward_ssh_agent = @options[:forward_ssh_agent] || false host['ip'] = ip host['port'] = port host['ssh'] = { password: root_password, port: port, forward_agent: forward_ssh_agent, auth_methods: %w[password publickey hostbased keyboard-interactive], } @logger.debug("node available as ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@#{ip} -p #{port}") host['docker_container_id'] = container.id host['docker_image_id'] = image.id host['vm_ip'] = container.json['NetworkSettings']['IPAddress'].to_s def host.reboot @logger.warn('Rebooting containers is ineffective...ignoring') end end hack_etc_hosts @hosts, @options end # This sideloads sshd after a container starts def install_ssh_components(container, host) case host['platform'] when /ubuntu/, /debian/, /cumulus/ container.exec(%w[apt-get update]) container.exec(%w[apt-get install -y openssh-server openssh-client]) container.exec(%w[sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/*]) when /el-[89]/, /fedora-(2[2-9]|3[0-9])/, /amazon-2023/ container.exec(%w[dnf clean all]) container.exec(%w[dnf install -y sudo openssh-server openssh-clients]) container.exec(%w[ssh-keygen -A]) container.exec(%w[sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/*]) when /^el-/, /centos/, /fedora/, /redhat/, /eos/ container.exec(%w[yum clean all]) container.exec(%w[yum install -y sudo openssh-server openssh-clients]) container.exec(%w[ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key]) container.exec(%w[ssh-keygen -t dsa -f /etc/ssh/ssh_host_dsa_key]) container.exec(%w[sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/*]) when /opensuse/, /sles/ container.exec(%w[zypper -n in openssh]) container.exec(%w[ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key]) container.exec(%w[ssh-keygen -t dsa -f /etc/ssh/ssh_host_dsa_key]) container.exec(%w[sed -ri 's/^#?UsePAM .*/UsePAM no/' /etc/ssh/sshd_config]) when /archlinux/ container.exec(%w[pacman --noconfirm -Sy archlinux-keyring]) container.exec(%w[pacman --noconfirm -Syu]) container.exec(%w[pacman -S --noconfirm openssh]) container.exec(%w[ssh-keygen -A]) container.exec(%w[sed -ri 's/^#?UsePAM .*/UsePAM no/' /etc/ssh/sshd_config]) container.exec(%w[systemctl enable sshd]) when /alpine/ container.exec(%w[apk add --update openssh]) container.exec(%w[ssh-keygen -A]) else # TODO: add more platform steps here raise "platform #{host['platform']} not yet supported on docker" end # Make sshd directory, set root password container.exec(%w[mkdir -p /var/run/sshd]) container.exec(['/bin/sh', '-c', "echo root:#{root_password} | chpasswd"]) end def cleanup @logger.notify 'Cleaning up docker' @hosts.each do |host| # leave the container running if docker_preserve_container is set # setting docker_preserve_container also implies docker_preserve_image # is set, since you can't delete an image that's the base of a running # container next if host['docker_preserve_container'] container = find_container(host) if container @logger.debug("stop container #{container.id}") begin container.kill sleep 2 # avoid a race condition where the root FS can't unmount rescue Excon::Errors::ClientError => e @logger.warn("stop of container #{container.id} failed: #{e.response.body}") end @logger.debug("delete container #{container.id}") begin container.delete(force: true) rescue Excon::Errors::ClientError => e @logger.warn("deletion of container #{container.id} failed: #{e.response.body}") end end # Do not remove the image if docker_preserve_image is set to true, otherwise remove it next if host['docker_preserve_image'] image_id = host['docker_image_id'] if image_id @logger.debug("deleting image #{image_id}") begin ::Docker::Image.remove(image_id) rescue Excon::Errors::ClientError => e @logger.warn("deletion of image #{image_id} failed: #{e.response.body}") rescue ::Docker::Error::DockerError => e @logger.warn("deletion of image #{image_id} caused internal Docker error: #{e.message}") end else @logger.warn("Intended to delete the host's docker image, but host['docker_image_id'] was not set") end end end private def root_password 'root' end def buildargs_for(host) docker_buildargs = {} docker_buildargs_env = ENV.fetch('DOCKER_BUILDARGS', nil) docker_buildargs_env&.split(/ +|\t+/)&.each do |arg| key, value = arg.split('=') if key docker_buildargs[key] = value else @logger.warn("DOCKER_BUILDARGS environment variable appears invalid, no key found for value #{value}") end end buildargs = if docker_buildargs.empty? host['docker_buildargs'] || {} else docker_buildargs end @logger.debug("Docker build buildargs: #{buildargs}") JSON.generate(buildargs) end def dockerfile_for(host) # specify base image dockerfile = <<~DF FROM #{host['image']} ENV container docker DF # Commands before any other commands. Can be used for eg. proxy configuration dockerfile += (host['docker_image_first_commands'] || []).map { |cmd| "RUN #{cmd}\n" }.join # add platform-specific actions service_name = 'sshd' additional_packages = host_packages(host) case host['platform'] when /ubuntu/, /debian/ service_name = 'ssh' dockerfile += <<~DF RUN apt-get update \ && apt-get install -y openssh-server openssh-client #{additional_packages.join(' ')} \ && sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/* DF when /cumulus/ dockerfile += <<~DF RUN apt-get update \ && apt-get install -y openssh-server openssh-client #{additional_packages.join(' ')} DF when /el-[89]/, /fedora-(2[2-9]|3)/, /amazon-2023/ dockerfile += <<~DF RUN dnf clean all \ && dnf install -y sudo openssh-server openssh-clients #{additional_packages.join(' ')} \ && ssh-keygen -A \ && sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/* DF when /^el-/, /centos/, /fedora/, /redhat/, /eos/ dockerfile += <<~DF RUN yum clean all \ && yum install -y sudo openssh-server openssh-clients #{additional_packages.join(' ')} \ && ssh-keygen -A \ && sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/* DF when /opensuse/, /sles/ dockerfile += <<~DF RUN zypper -n in openssh #{additional_packages.join(' ')} \ && ssh-keygen -A \ && sed -ri 's/^#?UsePAM .*/UsePAM no/' /etc/ssh/sshd_config \ && sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/* DF when /archlinux/ dockerfile += <<~DF RUN pacman --sync --refresh --noconfirm archlinux-keyring \ && pacman --sync --refresh --noconfirm --sysupgrade \ && pacman --sync --noconfirm #{additional_packages.join(' ')} \ && ssh-keygen -A \ && sed -ri 's/^#?UsePAM .*/UsePAM no/' /etc/ssh/sshd_config \ && systemctl enable sshd DF else # TODO: add more platform steps here raise "platform #{host['platform']} not yet supported on docker" end # Make sshd directory, set root password dockerfile += <<~DF RUN mkdir -p /var/run/sshd \ && echo root:#{root_password} | chpasswd DF # Configure sshd service to allowroot login using password # Also, disable reverse DNS lookups to prevent every. single. ssh # operation taking 30 seconds while the lookup times out. # Also unbreak users with a bunch of SSH keys loaded in their keyring. dockerfile += <<~DF RUN sed -ri \ -e 's/^#?PermitRootLogin .*/PermitRootLogin yes/' \ -e 's/^#?PasswordAuthentication .*/PasswordAuthentication yes/' \ -e 's/^#?UseDNS .*/UseDNS no/' \ -e 's/^#?MaxAuthTries.*/MaxAuthTries 1000/' \ /etc/ssh/sshd_config DF # Any extra commands specified for the host dockerfile += (host['docker_image_commands'] || []).map { |cmd| "RUN #{cmd}\n" }.join # Override image entrypoint dockerfile += "ENTRYPOINT #{host['docker_image_entrypoint']}\n" if host['docker_image_entrypoint'] # How to start a sshd on port 22. May be an init for more supervision # Ensure that the ssh server can be restarted (done from set_env) and container keeps running cmd = host['docker_cmd'] || ['sh', '-c', "service #{service_name} start; tail -f /dev/null"] dockerfile += <<~DF EXPOSE 22 CMD #{cmd} DF @logger.debug("Dockerfile is #{dockerfile}") dockerfile end # a puppet run may have changed the ssh config which would # keep us out of the container. This is a best effort to fix it. # Optionally pass in a host object to to determine which ssh # restart command we should try. def fix_ssh(container, host = nil) @logger.debug("Fixing ssh on container #{container.id}") container.exec(['sed', '-ri', '-e', 's/^#?PermitRootLogin .*/PermitRootLogin yes/', '-e', 's/^#?PasswordAuthentication .*/PasswordAuthentication yes/', '-e', 's/^#?UseDNS .*/UseDNS no/', # Unbreak users with a bunch of SSH keys loaded in their keyring. '-e', 's/^#?MaxAuthTries.*/MaxAuthTries 1000/', '/etc/ssh/sshd_config',]) return unless host case host['platform'] when /alpine/ container.exec(%w[/usr/sbin/sshd]) when /ubuntu/, /debian/ container.exec(%w[service ssh restart]) else container.exec(%w[service sshd restart]) end end # return the existing container if we're not provisioning # and docker_container_name is set def find_container(host) id = host['docker_container_id'] name = host['docker_container_name'] return unless id || name containers = ::Docker::Container.all if id @logger.debug("Looking for an existing container with ID #{id}") container = containers.find { |c| c.id == id } end if name && container.nil? @logger.debug("Looking for an existing container with name #{name}") container = containers.find do |c| c.info['Names'].include? "/#{name}" end end return container unless container.nil? @logger.debug('Existing container not found') nil end # return true if we are inside a docker container def in_container? File.file?('/.dockerenv') end end end