require 'cl'
require 'fileutils'
require 'forwardable'
require 'shellwords'
require 'dpl/helper/assets'
require 'dpl/helper/cmd'
require 'dpl/helper/config_file'
require 'dpl/helper/env'
require 'dpl/helper/interpolate'
require 'dpl/helper/memoize'
require 'dpl/helper/squiggle'
require 'dpl/provider/dsl'
require 'dpl/provider/examples'
require 'dpl/version'
module Dpl
# Base class for all concrete providers that `dpl` supports.
#
# These are subclasses of `Cl::Cmd` which means they are going to be detected
# by the first argument passed to `dpl [provider]`, instantiated, and run.
#
# Implementors are encouraged to use the provider DSL to declare various
# features, requirements, and attributes that apply to their provider, to
# implement any of the following stages (methods) according to their needs
# and semantics:
#
# * init
# * install
# * login
# * setup
# * validate
# * prepare
# * deploy
# * finish
#
# The main logic should sit in the `deploy` stage.
#
# If at any time the method `error` is called, or any exception raised the
# deploy process will be halted, and subsequent stages skipped. However, the
# stage `finish` will run even if previous stages have raised an error,
# giving the provider the opportunity to potentially clean up stage.
#
# In addition to this the following methods will be called if implemented
# by the provider:
#
# * run_cmd
# * add_key
# * remove_key
#
# Like the `finish` stage, the method `remove_key` will be called even if
# previous stages have raised an error.
#
# See the respective method's documentation for details on these.
#
# The following stages are not meant to be overwritten, but considered
# internal:
#
# * before_install
# * before_setup
# * before_prepare
# * before_finish
#
# Dependencies declared as required, such as APT, NPM, or Python are going to
# be installed as part of the `before_install` stage .
#
# Cleanup is run as part of the `before_prepare` stage if the option
# `--cleanup` was given. This will use `git stash --all` in order to reset
# the working directory to the committed state, and cleanup any left over
# artifacts from the build process. Providers can use the DSL method `keep`
# in order to declare known artifacts (such as CLI tooling installed to the
# working directory) that needs to be moved out of the way and restored after
# the cleanup process. (It is recommended to place such artifacts outside of
# the build working directory though, for example in `~/.dpl`).
#
# The method `run_cmd` is called for each command specified using the `--run`
# option. By default, these command are going to be run as local shell
# commands, but providers can choose to overwrite this method in order to run
# the command on a remote machine.
#
# @see https://github.com/svenfuchs/cl Cl's documentation for details on how
# providers (commands) are declared and run.
class Provider < Cl::Cmd
extend Dsl, Forwardable
include Assets, Env, ConfigFile, FileUtils, Interpolate, Memoize, Squiggle
class << self
def examples
@examples ||= super || Examples.new(self).cmds
end
def move_files(ctx)
ctx.move_files(move) if move.any?
end
def unmove_files(ctx)
ctx.unmove_files(move) if move.any?
end
def install_deps?
apt? || gem? || npm? || pip?
end
def install_deps(ctx)
ctx.apts_get(apt) if apt?
ctx.gems_require(gem) if gem?
npm.each { |npm| ctx.npm_install *npm } if npm?
pip.each { |pip| ctx.pip_install *pip } if pip?
end
def validate_runtimes(ctx)
ctx.validate_runtimes(runtimes) if runtimes.any?
end
end
# Fold names to display in the build log.
FOLDS = {
init: 'Initialize deployment',
setup: 'Setup deployment',
validate: 'Validate deployment',
install: 'Install deployment dependencies',
login: 'Authenticate deployment',
prepare: 'Prepare deployment',
deploy: 'Run deployment',
finish: 'Finish deployment',
}
# Deployment process stages.
#
# In addition to the stages listed here the stage `finish` will be run at
# the end of the process.
#
# Also, the methods `add_key` (called before `setup`), `remove_key` (called
# before `finish`), and `run_cmd` (called after `deploy`) may be of
# interest to implementors.
STAGES = %i(
init
install
login
setup
validate
prepare
deploy
)
abstract
arg :provider, 'The provider name', required: true
opt '--cleanup', 'Clean up build artifacts from the Git working directory before the deployment', negate: %w(skip)
opt '--run CMD', 'Commands to execute after the deployment finished successfully', type: :array
opt '--stage NAME', 'Execute the given stage(s) only', type: :array, internal: true, default: STAGES
opt '--backtrace', 'Print the backtrace for exceptions', internal: true
opt '--fold', 'Wrap log output in folds', internal: true
opt '--edge', internal: true
msgs before_install: 'Installing deployment dependencies',
before_setup: 'Setting the build environment up for the deployment',
setup_git_ssh: 'Setting up git-ssh',
cleanup: 'Cleaning up git repository with `git stash --all`',
ssh_keygen: 'Generating SSH key',
setup_git_ua: 'Setting up git HTTP user agent',
ssh_remote_host: 'SSH remote is %s at port %s',
ssh_try_connect: 'Waiting for SSH connection ...',
ssh_connected: 'SSH connection established.',
ssh_failed: 'Failed to establish SSH connection.'
def_delegators :'self.class', :status, :full_name, :install_deps,
:install_deps?, :keep, :move_files, :unmove_files, :needs?, :runtimes,
:validate_runtimes, :user_agent
def_delegators :ctx, :apt_get, :gem_require, :npm_install, :pip_install,
:build_dir, :build_number, :encoding, :file_size, :git_author_email,
:git_author_name, :git_branch, :git_branch, :git_commit_author,
:git_commit_msg, :git_commit_msg, :git_dirty?, :git_log, :git_log,
:git_ls_files, :git_ls_remote?, :git_remote_urls, :git_remote_urls,
:git_rev_parse, :git_rev_parse, :git_sha, :git_tag, :last_err, :last_out,
:last_out, :logger, :machine_name, :mv, :node_version, :node_version,
:npm_version, :rendezvous, :rendezvous, :repo_slug, :sleep, :sleep,
:ssh_keygen, :success?, :test?, :test?, :tmp_dir, :tty?, :which, :which,
:write_file, :write_netrc
attr_reader :repo_name, :key_name
def initialize(ctx, *args)
@repo_name = ctx.repo_name
@key_name = ctx.machine_name
super
end
# Runs all stages, all commands provided by the user, as well as the final
# stage `finish` (which will be run even if an error has been raised during
# previous stages).
def run
stages = stage.select { |stage| run_stage?(stage) }
stages.each { |stage| run_stage(stage) }
run_cmds
rescue Error
raise
rescue Exception => e
raise Error.new("#{e.message} (#{e.class})", backtrace: backtrace? ? e.backtrace : nil) unless test?
raise
ensure
run_stage(:finish, fold: false) if finish?
end
# Whether or not a stage needs to be run
def run_stage?(stage)
respond_to?(:"before_#{stage}") || respond_to?(stage)
end
def finish?
stage.size == STAGES.size
end
# Runs a single stage.
#
# For each stage the base class has the opportunity to implement a `before`
# stage method, in order to apply default behaviour. Provider implementors
# are asked to not overwrite these methods.
#
# Any log output from both the before stage and stage method is going to be
# folded in the resulting build log.
def run_stage(stage, opts = {})
fold(stage, opts) do
send(:"before_#{stage}") if respond_to?(:"before_#{stage}")
send(stage) if respond_to?(stage)
end
end
# Initialize the deployment process.
#
# This will:
#
# * Displays warning messages about the provider's maturity status, and deprecated
# options used.
# * Setup a ~/.dpl working directory
# * Move files out of the way that have been declared as such
def before_init
warn status.msg if status && status.announce?
deprecations.each { |(key, msg)| ctx.deprecate_opt(key, msg) }
setup_dpl_dir
move_files(ctx)
end
# Install APT, NPM, and Python dependencies as declared by the provider.
def before_install
validate_runtimes(ctx)
return unless install_deps?
info :before_install
install_deps(ctx)
end
# Sets the build environment up for the deployment.
#
# This will:
#
# * Setup a ~/.dpl working directory
# * Create a temporary, per build SSH key, and call `add_key` if the feature `ssh_key` has been declared as required.
# * Setup git config (email and user name) if the feature `git` has been declared as required.
# * Either set or unset the environment variable `GIT_HTTP_USER_AGENT` depending if the feature `git_http_user_agent` has been declared as required.
def before_setup
info :before_setup
setup_ssh_key if needs?(:ssh_key)
setup_git_config if needs?(:git)
setup_git_http_user_agent
end
# Prepares the deployment by cleaning up the working directory.
#
# @see Provider#cleanup
def before_prepare
cleanup if cleanup?
end
# Runs each command as given by the user using the `--run` option.
#
# For a command that matches `restart` the method `restart` will be called
# (which can be overwritten by providers, e.g. in order to restart service
# instances).
#
# All other commands will be passed to the method `run_cmd`. By default this
# will be run as a shell command locally, but providers can choose to
# overwrite this method in order to run the command on a remote machine.
def run_cmds
Array(opts[:run]).each do |cmd|
cmd.downcase == 'restart' ? restart : run_cmd(cmd)
end
end
def run_cmd(cmd)
cmd.downcase == 'restart' ? restart : shell(cmd)
end
# Finalizes the deployment process.
#
# This will:
#
# * Call the method `remove_key` if implemented by the provider, and if the
# feature `ssh_key` has been declared as required.
# * Revert the cleanup process, i.e. restore files moved out of the way
# during `cleanup`.
# * Remove the temporary directory `~/.dpl`
def before_finish
remove_key if needs?(:ssh_key) && respond_to?(:remove_key)
uncleanup if cleanup?
unmove_files(ctx)
remove_dpl_dir
end
# Resets the current working directory to the commited state.
#
# Cleanup will use `git stash --all` in order to reset the working
# directory to the committed state, and cleanup any left over artifacts
# from the build process. Providers can use the DSL method `keep` in order
# to declare known artifacts (such as CLI tooling installed to the working
# directory) that needs to be moved out of the way and restored after the
# cleanup process.
def cleanup
info :cleanup
keep.each { |path| shell "mv ./#{path} ~/#{path}", echo: false, assert: false }
shell 'git stash --all'
keep.each { |path| shell "mv ~/#{path} ./#{path}", echo: false, assert: false }
end
# Restore files that have been cleaned up.
def uncleanup
shell 'git stash pop', assert: false
end
# Creates the directory `~/.dpl` as an internal working directory.
def setup_dpl_dir
rm_rf '~/.dpl'
mkdir_p '~/.dpl'
chmod 0700, '~/.dpl'
end
# Remove the internal working directory `~/.dpl`.
def remove_dpl_dir
rm_rf '~/.dpl'
end
# Creates an SSH key, and sets up git-ssh if needed.
#
# This will:
#
# * Create a temporary, per build SSH key.
# * Setup a `git-ssh` executable to use that key.
# * Call the method `add_key` if implemented by the provider.
def setup_ssh_key
ssh_keygen(key_name, '~/.dpl/id_rsa')
setup_git_ssh('~/.dpl/id_rsa')
add_key('~/.dpl/id_rsa.pub') if respond_to?(:add_key)
end
# Setup git config
#
# This adds the current user's name and email address (as user@localhost)
# to the git config.
def setup_git_config
shell "git config user.email >/dev/null 2>/dev/null || git config user.email `whoami`@localhost", echo: false, assert: false
shell "git config user.name >/dev/null 2>/dev/null || git config user.name `whoami`", echo: false, assert: false
end
# Sets up `git-ssh` and the GIT_SSH env var
def setup_git_ssh(key)
info :setup_git_ssh
path, conf = '~/.dpl/git-ssh', asset(:dpl, :git_ssh).read % expand(key)
open(path, 'w+') { |file| file.write(conf) }
chmod(0740, path)
ENV['GIT_SSH'] = expand(path)
end
# Generates an SSH key.
def ssh_keygen(key, path)
info :ssh_keygen
ctx.ssh_keygen(key, expand(path))
end
# Sets or unsets the environment variable `GIT_HTTP_USER_AGENT`.
def setup_git_http_user_agent
return ENV.delete('GIT_HTTP_USER_AGENT') unless needs?(:git_http_user_agent)
info :setup_git_ua
ENV['GIT_HTTP_USER_AGENT'] = user_agent(git: `git --version`[/[\d\.]+/])
end
# Waits for SSH access on the given host and port.
#
# This will try to connect to the given SSH host and port, and keep
# retrying 30 times, waiting a second inbetween retries.
def wait_for_ssh_access(host, port)
info :ssh_remote_host, host, port
1.upto(20) { try_ssh_access(host, port) && break || sleep(3) }
success? ? info(:ssh_connected) : error(:ssh_failed)
end
# Tries to connect to the given SSH host and port.
def try_ssh_access(host, port)
info :ssh_try_connect
shell "#{ENV['GIT_SSH']} #{host} -p #{port} 2>&1 | grep -c 'PTY allocation request failed' > /dev/null", echo: false, assert: false
end
# Creates a log fold.
#
# Folds any log output from the given block into a fold with the given
# name.
def fold(name, opts = {}, &block)
return yield unless fold?(name, opts)
title = FOLDS[name] || "deploy.#{name}"
ctx.fold(title, &block)
end
# Checks if the given stage needs to be folded.
#
# Depends on the option `--fold`, also omits folds for the init and finish
# stages. Can be overwritten by passing `fold: false`.
def fold?(name, opts = {})
!opts[:fold].is_a?(FalseClass) && super() && !%i(init).include?(name)
end
# Runs a script as a shell command.
#
# Scripts can be stored as separate files (assets) in the directory
# `lib/dpl/assets/[provider]`.
#
# This is meant for large shell commands that would be hard to read if
# embedded in Ruby code. Storing them as separate files helps with proper
# syntax highlighting etc in editors, and allows to execute them for
# testing purposes.
#
# Scripts can have interpolation variables. See Dpl::Interpolate for
# details on interpolating variables.
#
# See Ctx::Bash#shell for details on the options accepted.
def script(name, opts = {})
opts[:assert] = name if opts[:assert].is_a?(TrueClass)
shell(asset(name).read, opts.merge(echo: false))
end
# Runs a single shell command.
#
# Shell commands can have interpolation variables. See Dpl::Interpolate for
# details on interpolating variables.
#
# See Ctx::Bash#shell for details on the options accepted.
def shell(cmd, *args)
opts = args.last.is_a?(Hash) ? args.pop : {}
cmd = Cmd.new(self, cmd, opts)
ctx.shell(cmd)
end
# @!method print
# Prints a partial message to stdout
#
# This method does not append a newline character to the given message,
# which usually is not the desired behaviour. The method is intended to be
# used if an initial, partial message is supposed to be printed, which will
# be completed later (using the method `info`).
#
# For example:
#
# print 'Starting a long running task ...'
# run_long_running_task
# info 'done.'
#
# Messages support interpolation variables. See Dpl::Interpolate for
# details on interpolating variables.
# @!method info
# Outputs an info message to stdout
#
# This method is intended to be used for default, info level messages that
# are supposed to show up in the build log.
#
# @!method warn
# Outputs an warning message to stderr
#
# This method is intended to be used for warning messages that are supposed
# to show up in the build log, but do not qualify as errors that would
# abort the deployment process. The warning will be highlighted as red
# text. Use sparingly.
#
# Messages support interpolation variables. See Dpl::Interpolate for
# details on interpolating variables.
# @!method error
# Outputs an error message to stderr, and raises an error, halting the
# deployment process.
#
# This method is intended to be used for all error conditions that require
# the deployment process to be aborted.
#
# Messages support interpolation variables. See Dpl::Interpolate for
# details on interpolating variables.
%i(print info warn error).each do |level|
define_method(level) do |msg, *args|
msg = interpolate(self.msg(msg), args) if msg.is_a?(Symbol)
ctx.send(level, msg)
end
end
# @!method cmd
# Looks up a shell command from the commands declared by the provider
# (using the class level DSL).
#
# Not usually useful to be used by provider implementors directly. Use the
# method `shell` in order to execute shell commands.
# @!method err
# Looks up an error message from the error messages declared by the
# provider (using the class level DSL), as needed by the option `assert`
# when passed to the method `shell`.
# @!method msg
# Looks up a message from the messages declared by the provider (using the
# class level DSL).
#
# For example, a message declared on the class body like so:
#
# ```ruby
# msgs commit_msg: 'Commit build artifacts on build %{build_number}'
# ```
#
# could be used by the implementation like so:
#
# ```ruby
# def commit_msg
# interpolate(msg(:commit_msg))
# end
# ```
#
# Note that the the method `interpolate` needs to be used in order to
# interpolate variables used in a message (if any).
%i(cmd err msg str).each do |name|
define_method(name) do |*keys|
key = keys.detect { |key| key.is_a?(Symbol) }
self.class.send(:"#{name}s")[key] if key
end
end
# Escapes the given string so it can be safely used in Bash.
def escape(str)
Shellwords.escape(str)
end
# Double quotes the given string.
def quote(str)
%("#{str.gsub('"', '\"')}")
end
# Outdents the given string.
#
# @see Dpl::Squiggle
def sq(str)
self.class.sq(str)
end
# Generate shell option strings to be passed to a shell command.
#
# This generates strings like `--key="value"` for the option keys passed.
# These keys are supposed to correspond to methods on the provider
# instance, which will be called in order to determine the option value.
#
# If the returned value is an array then the option will be repeated
# multiple times. If it is a String then it will be double quoted.
# Otherwise it is assumed to be a flag that does not have a value.
#
# @option prefix [String] Use this to set a single dash as an option prefix (defaults to two dashes).
# @option dashed [Boolean] Use this to dasherize the option key (rather than underscore it, defaults to underscore).
def opts_for(keys, opts = {})
strs = Array(keys).map { |key| opt_for(key, opts) if send(:"#{key}?") }.compact
strs.join(' ') if strs.any?
end
def opt_for(key, opts = {})
case value = send(key)
when String then "#{opt_key(key, opts)}=#{value.inspect}"
when Array then value.map { |value| "#{opt_key(key, opts)}=#{value.inspect}" }
else opt_key(key, opts)
end
end
def opt_key(key, opts)
"#{opts[:prefix] || '--'}#{opts[:dashed] ? key.to_s.gsub('_', '-') : key}"
end
# Compacts the given hash by rejecting nil values.
def compact(hash)
hash.reject { |_, value| value.nil? }
end
# Returns a new hash with the given keys selected from the given hash.
def only(hash, *keys)
hash.select { |key, _| keys.include?(key) }
end
# Deep symbolizes the given hash's keys
def symbolize(obj)
case obj
when Hash
obj.map { |key, obj| [key.to_sym, symbolize(obj)] }.to_h
when Array
obj.map { |obj| symbolize(obj) }
else
obj
end
end
def file?(path)
File.file?(expand(path))
end
def mkdir_p(path)
FileUtils.mkdir_p(expand(path))
end
def chmod(perm, path)
super(perm, expand(path))
end
def mv(src, dest)
super(expand(src), expand(dest))
end
def rm_rf(path)
super(expand(path))
end
def open(path, *args, &block)
File.open(expand(path), *args, &block)
end
def read(path)
File.read(expand(path))
end
def expand(*args)
File.expand_path(*args)
end
end
end
require 'dpl/providers'