modules/mu/clouds/aws/database.rb in cloud-mu-2.1.0beta vs modules/mu/clouds/aws/database.rb in cloud-mu-3.0.0beta
- old
+ new
@@ -17,42 +17,26 @@
module MU
class Cloud
class AWS
# A database as configured in {MU::Config::BasketofKittens::databases}
class Database < MU::Cloud::Database
- @deploy = nil
- @config = nil
- attr_reader :mu_name
- attr_reader :cloud_id
- attr_reader :config
- attr_reader :groomer
- @cloudformation_data = {}
- attr_reader :cloudformation_data
-
- # @param mommacat [MU::MommaCat]: A {MU::Mommacat} object containing the deploy of which this resource is/will be a member.
- # @param kitten_cfg [Hash]: The fully parsed and resolved {MU::Config} resource descriptor as defined in {MU::Config::BasketofKittens::databases}
- def initialize(mommacat: nil, kitten_cfg: nil, mu_name: nil, cloud_id: nil)
- @deploy = mommacat
- @config = MU::Config.manxify(kitten_cfg)
- @cloud_id ||= cloud_id
- # @mu_name = mu_name ? mu_name : @deploy.getResourceName(@config["name"])
+ # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
+ # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
+ def initialize(**args)
+ super
@config["groomer"] = MU::Config.defaultGroomer unless @config["groomer"]
@groomclass = MU::Groomer.loadGroomer(@config["groomer"])
- if !mu_name.nil?
- @mu_name = mu_name
- else
- @mu_name ||=
- if @config and @config['engine'] and @config["engine"].match(/^sqlserver/)
- @deploy.getResourceName(@config["name"], max_length: 15)
- else
- @deploy.getResourceName(@config["name"], max_length: 63)
- end
+ @mu_name ||=
+ if @config and @config['engine'] and @config["engine"].match(/^sqlserver/)
+ @deploy.getResourceName(@config["name"], max_length: 15)
+ else
+ @deploy.getResourceName(@config["name"], max_length: 63)
+ end
- @mu_name.gsub(/(--|-$)/i, "").gsub(/(_)/, "-").gsub!(/^[^a-z]/i, "")
- end
+ @mu_name.gsub(/(--|-$)/i, "").gsub(/(_)/, "-").gsub!(/^[^a-z]/i, "")
end
# Called automatically by {MU::Deploy#createResources}
# @return [String]: The cloud provider's identifier for this database instance.
def create
@@ -187,12 +171,12 @@
# @param flags [Hash]: Optional flags
# @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching Databases
def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, credentials: nil, flags: {})
map = {}
if cloud_id
- db = MU::Cloud::AWS::Database.getDatabaseById(cloud_id, region: region, credentials: credentials)
- map[cloud_id] = db if db
+ resp = MU::Cloud::AWS::Database.getDatabaseById(cloud_id, region: region, credentials: credentials)
+ map[cloud_id] = resp if resp
end
if tag_value
MU::Cloud::AWS.rds(credentials: credentials, region: region).describe_db_instances.db_instances.each { |db|
resp = MU::Cloud::AWS.rds(credentials: credentials, region: region).list_tags_for_resource(
@@ -310,10 +294,11 @@
config[:db_cluster_identifier] = @config["cluster_identifier"] if @config["add_cluster_node"]
end
if %w{existing_snapshot new_snapshot}.include?(@config["creation_style"])
config[:db_snapshot_identifier] = @config["snapshot_id"]
+ config[:db_cluster_identifier] = @config["cluster_identifier"] if @config["add_cluster_node"]
end
if @config["creation_style"] == "point_in_time"
point_in_time_config = config
point_in_time_config.delete(:db_instance_identifier)
@@ -382,15 +367,15 @@
begin
MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).wait_until(:db_instance_available, db_instance_identifier: @config['identifier']) do |waiter|
# Does create_db_instance implement wait_until_available ?
waiter.max_attempts = nil
- waiter.before_attempt do |attempts|
- MU.log "Waiting for RDS database #{@config['identifier']} to be ready..", MU::NOTICE if attempts % 10 == 0
+ waiter.before_attempt do |w_attempts|
+ MU.log "Waiting for RDS database #{@config['identifier']} to be ready..", MU::NOTICE if w_attempts % 10 == 0
end
- waiter.before_wait do |attempts, resp|
- throw :success if resp.db_instances.first.db_instance_status == "available"
+ waiter.before_wait do |w_attempts, r|
+ throw :success if r.db_instances.first.db_instance_status == "available"
throw :failure if Time.now - wait_start_time > 3600
end
end
rescue Aws::Waiters::Errors::TooManyAttemptsError => e
raise MuError, "Waited #{(Time.now - wait_start_time).round/60*(retries+1)} minutes for #{@config['identifier']} to become available, giving up. #{e}" if retries > 2
@@ -451,15 +436,15 @@
begin
MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).wait_until(:db_instance_available, db_instance_identifier: @config['identifier']) do |waiter|
# Does create_db_instance implement wait_until_available ?
waiter.max_attempts = nil
- waiter.before_attempt do |attempts|
- MU.log "Waiting for RDS database #{@config['identifier'] } to be ready..", MU::NOTICE if attempts % 10 == 0
+ waiter.before_attempt do |w_attempts|
+ MU.log "Waiting for RDS database #{@config['identifier'] } to be ready..", MU::NOTICE if w_attempts % 10 == 0
end
- waiter.before_wait do |attempts, resp|
- throw :success if resp.db_instances.first.db_instance_status == "available"
+ waiter.before_wait do |w_attempts, r|
+ throw :success if r.db_instances.first.db_instance_status == "available"
throw :failure if Time.now - wait_start_time > 2400
end
end
rescue Aws::Waiters::Errors::TooManyAttemptsError => e
raise MuError, "Waited #{(Time.now - wait_start_time).round/60*(retries+1)} minutes for #{@config['identifier']} to become available, giving up. #{e}" if retries > 2
@@ -530,10 +515,14 @@
cluster_config_struct[:source_db_cluster_identifier] = @config["source_identifier"]
cluster_config_struct[:restore_to_time] = @config["restore_time"] unless @config["restore_time"] == "latest"
cluster_config_struct[:use_latest_restorable_time] = true if @config["restore_time"] == "latest"
end
+ if @config['cloudwatch_logs']
+ cluster_config_struct[:enable_cloudwatch_logs_exports ] = @config['cloudwatch_logs']
+ end
+
attempts = 0
begin
resp =
if @config["creation_style"] == "new"
MU.log "Creating new database cluster #{@config['identifier']}"
@@ -653,12 +642,12 @@
subnet_ids << subnet.subnet_id
mu_subnets << {"subnet_id" => subnet.subnet_id}
}
@config['vpc'] = {
- "vpc_id" => vpc_id,
- "subnets" => mu_subnets
+ "vpc_id" => vpc_id,
+ "subnets" => mu_subnets
}
# Default VPC has only public subnets by default so setting publicly_accessible = true
@config["publicly_accessible"] = true
using_default_vpc = true
MU.log "Using default VPC for cache cluster #{@config['identifier']}"
@@ -796,11 +785,19 @@
#we're fine returning nil
end
# Called automatically by {MU::Deploy#createResources}
def groom
- unless @config["create_cluster"]
+ if @config["create_cluster"]
+ @config['cluster_node_count'] ||= 1
+ if @config['cluster_mode'] == "serverless"
+ MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_current_db_cluster_capacity(
+ db_cluster_identifier: @cloud_id,
+ capacity: @config['cluster_node_count']
+ )
+ end
+ else
database = MU::Cloud::AWS::Database.getDatabaseById(@config['identifier'], region: @config['region'], credentials: @config['credentials'])
# Run SQL on deploy
if @config['run_sql_on_deploy']
MU.log "Running initial SQL commands on #{@config['name']}", details: @config['run_sql_on_deploy']
@@ -1426,22 +1423,35 @@
}
}
}
}
+
schema = {
"db_parameter_group_parameters" => rds_parameters_primitive,
"cluster_parameter_group_parameters" => rds_parameters_primitive,
+ "parameter_group_family" => {
+ "type" => "String",
+ "description" => "An RDS parameter group family. See also https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_WorkingWithParamGroups.html"
+ },
"cluster_mode" => {
"type" => "string",
"description" => "The DB engine mode of the DB cluster",
"enum" => ["provisioned", "serverless", "parallelquery", "global"],
"default" => "provisioned"
},
+ "cloudwatch_logs" => {
+ "type" => "array",
+ "default" => ["error"],
+ "items" => {
+ "type" => "string",
+ "enum" => ["error", "general", "audit", "slow_query"],
+ }
+ },
"serverless_scaling" => {
"type" => "object",
- "descriptions" => "Scaling configuration for a +serverless+ Aurora cluster",
+ "description" => "Scaling configuration for a +serverless+ Aurora cluster",
"default" => {
"auto_pause" => false,
"min_capacity" => 2,
"max_capacity" => 2
},
@@ -1503,26 +1513,91 @@
# @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
# @return [Boolean]: True if validation succeeded, False otherwise
def self.validateConfig(db, configurator)
ok = true
+ if db['creation_style'] == "existing_snapshot" and
+ !db['create_cluster'] and
+ db['identifier'] and db['identifier'].match(/:cluster-snapshot:/)
+ MU.log "Database #{db['name']}: Existing snapshot #{db['identifier']} looks like a cluster snapshot, but create_cluster is not set. Add 'create_cluster: true' if you're building an RDS cluster.", MU::ERR
+ ok = false
+ end
+
+ pgroup_families = []
+ engines = {}
+
+ marker = nil
+ begin
+ resp = MU::Cloud::AWS.rds(credentials: db['credentials'], region: db['region']).describe_db_engine_versions(marker: marker)
+ marker = resp.marker
+
+ if resp and resp.db_engine_versions
+ resp.db_engine_versions.each { |version|
+ engines[version.engine] ||= {
+ "versions" => [],
+ "families" => []
+ }
+ engines[version.engine]['versions'] << version.engine_version
+ engines[version.engine]['families'] << version.db_parameter_group_family
+
+ }
+ engines.keys.each { |engine|
+ engines[engine]["versions"].uniq!
+ engines[engine]["families"].uniq!
+ }
+
+ else
+ MU.log "Failed to get list of valid RDS engine versions in #{db['region']}, proceeding without proper validation", MU::WARN
+ end
+ end while !marker.nil?
+
if db['create_cluster'] or db['engine'] == "aurora" or db["member_of_cluster"]
case db['engine']
when "mysql", "aurora", "aurora-mysql"
- if db["engine_version"] == "5.6" or db["cluster_mode"] == "serverless"
+ if db["engine_version"].match(/^5\.6/) or db["cluster_mode"] == "serverless"
db["engine"] = "aurora"
else
db["engine"] = "aurora-mysql"
end
when "postgres", "postgresql", "postgresql-mysql"
db["engine"] = "aurora-postgresql"
else
ok = false
- MU.log "Requested a clustered database, but engine #{db['engine']} is not supported for clustering", MU::ERR
+ MU.log "Database #{db['name']}: Requested a clustered database, but engine #{db['engine']} is not supported for clustering", MU::ERR
end
end
+ if db['engine'].match(/^aurora/) and !db['create_cluster'] and !db['add_cluster_node']
+ MU.log "Database #{db['name']}: #{db['engine']} looks like a cluster engine, but create_cluster is not set. Add 'create_cluster: true' if you're building an RDS cluster.", MU::ERR
+ ok = false
+ end
+
+ if engines.size > 0
+ if !engines[db['engine']]
+ MU.log "RDS engine #{db['engine']} is not supported in #{db['region']}", MU::ERR, details: engines.keys.sort
+ ok = false
+ else
+ if db["engine_version"] and
+ engines[db['engine']]['versions'].size > 0 and
+ !engines[db['engine']]['versions'].include?(db['engine_version']) and
+ !engines[db['engine']]['versions'].grep(/^#{Regexp.quote(db["engine_version"])}.+/)
+ MU.log "RDS engine '#{db['engine']}' version '#{db['engine_version']}' is not supported in #{db['region']}", MU::ERR, details: { "Known-good versions:" => engines[db['engine']]['versions'].uniq.sort }
+ ok = false
+ end
+ if db["parameter_group_family"] and
+ engines[db['engine']]['families'].size > 0 and
+ !engines[db['engine']]['families'].include?(db['parameter_group_family'])
+ MU.log "RDS engine '#{db['engine']}' parameter group family '#{db['parameter_group_family']}' is not supported in #{db['region']}", MU::ERR, details: { "Valid parameter families:" => engines[db['engine']]['families'].uniq.sort }
+ ok = false
+ end
+ end
+ end
+
+ if db['parameter_group_family'] and pgroup_families.size > 0 and
+ !pgroup_families.include?(db['parameter_group_family'])
+ end
+
db["license_model"] ||=
if ["postgres", "postgresql", "aurora-postgresql"].include?(db["engine"])
"postgresql-license"
elsif db["engine"] == "mysql"
"general-public-license"
@@ -1603,10 +1678,10 @@
end
end
end
if db["vpc"]
- if db["vpc"]["subnet_pref"] == "all_public" and !db['publicly_accessible']
+ if db["vpc"]["subnet_pref"] == "all_public" and !db['publicly_accessible'] and (db["vpc"]['subnets'].nil? or db["vpc"]['subnets'].empty?)
MU.log "Setting publicly_accessible to true on database '#{db['name']}', since deploying into public subnets.", MU::WARN
db['publicly_accessible'] = true
elsif db["vpc"]["subnet_pref"] == "all_private" and db['publicly_accessible']
MU.log "Setting publicly_accessible to false on database '#{db['name']}', since deploying into private subnets.", MU::NOTICE
db['publicly_accessible'] = false