modules/mu/providers/aws/database.rb in cloud-mu-3.4.0 vs modules/mu/providers/aws/database.rb in cloud-mu-3.5.0

- old
+ new

@@ -222,11 +222,11 @@ master_user_password: @config["password"], preferred_backup_window: @config["preferred_backup_window"] } modify_db_cluster_struct[:preferred_maintenance_window] = @config["preferred_maintenance_window"] if @config["preferred_maintenance_window"] - MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_db_cluster(modify_db_cluster_struct) + MU::Cloud::AWS.rds(region: @region, credentials: @credentials).modify_db_cluster(modify_db_cluster_struct) wait_until_available end do_naming elsif @config["add_cluster_node"] @@ -303,11 +303,11 @@ # We assume that any values we have in +@config+ are placeholders, and # calculate our own accordingly based on what's live in the cloud. def toKitten(**_args) bok = { "cloud" => "AWS", - "region" => @config['region'], + "region" => @region, "credentials" => @credentials, "cloud_id" => @cloud_id, } # Don't adopt cluster members, they'll be picked up by the parent @@ -315,12 +315,12 @@ if !@config["create_cluster"] and cloud_desc.db_cluster_identifier and !cloud_desc.db_cluster_identifier.empty? return nil end noun = @config["create_cluster"] ? "cluster" : "db" - tags = MU::Cloud::AWS.rds(credentials: @credentials, region: @config['region']).list_tags_for_resource( - resource_name: MU::Cloud::AWS::Database.getARN(@cloud_id, noun, "rds", region: @config['region'], credentials: @credentials) + tags = MU::Cloud::AWS.rds(credentials: @credentials, region: @region).list_tags_for_resource( + resource_name: MU::Cloud::AWS::Database.getARN(@cloud_id, noun, "rds", region: @region, credentials: @credentials) ).tag_list if tags and !tags.empty? bok['tags'] = MU.structToHash(tags, stringify_keys: true) bok['name'] = MU::Adoption.tagsToName(bok['tags']) end @@ -330,15 +330,15 @@ bok['master_user'] = cloud_desc.master_username bok['backup_retention_period'] = cloud_desc.backup_retention_period bok["create_cluster"] = true if @config['create_cluster'] params = if bok['create_cluster'] - MU::Cloud::AWS.rds(credentials: @credentials, region: @config['region']).describe_db_cluster_parameters( + MU::Cloud::AWS.rds(credentials: @credentials, region: @region).describe_db_cluster_parameters( db_cluster_parameter_group_name: cloud_desc.db_cluster_parameter_group ).parameters else - MU::Cloud::AWS.rds(credentials: @credentials, region: @config['region']).describe_db_parameters( + MU::Cloud::AWS.rds(credentials: @credentials, region: @region).describe_db_parameters( db_parameter_group_name: cloud_desc.db_parameter_groups.first.db_parameter_group_name ).parameters end params.reject! { |p| ["engine-default", "system"].include?(p.source) } @@ -351,11 +351,11 @@ bok['add_firewall_rules'] = cloud_desc.vpc_security_groups.map { |sg| MU::Config::Ref.get( id: sg.vpc_security_group_id, cloud: "AWS", credentials: @credentials, - region: @config['region'], + region: @region, type: "firewall_rules", ) } bok['preferred_backup_window'] = cloud_desc.preferred_backup_window bok['preferred_maintenance_window'] = cloud_desc.preferred_maintenance_window @@ -371,11 +371,11 @@ sizes = [] vpcs = [] # we have no sensible way to handle heterogenous cluster members, so # for now just assume they're all the same cloud_desc.db_cluster_members.each { |db| - member = MU::Cloud::AWS::Database.find(cloud_id: db.db_instance_identifier, region: @config['region'], credentials: @credentials).values.first + member = MU::Cloud::AWS::Database.find(cloud_id: db.db_instance_identifier, region: @region, credentials: @credentials).values.first sizes << member.db_instance_class if member.db_subnet_group and member.db_subnet_group.vpc_id vpcs << member.db_subnet_group end @@ -383,18 +383,18 @@ } sizes.uniq! vpcs.uniq! bok['size'] = sizes.sort.first if !sizes.empty? if !vpcs.empty? - myvpc = MU::MommaCat.findStray("AWS", "vpc", cloud_id: vpcs.sort.first.vpc_id, credentials: @credentials, region: @config['region'], dummy_ok: true, no_deploy_search: true).first + myvpc = MU::MommaCat.findStray("AWS", "vpc", cloud_id: vpcs.sort.first.vpc_id, credentials: @credentials, region: @region, dummy_ok: true, no_deploy_search: true).first bok['vpc'] = myvpc.getReference(vpcs.sort.first.subnets.map { |s| s.subnet_identifier }) end else bok['size'] = cloud_desc.db_instance_class bok['auto_minor_version_upgrade'] = true if cloud_desc.auto_minor_version_upgrade if cloud_desc.db_subnet_group - myvpc = MU::MommaCat.findStray("AWS", "vpc", cloud_id: cloud_desc.db_subnet_group.vpc_id, credentials: @credentials, region: @config['region'], dummy_ok: true, no_deploy_search: true).first + myvpc = MU::MommaCat.findStray("AWS", "vpc", cloud_id: cloud_desc.db_subnet_group.vpc_id, credentials: @credentials, region: @region, dummy_ok: true, no_deploy_search: true).first bok['vpc'] = myvpc.getReference(cloud_desc.db_subnet_group.subnets.map { |s| s.subnet_identifier }) end bok['storage_type'] = cloud_desc.storage_type bok['storage'] = cloud_desc.allocated_storage bok['license_model'] = cloud_desc.license_model @@ -465,17 +465,17 @@ if subnet_ids.empty? raise MuError, "Couldn't find subnets in #{@vpc} to add to #{@config["subnet_group_name"]}. Make sure the subnets are valid and publicly_accessible is set correctly" else resp = begin - MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).describe_db_subnet_groups( + MU::Cloud::AWS.rds(region: @region, credentials: @credentials).describe_db_subnet_groups( db_subnet_group_name: @config["subnet_group_name"] ) # XXX ensure subnet group matches our config? rescue ::Aws::RDS::Errors::DBSubnetGroupNotFoundFault # Create subnet group - resp = MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_subnet_group( + resp = MU::Cloud::AWS.rds(region: @region, credentials: @credentials).create_db_subnet_group( db_subnet_group_name: @config["subnet_group_name"], db_subnet_group_description: @config["subnet_group_name"], subnet_ids: subnet_ids, tags: @tags.each_key.map { |k| { :key => k, :value => @tags[k] } } ) @@ -509,17 +509,17 @@ params[name_param] = @config["parameter_group_name"] if create MU.log "Creating a #{cluster ? "cluster" : "database" } parameter group #{@config["parameter_group_name"]}" - MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).send(cluster ? :create_db_cluster_parameter_group : :create_db_parameter_group, params) + MU::Cloud::AWS.rds(region: @region, credentials: @credentials).send(cluster ? :create_db_cluster_parameter_group : :create_db_parameter_group, params) end if @config[fieldname] and !@config[fieldname].empty? - old_values = MU::Cloud::AWS.rds(credentials: @credentials, region: @config['region']).send(cluster ? :describe_db_cluster_parameters : :describe_db_parameters, { name_param => @config["parameter_group_name"] } ).parameters + old_values = MU::Cloud::AWS.rds(credentials: @credentials, region: @region).send(cluster ? :describe_db_cluster_parameters : :describe_db_parameters, { name_param => @config["parameter_group_name"] } ).parameters old_values.map! { |p| [p.parameter_name, p.parameter_value] }.flatten old_values = old_values.to_h params = [] @config[fieldname].each { |item| @@ -530,16 +530,16 @@ MU.log "Modifying parameter group #{@config["parameter_group_name"]}", MU::NOTICE, details: params.map { |p| { p[:parameter_name] => p[:parameter_value] } } MU.retrier([Aws::RDS::Errors::InvalidDBParameterGroupState], wait: 30, max: 10) { if cluster - MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_db_cluster_parameter_group( + MU::Cloud::AWS.rds(region: @region, credentials: @credentials).modify_db_cluster_parameter_group( db_cluster_parameter_group_name: @config["parameter_group_name"], parameters: params ) else - MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_db_parameter_group( + MU::Cloud::AWS.rds(region: @region, credentials: @credentials).modify_db_parameter_group( db_parameter_group_name: @config["parameter_group_name"], parameters: params ) end } @@ -584,11 +584,11 @@ end if @config["create_cluster"] @config['cluster_node_count'] ||= 1 if @config['cluster_mode'] == "serverless" - MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_current_db_cluster_capacity( + MU::Cloud::AWS.rds(region: @region, credentials: @credentials).modify_current_db_cluster_capacity( db_cluster_identifier: @cloud_id, capacity: @config['cluster_node_count'] ) end else @@ -610,12 +610,13 @@ end if mods.size > 1 MU.log "Modifying RDS instance #{@cloud_id}", MU::NOTICE, details: mods mods[:apply_immediately] = true + mods[:allow_major_version_upgrade] = true wait_until_available - MU::Cloud::AWS.rds(region: @config['region'], credentials: @credentials).send("modify_db_#{noun}".to_sym, mods) + MU::Cloud::AWS.rds(region: @region, credentials: @credentials).send("modify_db_#{noun}".to_sym, mods) wait_until_available end end @@ -658,11 +659,11 @@ # If we're an old, Classic-style database with RDS-specific # authorization, punch holes in that. if !cloud_desc.db_security_groups.empty? cloud_desc.db_security_groups.each { |rds_sg| begin - MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).authorize_db_security_group_ingress( + MU::Cloud::AWS.rds(region: @region, credentials: @credentials).authorize_db_security_group_ingress( db_security_group_name: rds_sg.db_security_group_name, cidrip: cidr ) rescue Aws::RDS::Errors::AuthorizationAlreadyExists MU.log "CIDR #{cidr} already in database instance #{@cloud_id} security group", MU::WARN @@ -680,11 +681,11 @@ # Return the metadata for this ContainerCluster # @return [Hash] def notify deploy_struct = MU.structToHash(cloud_desc, stringify_keys: true) deploy_struct['cloud_id'] = @cloud_id - deploy_struct["region"] ||= @config['region'] + deploy_struct["region"] ||= @region deploy_struct["db_name"] ||= @config['db_name'] deploy_struct end # Generate a snapshot from the database described in this instance. @@ -706,18 +707,18 @@ params[:db_snapshot_identifier] = snap_id params[:db_instance_identifier] = src_ref.id end MU.retrier([Aws::RDS::Errors::InvalidDBInstanceState, Aws::RDS::Errors::InvalidDBClusterStateFault], wait: 60, max: 10) { - MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).send("create_db_#{@config['create_cluster'] ? "cluster_" : ""}snapshot".to_sym, params) + MU::Cloud::AWS.rds(region: @region, credentials: @credentials).send("create_db_#{@config['create_cluster'] ? "cluster_" : ""}snapshot".to_sym, params) } loop_if = Proc.new { if @config["create_cluster"] - MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).describe_db_cluster_snapshots(db_cluster_snapshot_identifier: snap_id).db_cluster_snapshots.first.status != "available" + MU::Cloud::AWS.rds(region: @region, credentials: @credentials).describe_db_cluster_snapshots(db_cluster_snapshot_identifier: snap_id).db_cluster_snapshots.first.status != "available" else - MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).describe_db_snapshots(db_snapshot_identifier: snap_id).db_snapshots.first.status != "available" + MU::Cloud::AWS.rds(region: @region, credentials: @credentials).describe_db_snapshots(db_snapshot_identifier: snap_id).db_snapshots.first.status != "available" end } MU.retrier(wait: 15, loop_if: loop_if) { |retries, _wait| MU.log "Waiting for RDS snapshot of #{src_ref.id} to be ready...", MU::NOTICE if retries % 20 == 0 @@ -730,13 +731,13 @@ # @return [String]: The cloud provider's identifier for the snapshot. def getExistingSnapshot src_ref = MU::Config::Ref.get(@config["source"]) resp = if @config["create_cluster"] - MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).describe_db_cluster_snapshots(db_cluster_snapshot_identifier: src_ref.id) + MU::Cloud::AWS.rds(region: @region, credentials: @credentials).describe_db_cluster_snapshots(db_cluster_snapshot_identifier: src_ref.id) else - MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).describe_db_snapshots(db_snapshot_identifier: src_ref.id) + MU::Cloud::AWS.rds(region: @region, credentials: @credentials).describe_db_snapshots(db_snapshot_identifier: src_ref.id) end snapshots = @config["create_cluster"] ? resp.db_cluster_snapshots : resp.db_snapshots if snapshots.empty? @@ -810,19 +811,19 @@ } threads = threaded_resource_purge(:describe_db_subnet_groups, :db_subnet_groups, :db_subnet_group_name, "subgrp", region, credentials, ignoremaster, known: flags['known'], deploy_id: deploy_id) { |id| MU.log "Deleting RDS subnet group #{id}" MU.retrier([Aws::RDS::Errors::InvalidDBSubnetGroupStateFault], wait: 30, max: 5, ignoreme: [Aws::RDS::Errors::DBSubnetGroupNotFoundFault]) { - MU::Cloud::AWS.rds(region: region).delete_db_subnet_group(db_subnet_group_name: id) if !noop + MU::Cloud::AWS.rds(region: region, credentials: credentials).delete_db_subnet_group(db_subnet_group_name: id) if !noop } } ["db", "db_cluster"].each { |type| threads.concat threaded_resource_purge("describe_#{type}_parameter_groups".to_sym, "#{type}_parameter_groups".to_sym, "#{type}_parameter_group_name".to_sym, (type == "db" ? "pg" : "cluster-pg"), region, credentials, ignoremaster, known: flags['known'], deploy_id: deploy_id) { |id| MU.log "Deleting RDS #{type} parameter group #{id}" MU.retrier([Aws::RDS::Errors::InvalidDBParameterGroupState], wait: 30, max: 5, ignoreme: [Aws::RDS::Errors::DBParameterGroupNotFound]) { - MU::Cloud::AWS.rds(region: region).send("delete_#{type}_parameter_group", { "#{type}_parameter_group_name".to_sym => id }) if !noop + MU::Cloud::AWS.rds(region: region, credentials: credentials).send("delete_#{type}_parameter_group", { "#{type}_parameter_group_name".to_sym => id }) if !noop } } } # Wait for all of the databases subnet/parameter groups to finish cleanup before proceeding @@ -1260,11 +1261,11 @@ private_class_method :validate_engine def add_basic getPassword - if @config['source'].nil? or @config['region'] != @config['source'].region + if @config['source'].nil? or @region != @config['source'].region manageSubnetGroup if @vpc else MU.log "Note: Read Replicas automatically reside in the same subnet group as the source database, if they're both in the same region. This replica may not land in the VPC you intended.", MU::WARN end @@ -1345,15 +1346,15 @@ MU.retrier([Aws::RDS::Errors::InvalidParameterValue, Aws::RDS::Errors::DBSubnetGroupNotFoundFault], max: 10, wait: 15) { if %w{existing_snapshot new_snapshot}.include?(@config["creation_style"]) clean_parent_opts.call MU.log "Creating database #{noun} #{@cloud_id} from snapshot #{@config["snapshot_id"]}" - MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).send("restore_db_#{noun}_from_#{noun == "instance" ? "db_" : ""}snapshot".to_sym, params) + MU::Cloud::AWS.rds(region: @region, credentials: @credentials).send("restore_db_#{noun}_from_#{noun == "instance" ? "db_" : ""}snapshot".to_sym, params) else clean_parent_opts.call if noun == "instance" and params[:db_cluster_identifier] - MU.log "Creating pristine database #{noun} #{@cloud_id} (#{@config['name']}) in #{@config['region']}", MU::NOTICE, details: params - MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).send("create_db_#{noun}".to_sym, params) + MU.log "Creating pristine database #{noun} #{@cloud_id} (#{@config['name']}) in #{@region}", MU::NOTICE, details: params + MU::Cloud::AWS.rds(region: @region, credentials: @credentials).send("create_db_#{noun}".to_sym, params) end } end # creation_style = point_in_time @@ -1376,11 +1377,11 @@ params[:use_latest_restorable_time] = true if @config['restore_time'] == "latest" MU.retrier([Aws::RDS::Errors::InvalidParameterValue], max: 15, wait: 20) { MU.log "Creating database #{@config['create_cluster'] ? "cluster" : "instance" } #{@cloud_id} based on point in time backup '#{@config['restore_time']}' of #{@config['source'].id}" - MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).send("restore_db_#{@config['create_cluster'] ? "cluster" : "instance"}_to_point_in_time".to_sym, params) + MU::Cloud::AWS.rds(region: @region, credentials: @credentials).send("restore_db_#{@config['create_cluster'] ? "cluster" : "instance"}_to_point_in_time".to_sym, params) } end # creation_style = new, existing and read_replica_of is not nil def create_read_replica @@ -1397,12 +1398,12 @@ publicly_accessible: @config["publicly_accessible"], tags: @tags.each_key.map { |k| { :key => k, :value => @tags[k] } }, db_subnet_group_name: @config["subnet_group_name"], storage_type: @config["storage_type"] } - if @config["source"].region and @config['region'] != @config["source"].region - params[:source_db_instance_identifier] = MU::Cloud::AWS::Database.getARN(@config["source"].id, "db", "rds", region: @config["source"].region, credentials: @config['credentials']) + if @config["source"].region and @region != @config["source"].region + params[:source_db_instance_identifier] = MU::Cloud::AWS::Database.getARN(@config["source"].id, "db", "rds", region: @config["source"].region, credentials: @credentials) end params[:port] = @config["port"] if @config["port"] params[:iops] = @config["iops"] if @config['storage_type'] == "io1" @@ -1413,11 +1414,11 @@ end } MU.retrier([Aws::RDS::Errors::InvalidDBInstanceState, Aws::RDS::Errors::InvalidParameterValue, Aws::RDS::Errors::DBSubnetGroupNotAllowedFault], max: 10, wait: 30, on_retry: on_retry) { MU.log "Creating read replica database instance #{@cloud_id} for #{@config['source'].id}" - MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_instance_read_replica(params) + MU::Cloud::AWS.rds(region: @region, credentials: @credentials).create_db_instance_read_replica(params) } end # Sit on our hands until we show as available def wait_until_available @@ -1472,11 +1473,11 @@ if localdeploy_rule.nil? raise MU::MuError, "Database #{@config['name']} failed to find its generic security group 'database#{@config['name']}'" end mod_config[:vpc_security_group_ids] << localdeploy_rule.cloud_id - MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_db_instance(mod_config) + MU::Cloud::AWS.rds(region: @region, credentials: @credentials).modify_db_instance(mod_config) MU.log "Modified database #{@cloud_id} with new security groups: #{mod_config}", MU::NOTICE end # When creating from a snapshot or replicating an existing database, # some of the create arguments that we'd want to carry over aren't @@ -1484,11 +1485,11 @@ if %w{existing_snapshot new_snapshot point_in_time}.include?(@config["creation_style"]) or @config["read_replica_of"] mod_config = { db_instance_identifier: @cloud_id, apply_immediately: true } - if !@config["read_replica_of"] or @config['region'] == @config['source'].region + if !@config["read_replica_of"] or @region == @config['source'].region mod_config[:vpc_security_group_ids] = @config["vpc_security_group_ids"] end if !@config["read_replica_of"] mod_config[:preferred_backup_window] = @config["preferred_backup_window"] @@ -1501,18 +1502,18 @@ end if @config["preferred_maintenance_window"] mod_config[:preferred_maintenance_window] = @config["preferred_maintenance_window"] end - MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_db_instance(mod_config) + MU::Cloud::AWS.rds(region: @region, credentials: @credentials).modify_db_instance(mod_config) wait_until_available end # Maybe wait for DB instance to be in available state. DB should still be writeable at this state if @config['allow_major_version_upgrade'] && @config["creation_style"] == "new" MU.log "Setting major database version upgrade on #{@cloud_id}'" - MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_db_instance( + MU::Cloud::AWS.rds(region: @region, credentials: @credentials).modify_db_instance( db_instance_identifier: @cloud_id, apply_immediately: true, allow_major_version_upgrade: true ) end