modules/mu/clouds/google/vpc.rb in cloud-mu-3.1.3 vs modules/mu/clouds/google/vpc.rb in cloud-mu-3.1.4

- old
+ new

@@ -34,16 +34,14 @@ @mu_name ||= @config['scrub_mu_isms'] ? @config['name'] : @deploy.getResourceName(@config['name']) end # Called automatically by {MU::Deploy#createResources} def create - networkobj = MU::Cloud::Google.compute(:Network).new( name: MU::Cloud::Google.nameStr(@mu_name), description: @deploy.deploy_id, auto_create_subnetworks: false -# i_pv4_range: @config['ip_block'] ) MU.log "Creating network #{@mu_name} (#{@config['ip_block']}) in project #{@project_id}", details: networkobj resp = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_network(@project_id, networkobj) @url = resp.self_link @@ -56,11 +54,11 @@ subnetthreads << Thread.new { MU.dupGlobals(parent_thread_id) subnet_name = @config['name']+subnet['name'] subnet_mu_name = @config['scrub_mu_isms'] ? @cloud_id+subnet_name.downcase : MU::Cloud::Google.nameStr(@deploy.getResourceName(subnet_name, max_length: 61)) - MU.log "Creating subnetwork #{subnet_mu_name} (#{subnet['ip_block']}) in project #{@project_id}", details: subnet + MU.log "Creating subnetwork #{subnet_mu_name} (#{subnet['ip_block']}) in project #{@project_id} region #{subnet['availability_zone']}", details: subnet subnetobj = MU::Cloud::Google.compute(:Subnetwork).new( name: subnet_mu_name, description: @deploy.deploy_id, ip_cidr_range: subnet['ip_block'], network: @url, @@ -70,21 +68,28 @@ # make sure the subnet we created exists, before moving on subnetdesc = nil begin subnetdesc = MU::Cloud::Google.compute(credentials: @config['credentials']).get_subnetwork(@project_id, subnet['availability_zone'], subnet_mu_name) + if !subnetdesc.nil? + subnet_cfg = {} + subnet_cfg["ip_block"] = subnet['ip_block'] + subnet_cfg["name"] = subnet_name + subnet_cfg['mu_name'] = subnet_mu_name + subnet_cfg["cloud_id"] = subnetdesc.self_link.gsub(/.*?\/([^\/]+)$/, '\1') + subnet_cfg['az'] = subnet['availability_zone'] + @subnets << MU::Cloud::Google::VPC::Subnet.new(self, subnet_cfg, precache_description: false) + end sleep 1 end while subnetdesc.nil? - } } subnetthreads.each do |t| t.join end end - route_table_ids = [] if !@config['route_tables'].nil? @config['route_tables'].each { |rtb| rtb['routes'].each { |route| # GCP does these for us, by default next if route['destination_network'] == "0.0.0.0/0" and @@ -118,12 +123,12 @@ base end # Describe this VPC from the cloud platform's perspective # @return [Google::Apis::Core::Hashable] - def cloud_desc - if @cloud_desc_cache + def cloud_desc(use_cache: true) + if @cloud_desc_cache and use_cache return @cloud_desc_cache end resp = MU::Cloud::Google.compute(credentials: @config['credentials']).get_network(@project_id, @cloud_id) @@ -228,21 +233,21 @@ # support other search methods, such as +tag_key+ and +tag_value+, or # cloud-specific arguments like +project+. See also {MU::MommaCat.findStray}. # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching resources def self.find(**args) - args[:project] ||= args[:habitat] - args[:project] ||= MU::Cloud::Google.defaultProject(args[:credentials]) + args = MU::Cloud::Google.findLocationArgs(args) + resp = {} if args[:cloud_id] and args[:project] begin vpc = MU::Cloud::Google.compute(credentials: args[:credentials]).get_network( args[:project], args[:cloud_id].to_s.sub(/^.*?\/([^\/]+)$/, '\1') ) resp[args[:cloud_id]] = vpc if !vpc.nil? - rescue ::Google::Apis::ClientError => e + rescue ::Google::Apis::ClientError MU.log "VPC #{args[:cloud_id]} in project #{args[:project]} does not exist, or I do not have permission to view it", MU::WARN end else # XXX other criteria vpcs = begin MU::Cloud::Google.compute(credentials: args[:credentials]).list_networks( @@ -475,13 +480,12 @@ # Check whether we (the Mu Master) have a direct route to a particular # instance. Useful for skipping hops through bastion hosts to get # directly at child nodes in peered VPCs, the public internet, and the # like. # @param target_instance [OpenStruct]: The cloud descriptor of the instance to check. - # @param region [String]: The cloud provider region of the target subnet. # @return [Boolean] - def self.haveRouteToInstance?(target_instance, region: MU.curRegion, credentials: nil) + def self.haveRouteToInstance?(target_instance, credentials: nil) project ||= MU::Cloud::Google.defaultProject(credentials) return false if MU.myCloud != "Google" # XXX see if we reside in the same Network and overlap subnets # XXX see if we peer with the target's Network target_instance.network_interfaces.each { |iface| @@ -534,15 +538,19 @@ end # Remove all VPC resources associated with the currently loaded deployment. # @param noop [Boolean]: If true, will only print what would be done # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server - # @param region [String]: The cloud provider region # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {}) flags["project"] ||= MU::Cloud::Google.defaultProject(credentials) return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials) + filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")} + if !ignoremaster and MU.mu_public_ip + filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")} + end + MU.log "Placeholder: Google VPC artifacts do not support labels, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: filter purge_subnets(noop, project: flags['project'], credentials: credentials) ["route", "network"].each { |type| # XXX tagged routes aren't showing up in list, and the networks that own them # fail to delete silently @@ -560,11 +568,11 @@ if type == "network" MU.log e.message, MU::WARN if e.message.match(/Failed to delete network (.+)/) network_name = Regexp.last_match[1] fwrules = MU::Cloud::Google::FirewallRule.find(project: flags['project'], credentials: credentials) - fwrules.reject! { |name, desc| + fwrules.reject! { |_name, desc| !desc.network.match(/.*?\/#{Regexp.quote(network_name)}$/) } fwrules.keys.each { |name| MU.log "Attempting to delete firewall rule #{name} so that VPC #{network_name} can be removed", MU::NOTICE MU::Cloud::Google.compute(credentials: credentials).delete_firewall(flags['project'], name) @@ -584,21 +592,20 @@ # Reverse-map our cloud description into a runnable config hash. # We assume that any values we have in +@config+ are placeholders, and # calculate our own accordingly based on what's live in the cloud. # XXX add flag to return the diff between @config and live cloud - def toKitten(rootparent: nil, billing: nil, habitats: nil) + def toKitten(**_args) return nil if cloud_desc.name == "default" # parent project builds these bok = { "cloud" => "Google", "project" => @config['project'], "credentials" => @config['credentials'] } MU::Cloud::Google.listRegions.size - diff = {} - schema, valid = MU::Config.loadResourceSchema("VPC", cloud: "Google") + _schema, valid = MU::Config.loadResourceSchema("VPC", cloud: "Google") return [nil, nil] if !valid # pp schema # MU.log "++++++++++++++++++++++++++++++++" bok['name'] = cloud_desc.name.dup @@ -607,10 +614,11 @@ if @subnets and @subnets.size > 0 bok['subnets'] = [] regions_seen = [] names_seen = [] + @subnets.reject! { |x| x.cloud_desc.nil? } @subnets.map { |x| x.cloud_desc }.each { |s| subnet_name = s.name.dup names_seen << s.name.dup regions_seen << s.region bok['subnets'] << { @@ -628,11 +636,10 @@ bok.delete("subnets") bok['auto_create_subnetworks'] = true end end - peer_names = [] if cloud_desc.peerings and cloud_desc.peerings.size > 0 bok['peers'] = [] cloud_desc.peerings.each { |peer| peer.network.match(/projects\/([^\/]+?)\/[^\/]+?\/networks\/([^\/]+)$/) vpc_project = Regexp.last_match[1] @@ -686,13 +693,13 @@ # XXX validate that we've at least touched every required attribute (maybe upstream?) bok end # Cloud-specific configuration properties. - # @param config [MU::Config]: The calling MU::Config object + # @param _config [MU::Config]: The calling MU::Config object # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource - def self.schema(config = nil) + def self.schema(_config = nil) toplevel_required = [] schema = { "regions" => { "type" => "array", "items" => MU::Config.region_primitive @@ -734,11 +741,11 @@ return vpc_block if vpcs.size == 0 # see if one of this thing's siblings declared a subnet_pref we can # use to guess which one we should marry ourselves to - configurator.kittens.each_pair { |type, siblings| + configurator.kittens.values.each { |siblings| siblings.each { |sibling| next if !sibling['dependencies'] sibling['dependencies'].each { |dep| if [cfg_name, cfg_plural].include?(dep['type']) and dep['name'] == my_config['name'] @@ -898,11 +905,11 @@ vpc['route_tables'].first["routes"] << { "gateway"=>"#DENY", "destination_network"=>"0.0.0.0/0" } end - nat_count = 0 + # You know what, let's just guarantee that we'll have a route from # this master, always # XXX this confuses machines that don't have public IPs if !vpc['scrub_mu_isms'] # vpc['route_tables'].first["routes"] << { @@ -943,11 +950,11 @@ createRoute(route, network: @url, tags: [MU::Cloud::Google.nameStr(server.mu_name)]) end private - def self.genStandardSubnetACLs(vpc_cidr, vpc_name, configurator, project, publicroute = true, credentials: nil) + def self.genStandardSubnetACLs(vpc_cidr, vpc_name, configurator, project, _publicroute = true, credentials: nil) private_acl = { "name" => vpc_name+"-rt", "cloud" => "Google", "credentials" => credentials, "project" => project, @@ -971,10 +978,11 @@ # "egress" => true, "proto" => "all", "hosts" => ["0.0.0.0/0"], "deny" => true # } # end configurator.insertKitten(private_acl, "firewall_rules", true) end + private_class_method :genStandardSubnetACLs # Helper method for manufacturing routes. Expect to be called from # {MU::Cloud::Google::VPC#create} or {MU::Cloud::Google::VPC#groom}. # @param route [Hash]: A route description, per the Basket of Kittens schema # @param network [String]: Cloud identifier of the VPC to which we're adding this route @@ -1037,56 +1045,24 @@ begin MU::Cloud::Google.compute(credentials: @config['credentials']).get_route(@project_id, routename) rescue ::Google::Apis::ClientError, MU::MuError => e if e.message.match(/notFound/) MU.log "Creating route #{routename} in project #{@project_id}", details: routeobj - resp = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_route(@project_id, routeobj) + MU::Cloud::Google.compute(credentials: @config['credentials']).insert_route(@project_id, routeobj) else # TODO can't update GCP routes, would have to delete and re-create end end end end - - # Remove all network gateways associated with the currently loaded deployment. - # @param noop [Boolean]: If true, will only print what would be done - # @param region [String]: The cloud provider region - # @return [void] - def self.purge_gateways(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion) - end - - # Remove all NAT gateways associated with the VPC of the currently loaded deployment. - # @param noop [Boolean]: If true, will only print what would be done - # @param vpc_id [String]: The cloud provider's unique VPC identifier - # @param region [String]: The cloud provider region - # @return [void] - def self.purge_nat_gateways(noop = false, vpc_id: nil, region: MU.curRegion) - end - - # Remove all VPC endpoints associated with the VPC of the currently loaded deployment. - # @param noop [Boolean]: If true, will only print what would be done - # @param vpc_id [String]: The cloud provider's unique VPC identifier - # @param region [String]: The cloud provider region - # @return [void] - def self.purge_endpoints(noop = false, vpc_id: nil, region: MU.curRegion) - end - - # Remove all network interfaces associated with the currently loaded deployment. - # @param noop [Boolean]: If true, will only print what would be done - # @param tagfilters [Array<Hash>]: Labels to filter against when search for resources to purge - # @param region [String]: The cloud provider region - # @return [void] - def self.purge_interfaces(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion) - end - # Remove all subnets associated with the currently loaded deployment. # @param noop [Boolean]: If true, will only print what would be done - # @param tagfilters [Array<Hash>]: Labels to filter against when search for resources to purge + # @param _tagfilters [Array<Hash>]: Labels to filter against when search for resources to purge # @param regions [Array<String>]: The cloud provider regions to check # @return [void] - def self.purge_subnets(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], regions: MU::Cloud::Google.listRegions, project: nil, credentials: nil) + def self.purge_subnets(noop = false, _tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], regions: MU::Cloud::Google.listRegions, project: nil, credentials: nil) project ||= MU::Cloud::Google.defaultProject(credentials) parent_thread_id = Thread.current.object_id regionthreads = [] regions.each { |r| regionthreads << Thread.new { @@ -1096,29 +1072,27 @@ "subnetwork", project, r, noop ) - rescue MU::Cloud::MuDefunctHabitat => e + rescue MU::Cloud::MuDefunctHabitat Thread.exit end } } regionthreads.each do |t| t.join end end + private_class_method :purge_subnets - protected - # Subnets are almost a first-class resource. So let's kinda sorta treat # them like one. This should only be invoked on objects that already # exists in the cloud layer. class Subnet < MU::Cloud::Google::VPC attr_reader :cloud_id - attr_reader :url attr_reader :ip_block attr_reader :mu_name attr_reader :name attr_reader :cloud_desc_cache attr_reader :az @@ -1147,13 +1121,32 @@ # @return [Hash] def notify MU.structToHash(cloud_desc) end + # Return the +self_link+ to this subnet + def url + cloud_desc if !@url + @url + end + + @cloud_desc_cache = nil # Describe this VPC Subnet from the cloud platform's perspective # @return [Google::Apis::Core::Hashable] - def cloud_desc - @cloud_desc_cache ||= MU::Cloud::Google.compute(credentials: @parent.config['credentials']).get_subnetwork(@parent.habitat_id, @config['az'], @config['cloud_id']) + def cloud_desc(use_cache: true) + return @cloud_desc_cache if @cloud_desc_cache and use_cache + + begin + @cloud_desc_cache = MU::Cloud::Google.compute(credentials: @parent.config['credentials']).get_subnetwork(@parent.habitat_id, @az, @cloud_id) + rescue ::Google::Apis::ClientError => e + if e.message.match(/notFound: /) + MU.log "Failed to fetch cloud description for Google subnet #{@cloud_id}", MU::WARN, details: { "project" => @parent.habitat_id, "region" => @az, "name" => @cloud_id } + return nil + else + raise e + end + end + @url ||= @cloud_desc_cache.self_link @cloud_desc_cache end # Is this subnet privately-routable only, or public? # @return [Boolean]