# Copyright:: Copyright (c) 2019 eGlobalTech, Inc., all rights reserved
#
# Licensed under the BSD-3 license (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License in the root of the project or at
#
# http://egt-labs.com/mu/LICENSE.html
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
module MU
class Cloud
class Google
# Support for Google Cloud Storage
class Bucket < MU::Cloud::Bucket
# Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like @vpc, for us.
# @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
def initialize(**args)
super
@mu_name ||= @deploy.getResourceName(@config["name"])
end
# Called automatically by {MU::Deploy#createResources}
def create
@project_id = MU::Cloud::Google.projectLookup(@config['project'], @deploy).cloud_id
MU::Cloud::Google.storage(credentials: credentials).insert_bucket(@project_id, bucket_descriptor)
@cloud_id = @mu_name.downcase
end
# Called automatically by {MU::Deploy#createResources}
def groom
@project_id = MU::Cloud::Google.projectLookup(@config['project'], @deploy).cloudobj.cloud_id
current = cloud_desc
changed = false
if !current.versioning.enabled and @config['versioning']
MU.log "Enabling versioning on Cloud Storage bucket #{@cloud_id}", MU::NOTICE
changed = true
elsif current.versioning.enabled and !@config['versioning']
MU.log "Disabling versioning on Cloud Storage bucket #{@cloud_id}", MU::NOTICE
changed = true
end
if current.website.nil? and @config['web']
MU.log "Enabling website service on Cloud Storage bucket #{@cloud_id}", MU::NOTICE
changed = true
elsif !current.website.nil? and !@config['web']
MU.log "Disabling website service on Cloud Storage bucket #{@cloud_id}", MU::NOTICE
changed = true
end
if @config['bucket_wide_acls'] and (!current.iam_configuration or
!current.iam_configuration.bucket_policy_only or
!current.iam_configuration.bucket_policy_only.enabled)
MU.log "Converting Cloud Storage bucket #{@cloud_id} to use bucket-wide ACLs only", MU::NOTICE
changed = true
elsif !@config['bucket_wide_acls'] and current.iam_configuration and
current.iam_configuration.bucket_policy_only and
current.iam_configuration.bucket_policy_only.enabled
MU.log "Converting Cloud Storage bucket #{@cloud_id} to use bucket and object ACLs", MU::NOTICE
changed = true
end
if changed
MU::Cloud::Google.storage(credentials: credentials).patch_bucket(@cloud_id, bucket_descriptor)
end
if @config['policies']
@config['policies'].each { |pol|
pol['grant_to'].each { |grantee|
grantee['id'] ||= grantee["identifier"]
entity = if grantee["type"]
sibling = deploy_obj.findLitterMate(
name: grantee["id"],
type: grantee["type"]
)
if sibling
sibling.cloudobj.cloud_id
else
raise MuError, "Couldn't find a #{grantee["type"]} named #{grantee["id"]} when generating Cloud Storage access policy"
end
else
pol['grant_to'].first['id']
end
if entity.match(/@/) and !entity.match(/^(group|user)\-/)
entity = "user-"+entity if entity.match(/@/)
end
bucket_acl_obj = MU::Cloud::Google.storage(:BucketAccessControl).new(
bucket: @cloud_id,
role: pol['permissions'].first,
entity: entity
)
MU.log "Adding Cloud Storage policy to bucket #{@cloud_id}", MU::NOTICE, details: bucket_acl_obj
MU::Cloud::Google.storage(credentials: credentials).insert_bucket_access_control(
@cloud_id,
bucket_acl_obj
)
acl_obj = MU::Cloud::Google.storage(:ObjectAccessControl).new(
bucket: @cloud_id,
role: pol['permissions'].first,
entity: entity
)
MU::Cloud::Google.storage(credentials: credentials).insert_default_object_access_control(
@cloud_id,
acl_obj
)
}
}
end
end
# Upload a file to a bucket.
# @param url [String]: Target URL, of the form gs://bucket/folder/file
# @param acl [String]: Canned ACL permission to assign to the object we upload
# @param file [String]: Path to a local file to write to our target location. One of +file+ or +data+ must be specified.
# @param data [String]: Data to write to our target location. One of +file+ or +data+ must be specified.
def self.upload(url, acl: "private", file: nil, data: nil, credentials: nil)
end
# Does this resource type exist as a global (cloud-wide) artifact, or
# is it localized to a region/zone?
# @return [Boolean]
def self.isGlobal?
true
end
# Denote whether this resource implementation is experiment, ready for
# testing, or ready for production use.
def self.quality
MU::Cloud::BETA
end
# Remove all buckets associated with the currently loaded deployment.
# @param noop [Boolean]: If true, will only print what would be done
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
# @param region [String]: The cloud provider region
# @return [void]
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
resp = MU::Cloud::Google.storage(credentials: credentials).list_buckets(flags['project'])
if resp and resp.items
resp.items.each { |bucket|
if bucket.labels and bucket.labels["mu-id"] == MU.deploy_id.downcase
MU.log "Deleting Cloud Storage bucket #{bucket.name}"
if !noop
MU::Cloud::Google.storage(credentials: credentials).delete_bucket(bucket.name)
end
end
}
end
end
# Return the metadata for this user cofiguration
# @return [Hash]
def notify
desc = MU.structToHash(cloud_desc)
desc["project_id"] = @project_id
desc
end
# Locate an existing bucket.
# @return [OpenStruct]: The cloud provider's complete descriptions of matching bucket.
def self.find(**args)
args[:project] ||= args[:habitat]
args[:project] ||= MU::Cloud::Google.defaultProject(args[:credentials])
found = {}
if args[:cloud_id]
found[args[:cloud_id]] = MU::Cloud::Google.storage(credentials: args[:credentials]).get_bucket(args[:cloud_id])
else
resp = begin
MU::Cloud::Google.storage(credentials: args[:credentials]).list_buckets(args[:project])
rescue ::Google::Apis::ClientError => e
raise e if !e.message.match(/forbidden:/)
end
if resp and resp.items
resp.items.each { |bucket|
found[bucket.id] = bucket
}
end
end
found
end
# Reverse-map our cloud description into a runnable config hash.
# We assume that any values we have in +@config+ are placeholders, and
# calculate our own accordingly based on what's live in the cloud.
def toKitten(rootparent: nil, billing: nil, habitats: nil)
bok = {
"cloud" => "Google",
"credentials" => @config['credentials'],
"cloud_id" => @cloud_id
}
bok['name'] = cloud_desc.name
bok['project'] = @project_id
bok['storage_class'] = cloud_desc.storage_class
if cloud_desc.versioning and cloud_desc.versioning.enabled
bok['versioning'] = true
end
if cloud_desc.website
bok['web'] = true
if cloud_desc.website.not_found_page
bok['web_error_object'] = cloud_desc.website.not_found_page
end
if cloud_desc.website.main_page_suffix
bok['web_index_object'] = cloud_desc.website.main_page_suffix
end
pp cloud_desc
end
# MU.log "get_bucket_iam_policy", MU::NOTICE, details: MU::Cloud::Google.storage(credentials: @credentials).get_bucket_iam_policy(@cloud_id)
pols = MU::Cloud::Google.storage(credentials: @credentials).get_bucket_iam_policy(@cloud_id)
if pols and pols.bindings and pols.bindings.size > 0
bok['policies'] = []
count = 0
grantees = {}
pols.bindings.each { |binding|
grantees[binding.role] ||= []
binding.members.each { |grantee|
if grantee.match(/^(user|group):(.*)/)
grantees[binding.role] << MU::Config::Ref.get(
id: Regexp.last_match[2],
type: Regexp.last_match[1]+"s",
cloud: "Google",
credentials: @credentials
)
elsif grantee == "allUsers" or
grantee == "allAuthenticatedUsers" or
grantee.match(/^project(?:Owner|Editor|Viewer):/)
grantees[binding.role] << { "id" => grantee }
elsif grantee.match(/^serviceAccount:(.*)/)
sa_name = Regexp.last_match[1]
if MU::Cloud::Google::User.cannedServiceAcctName?(sa_name)
grantees[binding.role] << { "id" => grantee }
else
grantees[binding.role] << MU::Config::Ref.get(
id: sa_name,
type: "users",
cloud: "Google",
credentials: @credentials
)
end
else
# *shrug*
grantees[binding.role] << { "id" => grantee }
end
}
}
# munge together roles that apply to the exact same set of
# principals
reverse_map = {}
grantees.each_pair { |perm, grant_to|
reverse_map[grant_to] ||= []
reverse_map[grant_to] << perm
}
already_done = []
grantees.each_pair { |perm, grant_to|
if already_done.include?(perm+grant_to.to_s)
next
end
bok['policies'] << {
"name" => "policy"+count.to_s,
"grant_to" => grant_to,
"permissions" => reverse_map[grant_to]
}
reverse_map[grant_to].each { |doneperm|
already_done << doneperm+grant_to.to_s
}
count = count+1
}
end
if cloud_desc.iam_configuration and
cloud_desc.iam_configuration.bucket_policy_only and
cloud_desc.iam_configuration.bucket_policy_only.enabled
bok['bucket_wide_acls'] = true
else
# MU.log "list_bucket_access_controls", MU::NOTICE, details: MU::Cloud::Google.storage(credentials: @credentials).list_bucket_access_controls(@cloud_id)
# MU.log "list_default_object_access_controls", MU::NOTICE, details: MU::Cloud::Google.storage(credentials: @credentials).list_default_object_access_controls(@cloud_id)
end
bok
end
# Cloud-specific configuration properties.
# @param config [MU::Config]: The calling MU::Config object
# @return [Array]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
def self.schema(config)
toplevel_required = []
schema = {
"storage_class" => {
"type" => "string",
"enum" => ["MULTI_REGIONAL", "REGIONAL", "STANDARD", "NEARLINE", "COLDLINE", "DURABLE_REDUCED_AVAILABILITY"],
"default" => "STANDARD"
},
"bucket_wide_acls" => {
"type" => "boolean",
"default" => false,
"description" => "Disables object-level access controls in favor of bucket-wide policies"
}
}
[toplevel_required, schema]
end
# Cloud-specific pre-processing of {MU::Config::BasketofKittens::bucket}, bare and unvalidated.
# @param bucket [Hash]: The resource to process and validate
# @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
# @return [Boolean]: True if validation succeeded, False otherwise
def self.validateConfig(bucket, configurator)
ok = true
bucket['project'] ||= MU::Cloud::Google.defaultProject(bucket['credentials'])
if bucket['policies']
bucket['policies'].each { |pol|
if !pol['permissions'] or pol['permissions'].empty?
pol['permissions'] = ["READER"]
end
}
# XXX validate READER OWNER EDITOR w/e
end
ok
end
private
# create and return the Google::Apis::StorageV1::Bucket object used by
# both +insert_bucket+ and +patch_bucket+
def bucket_descriptor
labels = {}
MU::MommaCat.listStandardTags.each_pair { |name, value|
if !value.nil?
labels[name.downcase] = value.downcase.gsub(/[^a-z0-9\-\_]/i, "_")
end
}
labels["name"] = @mu_name.downcase
params = {
:name => @mu_name.downcase,
:labels => labels,
:storage_class => @config['storage_class'],
}
if @config['web']
params[:website] = MU::Cloud::Google.storage(:Bucket)::Website.new(
main_page_suffix: @config['web_index_object'],
not_found_page: @config['web_error_object']
)
end
if @config['versioning']
params[:versioning] = MU::Cloud::Google.storage(:Bucket)::Versioning.new(enabled: true)
else
params[:versioning] = MU::Cloud::Google.storage(:Bucket)::Versioning.new(enabled: false)
end
if @config['bucket_wide_acls']
params[:iam_configuration] = MU::Cloud::Google.storage(:Bucket)::IamConfiguration.new(
bucket_policy_only: MU::Cloud::Google.storage(:Bucket)::IamConfiguration::BucketPolicyOnly.new(
enabled: @config['bucket_wide_acls']
)
)
else
params[:iam_configuration] = MU::Cloud::Google.storage(:Bucket)::IamConfiguration.new(
bucket_policy_only: MU::Cloud::Google.storage(:Bucket)::IamConfiguration::BucketPolicyOnly.new(
enabled: false
)
)
end
MU::Cloud::Google.storage(:Bucket).new(params)
end
end
end
end
end