lib/cicd/builder/manifest/mixlib/build.rb in manifest-builder-0.6.2 vs lib/cicd/builder/manifest/mixlib/build.rb in manifest-builder-0.6.4
- old
+ new
@@ -7,10 +7,11 @@
module Build
# ---------------------------------------------------------------------------------------------------------------
# noinspection RubyHashKeysTypesInspection
def prepareBuild()
+ @logger.step CLASS+'::'+__method__.to_s
ret = super
if ret == 0
@vars[:artifacts] = []
yaml = YAML.load(IO.read(ENV['MANIFEST_FILE']))
keys = Hash[yaml.keys.map.with_index.to_a].keys.sort
@@ -155,37 +156,26 @@
[version,build]
end
# ---------------------------------------------------------------------------------------------------------------
def packageBuild()
- @logger.step __method__.to_s
+ @logger.info CLASS+'::'+__method__.to_s
if isSameDirectory(Dir.pwd, ENV['WORKSPACE'])
if @vars.has_key?(:components) and not @vars[:components].empty?
@vars[:return_code] = 0
- clazz = getRepoClass('S3')
- @logger.debug "Repo class == '#{clazz}'"
- if clazz.is_a?(Class) and not clazz.nil?
- @repo = clazz.new(self)
-
+ getRepoInstance('S3')
+ if 0 == @vars[:return_code]
+ lines = []
+ @vars[:artifacts] = []
+ # Deal with all artifacts of each component
+ @vars[:components].each { |comp|
+ processComponent(comp, lines)
+ }
if @vars[:return_code] == 0
- lines = []
- @vars[:artifacts] = []
- # Deal with all artifacts of each component
- @vars[:components].each { |comp|
- processComponent(comp, lines)
- }
- if @vars[:return_code] == 0
- cleanupAfterPackaging(lines)
- end
-
- else
- @logger.fatal "S3 repo error: Bucket #{ENV['AWS_S3_BUCKET']}"
+ cleanupAfterPackaging(lines)
end
- else
- @logger.error "CiCd::Builder::Repo::#{type} is not a valid repo class"
- @vars[:return_code] = Errors::BUILDER_REPO_TYPE
end
else
@logger.error 'No components found during preparation?'
@vars[:return_code] = Errors::NO_COMPONENTS
end
@@ -215,21 +205,21 @@
FileUtils.rm_f(@vars[:build_mff])
# @vars[:return_code] = File.exists?(@vars[:build_mff]) ? Errors::MANIFEST_DELETE : 0
end
end
- private
+ protected
# ---------------------------------------------------------------------------------------------------------------
def processComponent(comp, lines)
artifact, path, version, build = parseComponent(comp)
cname,cdata = comp
require 'uri'
require 'digest'
begin
- key, name, objects = getObjects(artifact, path)
+ key, bucket, objects = getObjects(artifact, path,cdata)
local_dir = File.join(@vars[:local_dirs]['artifacts'], cname, '')
Dir.mkdir(local_dir, 0700) unless File.directory?(local_dir)
artifacts = []
sha256 = {}
changed = false
@@ -242,11 +232,11 @@
base = File.basename(object.key)
local = File.join(local_dir, base)
etag = object.etag.gsub(%r/['"]/, '')
download = shouldDownload?(etag, local, object)
if download
- changed = doDownload(etag, local, object)
+ changed = doDownload(etag, local, object, bucket)
else
@logger.info "\t\tunchanged"
end
sha256[base] = nil
if cdata[:sha256]
@@ -259,11 +249,11 @@
# The local file will be 1 artifact or an archive of the local artifacts when artifacts.size > 1
if artifacts.size > 0
artifacts.each do |local|
base = File.basename(local)
if sha256[base].is_a?(String)
- msg = "Artifact checksum is invalid or manifest is incorrect. Artifact: s3://#{name}/#{key}#{artifact}, SHA256: Manifest=#{cdata[:sha256]}, Actual=#{sha256[base]}"
+ msg = "Artifact checksum is invalid or manifest is incorrect. Artifact: s3://#{bucket}/#{key}#{artifact}, SHA256: Manifest=#{cdata[:sha256]}, Actual=#{sha256[base]}"
if ENV['ENFORCE_CHECKSUMS'] and ENV['ENFORCE_CHECKSUMS'].downcase =~ %r/^(yes|on|set|1|enable|active|enforced?)$/
@logger.fatal msg
@vars[:return_code] = Errors::ARTIFACT_CHECKSUM_BAD
else
@logger.warn msg
@@ -278,45 +268,59 @@
module: cname,
name: cname,
build: build,
version: version,
file: local,
- sha256: (sha256.keys.include?(base)) ? (sha256[base].is_a?(TrueClass) ? cdata[:sha256] : sha256[base]) : Digest::SHA256.file(local).hexdigest
+ sha256: (sha256.keys.include?(base)) ? (sha256[base].is_a?(TrueClass) ? cdata[:sha256] : sha256[base]) : Digest::SHA256.file(local).hexdigest,
+ file_name: cdata[:file_name],
+ file_ext: cdata[:file_ext],
})
end
end
else
- @logger.fatal "Artifact not found: s3://#{name}/#{key}#{artifact}"
+ @logger.fatal "Artifact not found: s3://#{bucket}/#{key}#{artifact}"
@vars[:return_code] = Errors::ARTIFACT_NOT_FOUND
end
rescue Exception => e
@logger.error "Artifact error: #{artifact} #{e.class.name} #{e.message}"
raise e
end
end
# ---------------------------------------------------------------------------------------------------------------
- def getObjects(artifact, path)
+ def getObjects(artifact, path, cdata)
parts = URI(path).path.gsub(%r'^#{File::SEPARATOR}', '').split(File::SEPARATOR)
- name = parts.shift
- bucket = getBucket(name)
- key = File.join(parts, '')
- @logger.info "S3://#{name}:#{key} URL: #{path} #{artifact}"
+ bucket,prefix = if cdata[:s3_bucket] and cdata[:s3_key]
+ [cdata[:s3_bucket],File.join(File.dirname(cdata[:s3_key]),'')]
+ else
+ name = parts.shift
+ [name,File.join(parts, '')]
+ end
+ @logger.info "S3://#{bucket}:#{prefix} URL: #{path} #{artifact}"
objects = []
- bucket.objects(prefix: key).each do |object|
- if artifact.empty? or (not artifact.empty? and object.key =~ %r'#{key}#{artifact}')
- objects << object
+ @s3 = @repo.getS3() #bucket_client = getBucket(bucket)
+ bucket_objects = @s3.list_objects(bucket: bucket, prefix: prefix)
+ if bucket_objects
+ bucket_objects = bucket_objects[:contents]
+ bucket_objects.each do |object|
+ if artifact.empty? or (not artifact.empty? and object.key =~ %r'#{prefix}#{artifact}')
+ objects << object
+ end
end
end
- @logger.debug "S3://#{name}:#{key} has #{objects.size} objects"
- return key, name, objects
+ @logger.debug "S3://#{bucket}:#{prefix} has #{objects.size} objects"
+ return prefix, bucket, objects
end
# ---------------------------------------------------------------------------------------------------------------
- def doDownload(etag, local, object)
+ def doDownload(etag, local, object, bucket)
@logger.info "\t\tdownload #{object.size} bytes"
- response = object.get(:response_target => local)
+ begin
+ response = object.get(:response_target => local)
+ rescue NoMethodError
+ response = @s3.get_object(bucket: bucket, key: object.key, response_target: local)
+ end
File.utime(response.last_modified, response.last_modified, local)
@logger.info "\t\tdone"
check = calcLocalETag(etag, local)
if check.eql?(etag)
false
@@ -372,11 +376,11 @@
# ---------------------------------------------------------------------------------------------------------------
def parseComponent(comp)
cname,cdata = comp
if cdata[:url]
path, artifact = File.split(cdata[:url])
- version, build = getVersionBuild(path, artifact, comp)
+ version, build = (cdata[:version] and cdata[:build]) ? [cdata[:version], cdata[:build]] : getVersionBuild(path, artifact, comp)
elsif cdata[:base_url]
artifact = ''
if cdata[:build].nil?
# noinspection RubyUnusedLocalVariable
version, build = cdata[:version].split(%r'-')
@@ -387,10 +391,10 @@
path = File.join(cdata[:base_url], cdata[:version], cdata[:build])
end
else
path = ''
artifact = ''
- version, build = getVersionBuild(path, artifact, comp)
+ version, build = (cdata[:version] and cdata[:build]) ? [cdata[:version], cdata[:build]] : getVersionBuild(path, artifact, comp)
end
return artifact, path, version, build
end
# ---------------------------------------------------------------------------------------------------------------