lib/cicd/builder/mixlib/repo.rb in cicd-builder-0.9.23 vs lib/cicd/builder/mixlib/repo.rb in cicd-builder-0.9.25
- old
+ new
@@ -1,402 +1,51 @@
require 'json'
+require 'semverse'
module CiCd
module Builder
+ require 'cicd/builder/mixlib/repo/base'
+ require 'cicd/builder/mixlib/repo/S3'
+ # noinspection RubyResolve
+ if ENV.has_key?('REPO_TYPE') and (not ENV['REPO_TYPE'].capitalize.equal?('S3'))
+ require "cicd/builder/mixlib/repo/#{ENV['REPO_TYPE'].downcase}"
+ end
- # ---------------------------------------------------------------------------------------------------------------
- def getS3Bucket()
- unless @s3
- @s3 = AWS::S3.new(
- :access_key_id => ENV['AWS_ACCESS_KEY_ID'],
- :secret_access_key => ENV['AWS_SECRET_ACCESS_KEY'])
- end
- unless @s3_bucket
- @s3_bucket = @s3.buckets[ENV['AWS_S3_BUCKET']]
- end
- @s3_bucket
- end
-
- # ---------------------------------------------------------------------------------------------------------------
- def uploadToS3(artifacts)
- bucket = getS3Bucket()
- artifacts.each{|art|
- # makes no request, returns an AWS::S3::S3Object
- s3_obj = bucket.objects[art[:key]]
- upload = true
- if art[:data].has_key?(:file)
- md5 = Digest::MD5.file(art[:data][:file]).hexdigest
- else
- #noinspection RubyArgCount
- md5 = Digest::MD5.hexdigest(art[:data][:data])
+ # ---------------------------------------------------------------------------------------------------------------
+ def getRepoClass(type = nil)
+ if type.nil?
+ type ||= 'S3'
+ if ENV.has_key?('REPO_TYPE')
+ type = ENV['REPO_TYPE']
end
- if s3_obj.exists?
- @logger.info "s3://#{ENV['AWS_S3_BUCKET']}/#{art[:key]} exists"
- etag = s3_obj.etag.gsub(/"/,'')
- unless etag == md5
- checksum = s3_obj.metadata[:checksum]
- unless checksum and checksum == md5
- @logger.warn "s3://#{ENV['AWS_S3_BUCKET']}/#{art[:key]} is different from our #{art[:key]}(#{s3_obj.etag} <=> #{md5})"
- upload = true
- end
- end
- end
+ end
- if upload
- @logger.info "Upload new s3://#{ENV['AWS_S3_BUCKET']}/#{art[:key]}"
- # Get size before upload changes our object
- if art[:data].has_key?(:file)
- size = File.size(art[:data][:file])
- else
- size = art[:data][:data].length
- end
- art[:data][:metadata] = {checksum: md5, digest: "md5=#{md5}"}
- # art[:data][:'x-amz-meta-digest'] = "md5=#{md5}"
- s3_obj.write(art[:data])
- if art.has_key?(:public_url)
- @vars[art[:public_url]] = s3_obj.public_url
- end
- if art.has_key?(:read_url)
- @vars[art[:read_url]] = s3_obj.url_for(:read) if art.has_key?(:read_url)
- end
- @logger.info "#{art[:label]}: #{@vars[art[:public_url]]}" if art.has_key?(:public_url)
- # if size > 16 * 1024 * 1024
- # if size < 5 * 1024 * 1024 * 1000
- # @logger.debug "#{art[:label]}: Multipart etag: #{s3_obj.etag}"
- # s3_obj.copy_to("#{art[:key]}.copy")
- # s3_obj = bucket.objects["#{art[:key]}.copy"]
- # s3_obj.move_to(art[:key])
- # s3_obj = bucket.objects[art[:key]]
- # @logger.debug "#{art[:label]}: Revised etag: #{s3_obj.etag}"
- # else
- # @logger.warn "#{art[:label]}: Multipart etag: #{s3_obj.etag} on asset > 5Gb"
- # end
- # end
- end
- }
- 0
- end
-
- # ---------------------------------------------------------------------------------------------------------------
- def getArtifactsDefinition()
- nil
- end
-
- # ---------------------------------------------------------------------------------------------------------------
- def getNamingDefinition()
- nil
- end
-
- # ---------------------------------------------------------------------------------------------------------------
- def initInventory()
-
- hash =
- {
- id: "#{@vars[:project_name]}",
- # In case future generations introduce incompatible features
- gen: "#{@options[:gen]}",
- container: {
- artifacts: %w(assembly metainfo checksum),
- naming: '<product>-<major>.<minor>.<patch>-<branch>-release-<number>-build-<number>.<extension>',
- assembly: {
- extension: 'tar.gz',
- type: 'targz'
- },
- metainfo: {
- extension: 'MANIFEST.json',
- type: 'json'
- },
- checksum: {
- extension: 'checksum',
- type: 'Digest::SHA256'
- },
- variants: {
- :"#{@vars[:variant]}" => {
- latest: {
- build: 0,
- branch: 0,
- version: 0,
- release: 0,
- },
- versions: [ "#{@vars[:build_ver]}" ],
- branches: [ "#{@vars[:build_bra]}" ],
- builds: [
- {
- drawer: @vars[:build_nam],
- build_name: @vars[:build_rel],
- build_number: @vars[:build_num],
- release: @vars[:release],
- }
- ],
- }
- }
- }
- }
- artifacts = getArtifactsDefinition()
- naming = getNamingDefinition()
-
- # By default we use the internal definition ...
- if artifacts
- artifacts.each do |name,artifact|
- hash[:container][name] = artifact
+ clazz = Object.const_get("CiCd::Builder::Repo::#{type}")
+ if block_given?
+ if clazz.is_a?(Class) and not clazz.nil?
+ yield
end
end
- # By default we use the internal definition ...
- if naming
- hash[:container][:naming] = naming
- end
- JSON.pretty_generate( hash, { indent: "\t", space: ' '})
- end
+ clazz
+ end
- # ---------------------------------------------------------------------------------------------------------------
- def takeInventory()
- def _update(hash, key, value)
- h = {}
- i = -1
- hash[key].each { |v| h[v] = i+=1 }
- unless h.has_key?(value)
- h[value] = h.keys.size # No -1 because this is evaluated BEFORE we make the addition!
- end
- s = h.sort_by { |_, v| v }
- s = s.map { |v| v[0] }
- hash[key] = s
- h[value]
- end
-
- # Read and parse in JSON
- json_s = ''
- json = nil
- varianth = nil
-
- bucket = getS3Bucket()
- key = "#{@vars[:project_name]}/INVENTORY.json"
- s3_obj = bucket.objects[key]
- # If the inventory has started then add to it
- if s3_obj.exists?
- s3_obj.read(){|chunk|
- json_s << chunk
- }
- json = Yajl::Parser.parse(json_s)
- over = false
- # Is the inventory format up to date ...
- # TODO: [2014-07-27 Christo] Use semver gem ...
- require 'chef/exceptions'
- require 'chef/version_constraint'
- require 'chef/version_class'
-
- begin
- version = Chef::Version.new(json['gen'])
- rescue Chef::Exceptions::InvalidCookbookVersion => e
- json['gen'] = "#{json['gen']}.0.0"
- version = Chef::Version.new(json['gen'])
- end
-
- begin
- our_ver = Chef::Version.new(@options[:gen])
- constraint = Chef::VersionConstraint.new("<= #{@options[:gen]}")
- rescue Chef::Exceptions::InvalidVersionConstraint => e
- raise CiCd::Builder::Errors::InvalidVersionConstraint.new e.message
- rescue Chef::Exceptions::InvalidCookbookVersion => e
- raise CiCd::Builder::Errors::InvalidVersion.new e.message
- end
-
- unless constraint.include?(version)
- raise CiCd::Builder::Errors::InvalidVersion.new "The inventory generation is newer than I can manage: #{version} <=> #{our_ver}"
- end
- if json['container'] and json['container']['variants']
- # but does not have our variant then add it
- variants = json['container']['variants']
- unless variants[@vars[:variant]]
- variants[@vars[:variant]] = {}
- varianth = variants[@vars[:variant]]
- varianth['builds'] = []
- varianth['branches'] = []
- varianth['versions'] = []
- varianth['releases'] = []
- varianth['latest'] = {
- branch: -1,
- version: -1,
- build: -1,
- release: -1,
- }
- end
- varianth = variants[@vars[:variant]]
- # If the inventory 'latest' format is up to date ...
- unless varianth['latest'] and
- varianth['latest'].is_a?(Hash)
- # Start over ... too old/ incompatible
- over = true
- end
- else
- # Start over ... too old/ incompatible
- over = true
- end
- else
- # Start a new inventory
- over = true
- end
- # Starting fresh ?
- if over or json.nil?
- json_s = initInventory()
- else
- raise CiCd::Builder::Errors::Internal.new sprintf('Internal logic error! %s::%d', __FILE__,__LINE__) if varianth.nil?
- # Add the new build if we don't have it
- unless varianth['builds'].map { |b| b['build_name'] }.include?(@vars[:build_rel])
- #noinspection RubyStringKeysInHashInspection
- filing = {
- 'drawer' => @vars[:build_nam],
- 'build_name' => @vars[:build_rel],
- 'build_number' => @vars[:build_num],
- 'release' => @vars[:release],
- }
- if @vars.has_key?(:artifacts)
- filing['artifacts'] = @vars[:artifacts].map { |artifact| File.basename(artifact[:key]) }
- end
- assembly = json['container']['assembly'] or raise("Expected an 'assembly'")
- if assembly['extension'] != !vars[:build_ext]
- # noinspection RubyStringKeysInHashInspection
- filing['assembly'] = {
- 'extension' => @vars[:build_ext],
- 'type' => 'tarbzip2'
- }
- end
- varianth['builds'] << filing
- end
- build_lst = (varianth['builds'].size-1)
- build_rel = build_lst
- i = -1
- varianth['builds'].each{ |h|
- i += 1
- convert_build(h)
- convert_build(varianth['builds'][build_rel])
- if h['release'].to_i > varianth['builds'][build_rel]['release'].to_i
- build_rel = i
- elsif h['release'] == varianth['builds'][build_rel]['release']
- build_rel = i if h['build_number'].to_i > varianth['builds'][build_rel]['build_number'].to_i
- end
- }
-
- # Add new branch ...
- build_bra = _update(varianth, 'branches', @vars[:build_bra])
- # Add new version ...
- build_ver = _update(varianth, 'versions', @vars[:build_ver])
-
- # Set latest
- varianth['latest'] = {
- branch: build_bra,
- version: build_ver,
- build: build_lst,
- release: build_rel,
- }
- json['gen'] = @options[:gen]
- json_s = JSON.pretty_generate( json, { indent: "\t", space: ' '})
- end
- begin
- md5 = Digest::MD5.hexdigest(json_s)
- resp = s3_obj.write({
- :data => json_s,
- :'x-amz-meta-digest' => "md5=#{md5}",
- :metadata => { checksum: md5 },
- })
-
- @logger.info "Inventory URL: #{s3_obj.url_for(:read)}"
-
- case resp.class.name
- when %r'^AWS::S3::(S3Object|ObjectVersion)'
- return 0
- else
- return 1
- end
- rescue Exception => e
- return -1
- end
- end
-
- def convert_build(h)
- if h.has_key?('number')
- h['build_number'] = h['number']
- h.delete 'number'
- elsif h.has_key?('build_number')
- h.delete 'number'
+ # ---------------------------------------------------------------------------------------------------------------
+ def uploadBuildArtifacts()
+ clazz = getRepoClass()
+ if clazz.is_a?(Class) and not clazz.nil?
+ @repo = clazz.new(self)
+ @vars[:return_code] = @repo.uploadBuildArtifacts()
+ @vars[:return_code]
else
- h_build = h.has_key?('build') ? h['build'] : h['build_name']
- h_number = h_build.gsub(/^.*?-build-([0-9]+)$/, '\1').to_i
-
- h['build_number'] = h_number
- h['build_name'] = h_build
- h.delete 'build'
- h.delete 'number'
+ @logger.error "CiCd::Builder::Repo::#{type} is not a valid repo class"
+ @vars[:return_code] = Errors::BUILDER_REPO_TYPE
end
- if h.has_key?('build')
- h_build = h.has_key?('build')
- h_number = h_build.gsub(/^.*?-build-([0-9]+)$/, '\1').to_i
-
- h['build_number'] = h_number
- h['build_name'] = h_build
- h.delete 'build'
- h.delete 'number'
- end
- h
+ @vars[:return_code]
end
# ---------------------------------------------------------------------------------------------------------------
- def uploadBuildArtifacts()
- if @vars.has_key?(:build_dir) and @vars.has_key?(:build_pkg)
- begin
- if File.exists?(@vars[:build_pkg])
-
- artifacts = @vars[:artifacts] rescue []
-
- key = "#{@vars[:project_name]}/#{@vars[:variant]}/#{@vars[:build_nam]}/#{@vars[:build_rel]}"
- # Store the assembly - be sure to inherit possible overrides in pkg name and ext but dictate the drawer!
- artifacts << {
- key: "#{File.join(File.dirname(key),File.basename(@vars[:build_pkg]))}",
- data: {:file => @vars[:build_pkg]},
- public_url: :build_url,
- label: 'Package URL'
- }
-
- # Store the metadata
- manifest = manifestMetadata()
- artifacts << {
- key: "#{key}.MANIFEST.json",
- data: {:data => manifest},
- public_url: :manifest_url,
- read_url: :manifest_url,
- label: 'Manifest URL'
- }
-
- # Store the checksum
- artifacts << {
- key: "#{@vars[:project_name]}/#{@vars[:variant]}/#{@vars[:build_nam]}/#{@vars[:build_rel]}.checksum",
- data: {:data => @vars[:build_sha]},
- public_url: :checksum_url,
- read_url: :checksum_url,
- label: 'Checksum URL'
- }
-
- @vars[:return_code] = uploadToS3(artifacts)
- if 0 == @vars[:return_code]
- @vars[:return_code] = takeInventory()
- end
- @vars[:return_code]
- else
- @vars[:return_code] = 1
- end
- rescue => e
- @logger.error "#{e.class.name} #{e.message}"
- @vars[:return_code] = -99
- raise e
- end
- else
- @vars[:return_code] = 2
- end
- @vars[:return_code]
- end
-
- # ---------------------------------------------------------------------------------------------------------------
def manifestMetadata
manifest = @vars[:build_mdd].dup
manifest[:manifest] = getBuilderVersion
@@ -409,11 +58,11 @@
patch: version_patch,
build: @vars[:build_num],
branch: @vars[:build_bra],
}
manifest[:build] = {
- name: @vars[:build_rel],
+ name: @vars[:build_nmn],
base: @vars[:build_nam],
date: @vars[:build_dte],
vrb: @vars[:build_vrb],
branch: @vars[:build_bra],
checksum: @vars[:build_sha],
@@ -442,7 +91,17 @@
end
}
JSON.pretty_generate( manifest, { indent: "\t", space: ' '})
end
- end
+ # ---------------------------------------------------------------------------------------------------------------
+ def getArtifactsDefinition()
+ nil
+ end
+
+ # ---------------------------------------------------------------------------------------------------------------
+ def getNamingDefinition()
+ nil
+ end
+
+ end
end
\ No newline at end of file