lib/azure/storage/blob/block.rb in azure-storage-blob-1.1.0 vs lib/azure/storage/blob/block.rb in azure-storage-blob-2.0.0

- old
+ new

@@ -1,530 +1,530 @@ -# frozen_string_literal: true - -#------------------------------------------------------------------------- -# # Copyright (c) Microsoft and contributors. All rights reserved. -# -# The MIT License(MIT) - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files(the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and / or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions : - -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -#-------------------------------------------------------------------------- - -require "base64" - -module Azure::Storage - module Blob - # Represents a Block as part of a BlockList - # The type should be one of :uncommitted, :committed or :latest - class Block - def initialize - @type = :latest - yield self if block_given? - end - - attr_accessor :name - attr_accessor :size - attr_accessor :type - end - - # Public: Creates a new block blob or updates the content of an existing block blob. - # - # Updating an existing block blob overwrites any existing metadata on the blob - # Partial updates are not supported with create_block_blob the content of the - # existing blob is overwritten with the content of the new blob. To perform a - # partial update of the content of a block blob, use the create_block_list - # method. - # - # Note that the default content type is application/octet-stream. - # - # ==== Attributes - # - # * +container+ - String. The container name. - # * +blob+ - String. The blob name. - # * +content+ - IO or String. The content of the blob. - # * +options+ - Hash. Optional parameters. - # - # ==== Options - # - # Accepted key/value pairs in options parameter are: - # * +:transactional_md5+ - String. An MD5 hash of the blob content. This hash is used to verify the integrity of the blob during transport. - # When this header is specified, the storage service checks the hash that has arrived with the one that was sent. - # If the two hashes do not match, the operation will fail with error code 400 (Bad Request). - # * +:single_upload_threshold+ - Integer. Threshold in bytes for single upload, must be lower than 256MB or 256MB will be used. - # * +:content_length+ - Integer. Length of the content to upload, must be specified if 'content' does not implement 'size'. - # * +:content_type+ - String. Content type for the blob. Will be saved with blob. - # * +:content_encoding+ - String. Content encoding for the blob. Will be saved with blob. - # * +:content_language+ - String. Content language for the blob. Will be saved with blob. - # * +:content_md5+ - String. Content MD5 for the blob. Will be saved with blob. - # * +:cache_control+ - String. Cache control for the blob. Will be saved with blob. - # * +:content_disposition+ - String. Conveys additional information about how to process the response payload, - # and also can be used to attach additional metadata - # * +:metadata+ - Hash. Custom metadata values to store with the blob. - # * +:timeout+ - Integer. A timeout in seconds. - # * +:request_id+ - String. Provides a client-generated, opaque value with a 1 KB character limit that is recorded - # in the analytics logs when storage analytics logging is enabled. - # * +:if_modified_since+ - String. A DateTime value. Specify this conditional header to create a new blob - # only if the blob has been modified since the specified date/time. If the blob has not been modified, - # the Blob service returns status code 412 (Precondition Failed). - # * +:if_unmodified_since+ - String. A DateTime value. Specify this conditional header to create a new blob - # only if the blob has not been modified since the specified date/time. If the blob has been modified, - # the Blob service returns status code 412 (Precondition Failed). - # * +:if_match+ - String. An ETag value. Specify an ETag value for this conditional header to create a new blob - # only if the blob's ETag value matches the value specified. If the values do not match, - # the Blob service returns status code 412 (Precondition Failed). - # * +:if_none_match+ - String. An ETag value. Specify an ETag value for this conditional header to create a new blob - # only if the blob's ETag value does not match the value specified. If the values are identical, - # the Blob service returns status code 412 (Precondition Failed). - # * +:lease_id+ - String. Required if the blob has an active lease. To perform this operation on a blob with an active lease, - # specify the valid lease ID for this header. - # - # See http://msdn.microsoft.com/en-us/library/azure/dd179451.aspx - # - # Returns a Blob - def create_block_blob(container, blob, content, options = {}) - size = if content.respond_to? :size - content.size - elsif options[:content_length] - options[:content_length] - else - raise ArgumentError, "Either optional parameter 'content_length' should be set or 'content' should implement 'size' method to get payload's size." - end - - threshold = get_single_upload_threshold(options[:single_upload_threshold]) - if size > threshold - create_block_blob_multiple_put(container, blob, content, size, options) - else - create_block_blob_single_put(container, blob, content, options) - end - end - - # Public: Creates a new block to be committed as part of a block blob. - # - # ==== Attributes - # - # * +container+ - String. The container name. - # * +blob+ - String. The blob name. - # * +block_id+ - String. The block id. Note: this should be the raw block id, not Base64 encoded. - # * +content+ - IO or String. The content of the blob. - # * +options+ - Hash. Optional parameters. - # - # ==== Options - # - # Accepted key/value pairs in options parameter are: - # * +:content_md5+ - String. Content MD5 for the request contents. - # * +:timeout+ - Integer. A timeout in seconds. - # * +:request_id+ - String. Provides a client-generated, opaque value with a 1 KB character limit that is recorded - # in the analytics logs when storage analytics logging is enabled. - # * +:lease_id+ - String. Required if the blob has an active lease. To perform this operation on a blob with an - # active lease, specify the valid lease ID for this header. - # - # See http://msdn.microsoft.com/en-us/library/azure/dd135726.aspx - # - # Returns response of the operation - def put_blob_block(container, blob, block_id, content, options = {}) - query = { "comp" => "block" } - StorageService.with_query query, "blockid", Base64.strict_encode64(block_id) - StorageService.with_query query, "timeout", options[:timeout].to_s if options[:timeout] - - uri = blob_uri(container, blob, query) - - headers = {} - StorageService.with_header headers, "Content-MD5", options[:content_md5] - headers["x-ms-lease-id"] = options[:lease_id] if options[:lease_id] - - response = call(:put, uri, content, headers, options) - response.headers["Content-MD5"] - end - - # Public: Commits existing blob blocks to a blob. - # - # This method writes a blob by specifying the list of block IDs that make up the - # blob. In order to be written as part of a blob, a block must have been - # successfully written to the server in a prior put_blob_block method. - # - # You can call Put Block List to update a blob by uploading only those blocks - # that have changed, then committing the new and existing blocks together. - # You can do this by specifying whether to commit a block from the committed - # block list or from the uncommitted block list, or to commit the most recently - # uploaded version of the block, whichever list it may belong to. - # - # ==== Attributes - # - # * +container+ - String. The container name. - # * +blob+ - String. The blob name. - # * +block_list+ - Array. A ordered list of lists in the following format: - # [ ["block_id1", :committed], ["block_id2", :uncommitted], ["block_id3"], ["block_id4", :committed]... ] - # The first element of the inner list is the block_id, the second is optional - # and can be either :committed or :uncommitted to indicate in which group of blocks - # the id should be looked for. If it is omitted, the latest of either group will be used. - # * +options+ - Hash. Optional parameters. - # - # ==== Options - # - # Accepted key/value pairs in options parameter are: - # * +:transactional_md5+ - String. Content MD5 for the request contents (not the blob contents!) - # * +:content_type+ - String. Content type for the blob. Will be saved with blob. - # * +:content_encoding+ - String. Content encoding for the blob. Will be saved with blob. - # * +:content_language+ - String. Content language for the blob. Will be saved with blob. - # * +:content_md5+ - String. Content MD5 for the blob. Will be saved with blob. - # * +:cache_control+ - String. Cache control for the blob. Will be saved with blob. - # * +:content_disposition+ - String. Conveys additional information about how to process the response payload, - # and also can be used to attach additional metadata - # * +:metadata+ - Hash. Custom metadata values to store with the blob. - # * +:timeout+ - Integer. A timeout in seconds. - # * +:request_id+ - String. Provides a client-generated, opaque value with a 1 KB character limit that is recorded - # in the analytics logs when storage analytics logging is enabled. - # * +:lease_id+ - String. Required if the blob has an active lease. To perform this operation on a blob with an - # active lease, specify the valid lease ID for this header. - # - # This operation also supports the use of conditional headers to commit the block list if a specified condition is met. - # For more information, see https://msdn.microsoft.com/en-us/library/azure/dd179371.aspx - # - # See http://msdn.microsoft.com/en-us/library/azure/dd179467.aspx - # - # Returns nil on success - def commit_blob_blocks(container, blob, block_list, options = {}) - query = { "comp" => "blocklist" } - StorageService.with_query query, "timeout", options[:timeout].to_s if options[:timeout] - - uri = blob_uri(container, blob, query) - - headers = {} - unless options.empty? - StorageService.with_header headers, "Content-MD5", options[:transactional_md5] - StorageService.with_header headers, "x-ms-blob-content-type", options[:content_type] - StorageService.with_header headers, "x-ms-blob-content-encoding", options[:content_encoding] - StorageService.with_header headers, "x-ms-blob-content-language", options[:content_language] - StorageService.with_header headers, "x-ms-blob-content-md5", options[:content_md5] - StorageService.with_header headers, "x-ms-blob-cache-control", options[:cache_control] - StorageService.with_header headers, "x-ms-blob-content-disposition", options[:content_disposition] - - StorageService.add_metadata_to_headers(options[:metadata], headers) - add_blob_conditional_headers(options, headers) - headers["x-ms-lease-id"] = options[:lease_id] if options[:lease_id] - end - headers["x-ms-blob-content-type"] = Default::CONTENT_TYPE_VALUE unless headers["x-ms-blob-content-type"] - body = Serialization.block_list_to_xml(block_list) - call(:put, uri, body, headers, options) - nil - end - - # Public: Retrieves the list of blocks that have been uploaded as part of a block blob. - # - # There are two block lists maintained for a blob: - # 1) Committed Block List: The list of blocks that have been successfully - # committed to a given blob with commitBlobBlocks. - # 2) Uncommitted Block List: The list of blocks that have been uploaded for a - # blob using Put Block (REST API), but that have not yet been committed. - # These blocks are stored in Microsoft Azure in association with a blob, but do - # not yet form part of the blob. - # - # ==== Attributes - # - # * +container+ - String. The container name. - # * +blob+ - String. The blob name. - # * +options+ - Hash. Optional parameters. - # - # ==== Options - # - # Accepted key/value pairs in options parameter are: - # * +:blocklist_type+ - Symbol. One of :all, :committed, :uncommitted. Defaults to :all (optional) - # * +:snapshot+ - String. An opaque DateTime value that specifies the blob snapshot to - # retrieve information from. (optional) - # * +:timeout+ - Integer. A timeout in seconds. - # * +:request_id+ - String. Provides a client-generated, opaque value with a 1 KB character limit that is recorded - # in the analytics logs when storage analytics logging is enabled. - # * +:location_mode+ - LocationMode. Specifies the location mode used to decide - # which location the request should be sent to. - # * +:lease_id+ - String. If this header is specified, the operation will be performed only if both of the - # following conditions are met: - # - The blob's lease is currently active. - # - The lease ID specified in the request matches that of the blob. - # If this header is specified and both of these conditions are not met, the request will fail - # and the operation will fail with status code 412 (Precondition Failed). - # - # See http://msdn.microsoft.com/en-us/library/azure/dd179400.aspx - # - # Returns a list of Azure::Storage::Entity::Blob::Block instances - def list_blob_blocks(container, blob, options = {}) - options[:blocklist_type] = options[:blocklist_type] || :all - - query = { "comp" => "blocklist" } - StorageService.with_query query, "snapshot", options[:snapshot] - StorageService.with_query query, "blocklisttype", options[:blocklist_type].to_s if options[:blocklist_type] - StorageService.with_query query, "timeout", options[:timeout].to_s if options[:timeout] - - headers = options[:lease_id] ? { "x-ms-lease-id" => options[:lease_id] } : {} - - options[:request_location_mode] = Azure::Storage::Common::RequestLocationMode::PRIMARY_OR_SECONDARY - uri = blob_uri(container, blob, query, options) - - response = call(:get, uri, nil, headers, options) - - Serialization.block_list_from_xml(response.body) - end - - # Public: Creates a new block blob or updates the content of an existing block blob. - # - # Updating an existing block blob overwrites any existing metadata on the blob - # Partial updates are not supported with create_block_blob the content of the - # existing blob is overwritten with the content of the new blob. To perform a - # partial update of the content of a block blob, use the create_block_list - # method. - # - # Note that the default content type is application/octet-stream. - # - # ==== Attributes - # - # * +container+ - String. The container name. - # * +blob+ - String. The blob name. - # * +content+ - IO or String. The content of the blob. - # * +options+ - Hash. Optional parameters. - # - # ==== Options - # - # Accepted key/value pairs in options parameter are: - # * +:transactional_md5+ - String. An MD5 hash of the blob content. This hash is used to verify the integrity of the blob during transport. - # When this header is specified, the storage service checks the hash that has arrived with the one that was sent. - # If the two hashes do not match, the operation will fail with error code 400 (Bad Request). - # * +:single_upload_threshold+ - Integer. Threshold in bytes for single upload, must be lower than 256MB or 256MB will be used. - # * +:content_length+ - Integer. Length of the content to upload, must be specified if 'content' does not implement 'size'. - # * +:content_type+ - String. Content type for the blob. Will be saved with blob. - # * +:content_encoding+ - String. Content encoding for the blob. Will be saved with blob. - # * +:content_language+ - String. Content language for the blob. Will be saved with blob. - # * +:content_md5+ - String. Content MD5 for the blob. Will be saved with blob. - # * +:cache_control+ - String. Cache control for the blob. Will be saved with blob. - # * +:content_disposition+ - String. Conveys additional information about how to process the response payload, - # and also can be used to attach additional metadata - # * +:metadata+ - Hash. Custom metadata values to store with the blob. - # * +:timeout+ - Integer. A timeout in seconds. - # * +:request_id+ - String. Provides a client-generated, opaque value with a 1 KB character limit that is recorded - # in the analytics logs when storage analytics logging is enabled. - # * +:if_modified_since+ - String. A DateTime value. Specify this conditional header to create a new blob - # only if the blob has been modified since the specified date/time. If the blob has not been modified, - # the Blob service returns status code 412 (Precondition Failed). - # * +:if_unmodified_since+ - String. A DateTime value. Specify this conditional header to create a new blob - # only if the blob has not been modified since the specified date/time. If the blob has been modified, - # the Blob service returns status code 412 (Precondition Failed). - # * +:if_match+ - String. An ETag value. Specify an ETag value for this conditional header to create a new blob - # only if the blob's ETag value matches the value specified. If the values do not match, - # the Blob service returns status code 412 (Precondition Failed). - # * +:if_none_match+ - String. An ETag value. Specify an ETag value for this conditional header to create a new blob - # only if the blob's ETag value does not match the value specified. If the values are identical, - # the Blob service returns status code 412 (Precondition Failed). - # * +:lease_id+ - String. Required if the blob has an active lease. To perform this operation on a blob with an active lease, - # specify the valid lease ID for this header. - # - # See http://msdn.microsoft.com/en-us/library/azure/dd179451.aspx - # - # Returns a Blob - alias create_block_blob_from_content create_block_blob - - # Protected: Creates a new block blob or updates the content of an existing block blob with single API call - # - # Updating an existing block blob overwrites any existing metadata on the blob - # Partial updates are not supported with create_block_blob the content of the - # existing blob is overwritten with the content of the new blob. To perform a - # partial update of the content of a block blob, use the create_block_list - # method. - # - # Note that the default content type is application/octet-stream. - # - # ==== Attributes - # - # * +container+ - String. The container name. - # * +blob+ - String. The blob name. - # * +content+ - IO or String. The content of the blob. - # * +options+ - Hash. Optional parameters. - # - # ==== Options - # - # Accepted key/value pairs in options parameter are: - # * +:transactional_md5+ - String. An MD5 hash of the blob content. This hash is used to verify the integrity of the blob during transport. - # When this header is specified, the storage service checks the hash that has arrived with the one that was sent. - # If the two hashes do not match, the operation will fail with error code 400 (Bad Request). - # * +:content_length+ - Integer. Length of the content to upload, must be specified if 'content' does not implement 'size'. - # * +:content_type+ - String. Content type for the blob. Will be saved with blob. - # * +:content_encoding+ - String. Content encoding for the blob. Will be saved with blob. - # * +:content_language+ - String. Content language for the blob. Will be saved with blob. - # * +:content_md5+ - String. Content MD5 for the blob. Will be saved with blob. - # * +:cache_control+ - String. Cache control for the blob. Will be saved with blob. - # * +:content_disposition+ - String. Conveys additional information about how to process the response payload, - # and also can be used to attach additional metadata - # * +:metadata+ - Hash. Custom metadata values to store with the blob. - # * +:timeout+ - Integer. A timeout in seconds. - # * +:request_id+ - String. Provides a client-generated, opaque value with a 1 KB character limit that is recorded - # in the analytics logs when storage analytics logging is enabled. - # * +:if_modified_since+ - String. A DateTime value. Specify this conditional header to create a new blob - # only if the blob has been modified since the specified date/time. If the blob has not been modified, - # the Blob service returns status code 412 (Precondition Failed). - # * +:if_unmodified_since+ - String. A DateTime value. Specify this conditional header to create a new blob - # only if the blob has not been modified since the specified date/time. If the blob has been modified, - # the Blob service returns status code 412 (Precondition Failed). - # * +:if_match+ - String. An ETag value. Specify an ETag value for this conditional header to create a new blob - # only if the blob's ETag value matches the value specified. If the values do not match, - # the Blob service returns status code 412 (Precondition Failed). - # * +:if_none_match+ - String. An ETag value. Specify an ETag value for this conditional header to create a new blob - # only if the blob's ETag value does not match the value specified. If the values are identical, - # the Blob service returns status code 412 (Precondition Failed). - # * +:lease_id+ - String. Required if the blob has an active lease. To perform this operation on a blob with an active lease, - # specify the valid lease ID for this header. - # - # See http://msdn.microsoft.com/en-us/library/azure/dd179451.aspx - # - # Returns a Blob - protected - def create_block_blob_single_put(container, blob, content, options = {}) - query = {} - StorageService.with_query query, "timeout", options[:timeout].to_s if options[:timeout] - - uri = blob_uri(container, blob, query) - - headers = {} - - # set x-ms-blob-type to BlockBlob - StorageService.with_header headers, "x-ms-blob-type", "BlockBlob" - - # set the rest of the optional headers - StorageService.with_header headers, "Content-MD5", options[:transactional_md5] - StorageService.with_header headers, "Content-Length", options[:content_length] - StorageService.with_header headers, "x-ms-blob-content-encoding", options[:content_encoding] - StorageService.with_header headers, "x-ms-blob-content-language", options[:content_language] - StorageService.with_header headers, "x-ms-blob-content-md5", options[:content_md5] - StorageService.with_header headers, "x-ms-blob-cache-control", options[:cache_control] - StorageService.with_header headers, "x-ms-blob-content-disposition", options[:content_disposition] - StorageService.with_header headers, "x-ms-lease-id", options[:lease_id] - - StorageService.add_metadata_to_headers options[:metadata], headers - add_blob_conditional_headers options, headers - headers["x-ms-blob-content-type"] = get_or_apply_content_type(content, options[:content_type]) - # call PutBlob - response = call(:put, uri, content, headers, options) - - result = Serialization.blob_from_headers(response.headers) - result.name = blob - result.metadata = options[:metadata] if options[:metadata] - - result - end - - # Protected: Creates a new block blob or updates the content of an existing block blob with multiple upload - # - # Updating an existing block blob overwrites any existing metadata on the blob - # Partial updates are not supported with create_block_blob the content of the - # existing blob is overwritten with the content of the new blob. To perform a - # partial update of the content of a block blob, use the create_block_list - # method. - # - # Note that the default content type is application/octet-stream. - # - # ==== Attributes - # - # * +container+ - String. The container name. - # * +blob+ - String. The blob name. - # * +content+ - IO or String. The content of the blob. - # * +size+ - Integer. The size of the content. - # * +options+ - Hash. Optional parameters. - # - # ==== Options - # - # Accepted key/value pairs in options parameter are: - # * +:content_type+ - String. Content type for the blob. Will be saved with blob. - # * +:content_encoding+ - String. Content encoding for the blob. Will be saved with blob. - # * +:content_language+ - String. Content language for the blob. Will be saved with blob. - # * +:content_md5+ - String. Content MD5 for the blob. Will be saved with blob. - # * +:cache_control+ - String. Cache control for the blob. Will be saved with blob. - # * +:content_disposition+ - String. Conveys additional information about how to process the response payload, - # and also can be used to attach additional metadata - # * +:metadata+ - Hash. Custom metadata values to store with the blob. - # * +:timeout+ - Integer. A timeout in seconds. - # * +:request_id+ - String. Provides a client-generated, opaque value with a 1 KB character limit that is recorded - # in the analytics logs when storage analytics logging is enabled. - # * +:lease_id+ - String. Required if the blob has an active lease. To perform this operation on a blob with an active lease, - # specify the valid lease ID for this header. - # - # See http://msdn.microsoft.com/en-us/library/azure/dd179451.aspx - # - # Returns a Blob - protected - def create_block_blob_multiple_put(container, blob, content, size, options = {}) - content_type = get_or_apply_content_type(content, options[:content_type]) - content = StringIO.new(content) if content.is_a? String - block_size = get_block_size(size) - # Get the number of blocks - block_count = (Float(size) / Float(block_size)).ceil - block_list = [] - for block_id in 0...block_count - id = block_id.to_s.rjust(6, "0") - put_blob_block(container, blob, id, content.read(block_size), timeout: options[:timeout], lease_id: options[:lease_id]) - block_list.push([id]) - end - - # Commit the blocks put - commit_options = {} - commit_options[:content_type] = content_type - commit_options[:content_encoding] = options[:content_encoding] if options[:content_encoding] - commit_options[:content_language] = options[:content_language] if options[:content_language] - commit_options[:content_md5] = options[:content_md5] if options[:content_md5] - commit_options[:cache_control] = options[:cache_control] if options[:cache_control] - commit_options[:content_disposition] = options[:content_disposition] if options[:content_disposition] - commit_options[:metadata] = options[:metadata] if options[:metadata] - commit_options[:timeout] = options[:timeout] if options[:timeout] - commit_options[:request_id] = options[:request_id] if options[:request_id] - commit_options[:lease_id] = options[:lease_id] if options[:lease_id] - - commit_blob_blocks(container, blob, block_list, commit_options) - - get_properties_options = {} - get_properties_options[:lease_id] = options[:lease_id] if options[:lease_id] - - # Get the blob properties - get_blob_properties(container, blob, get_properties_options) - end - - # Protected: Gets the single upload threshold according to user's preference - # - # ==== Attributes - # - # * +container+ - String. The container name. - # - # Returns an Integer - protected - def get_single_upload_threshold(userThreshold) - if userThreshold.nil? - BlobConstants::DEFAULT_SINGLE_BLOB_PUT_THRESHOLD_IN_BYTES - elsif userThreshold <= 0 - raise ArgumentError, "Single Upload Threshold should be positive number" - elsif userThreshold < BlobConstants::MAX_SINGLE_UPLOAD_BLOB_SIZE_IN_BYTES - userThreshold - else - BlobConstants::MAX_SINGLE_UPLOAD_BLOB_SIZE_IN_BYTES - end - end - - protected - def get_block_size(size) - if size > BlobConstants::MAX_BLOCK_BLOB_SIZE - raise ArgumentError, "Block blob size should be less than #{BlobConstants::MAX_BLOCK_BLOB_SIZE} bytes in size" - elsif (size / BlobConstants::MAX_BLOCK_COUNT) < BlobConstants::DEFAULT_WRITE_BLOCK_SIZE_IN_BYTES - BlobConstants::DEFAULT_WRITE_BLOCK_SIZE_IN_BYTES - else - BlobConstants::MAX_BLOCK_SIZE - end - end - end -end +# frozen_string_literal: true + +#------------------------------------------------------------------------- +# # Copyright (c) Microsoft and contributors. All rights reserved. +# +# The MIT License(MIT) + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files(the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and / or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions : + +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +#-------------------------------------------------------------------------- + +require "base64" + +module Azure::Storage + module Blob + # Represents a Block as part of a BlockList + # The type should be one of :uncommitted, :committed or :latest + class Block + def initialize + @type = :latest + yield self if block_given? + end + + attr_accessor :name + attr_accessor :size + attr_accessor :type + end + + # Public: Creates a new block blob or updates the content of an existing block blob. + # + # Updating an existing block blob overwrites any existing metadata on the blob + # Partial updates are not supported with create_block_blob the content of the + # existing blob is overwritten with the content of the new blob. To perform a + # partial update of the content of a block blob, use the create_block_list + # method. + # + # Note that the default content type is application/octet-stream. + # + # ==== Attributes + # + # * +container+ - String. The container name. + # * +blob+ - String. The blob name. + # * +content+ - IO or String. The content of the blob. + # * +options+ - Hash. Optional parameters. + # + # ==== Options + # + # Accepted key/value pairs in options parameter are: + # * +:transactional_md5+ - String. An MD5 hash of the blob content. This hash is used to verify the integrity of the blob during transport. + # When this header is specified, the storage service checks the hash that has arrived with the one that was sent. + # If the two hashes do not match, the operation will fail with error code 400 (Bad Request). + # * +:single_upload_threshold+ - Integer. Threshold in bytes for single upload, must be lower than 256MB or 256MB will be used. + # * +:content_length+ - Integer. Length of the content to upload, must be specified if 'content' does not implement 'size'. + # * +:content_type+ - String. Content type for the blob. Will be saved with blob. + # * +:content_encoding+ - String. Content encoding for the blob. Will be saved with blob. + # * +:content_language+ - String. Content language for the blob. Will be saved with blob. + # * +:content_md5+ - String. Content MD5 for the blob. Will be saved with blob. + # * +:cache_control+ - String. Cache control for the blob. Will be saved with blob. + # * +:content_disposition+ - String. Conveys additional information about how to process the response payload, + # and also can be used to attach additional metadata + # * +:metadata+ - Hash. Custom metadata values to store with the blob. + # * +:timeout+ - Integer. A timeout in seconds. + # * +:request_id+ - String. Provides a client-generated, opaque value with a 1 KB character limit that is recorded + # in the analytics logs when storage analytics logging is enabled. + # * +:if_modified_since+ - String. A DateTime value. Specify this conditional header to create a new blob + # only if the blob has been modified since the specified date/time. If the blob has not been modified, + # the Blob service returns status code 412 (Precondition Failed). + # * +:if_unmodified_since+ - String. A DateTime value. Specify this conditional header to create a new blob + # only if the blob has not been modified since the specified date/time. If the blob has been modified, + # the Blob service returns status code 412 (Precondition Failed). + # * +:if_match+ - String. An ETag value. Specify an ETag value for this conditional header to create a new blob + # only if the blob's ETag value matches the value specified. If the values do not match, + # the Blob service returns status code 412 (Precondition Failed). + # * +:if_none_match+ - String. An ETag value. Specify an ETag value for this conditional header to create a new blob + # only if the blob's ETag value does not match the value specified. If the values are identical, + # the Blob service returns status code 412 (Precondition Failed). + # * +:lease_id+ - String. Required if the blob has an active lease. To perform this operation on a blob with an active lease, + # specify the valid lease ID for this header. + # + # See http://msdn.microsoft.com/en-us/library/azure/dd179451.aspx + # + # Returns a Blob + def create_block_blob(container, blob, content, options = {}) + size = if content.respond_to? :size + content.size + elsif options[:content_length] + options[:content_length] + else + raise ArgumentError, "Either optional parameter 'content_length' should be set or 'content' should implement 'size' method to get payload's size." + end + + threshold = get_single_upload_threshold(options[:single_upload_threshold]) + if size > threshold + create_block_blob_multiple_put(container, blob, content, size, options) + else + create_block_blob_single_put(container, blob, content, options) + end + end + + # Public: Creates a new block to be committed as part of a block blob. + # + # ==== Attributes + # + # * +container+ - String. The container name. + # * +blob+ - String. The blob name. + # * +block_id+ - String. The block id. Note: this should be the raw block id, not Base64 encoded. + # * +content+ - IO or String. The content of the blob. + # * +options+ - Hash. Optional parameters. + # + # ==== Options + # + # Accepted key/value pairs in options parameter are: + # * +:content_md5+ - String. Content MD5 for the request contents. + # * +:timeout+ - Integer. A timeout in seconds. + # * +:request_id+ - String. Provides a client-generated, opaque value with a 1 KB character limit that is recorded + # in the analytics logs when storage analytics logging is enabled. + # * +:lease_id+ - String. Required if the blob has an active lease. To perform this operation on a blob with an + # active lease, specify the valid lease ID for this header. + # + # See http://msdn.microsoft.com/en-us/library/azure/dd135726.aspx + # + # Returns response of the operation + def put_blob_block(container, blob, block_id, content, options = {}) + query = { "comp" => "block" } + StorageService.with_query query, "blockid", Base64.strict_encode64(block_id) + StorageService.with_query query, "timeout", options[:timeout].to_s if options[:timeout] + + uri = blob_uri(container, blob, query) + + headers = {} + StorageService.with_header headers, "Content-MD5", options[:content_md5] + headers["x-ms-lease-id"] = options[:lease_id] if options[:lease_id] + + response = call(:put, uri, content, headers, options) + response.headers["Content-MD5"] + end + + # Public: Commits existing blob blocks to a blob. + # + # This method writes a blob by specifying the list of block IDs that make up the + # blob. In order to be written as part of a blob, a block must have been + # successfully written to the server in a prior put_blob_block method. + # + # You can call Put Block List to update a blob by uploading only those blocks + # that have changed, then committing the new and existing blocks together. + # You can do this by specifying whether to commit a block from the committed + # block list or from the uncommitted block list, or to commit the most recently + # uploaded version of the block, whichever list it may belong to. + # + # ==== Attributes + # + # * +container+ - String. The container name. + # * +blob+ - String. The blob name. + # * +block_list+ - Array. A ordered list of lists in the following format: + # [ ["block_id1", :committed], ["block_id2", :uncommitted], ["block_id3"], ["block_id4", :committed]... ] + # The first element of the inner list is the block_id, the second is optional + # and can be either :committed or :uncommitted to indicate in which group of blocks + # the id should be looked for. If it is omitted, the latest of either group will be used. + # * +options+ - Hash. Optional parameters. + # + # ==== Options + # + # Accepted key/value pairs in options parameter are: + # * +:transactional_md5+ - String. Content MD5 for the request contents (not the blob contents!) + # * +:content_type+ - String. Content type for the blob. Will be saved with blob. + # * +:content_encoding+ - String. Content encoding for the blob. Will be saved with blob. + # * +:content_language+ - String. Content language for the blob. Will be saved with blob. + # * +:content_md5+ - String. Content MD5 for the blob. Will be saved with blob. + # * +:cache_control+ - String. Cache control for the blob. Will be saved with blob. + # * +:content_disposition+ - String. Conveys additional information about how to process the response payload, + # and also can be used to attach additional metadata + # * +:metadata+ - Hash. Custom metadata values to store with the blob. + # * +:timeout+ - Integer. A timeout in seconds. + # * +:request_id+ - String. Provides a client-generated, opaque value with a 1 KB character limit that is recorded + # in the analytics logs when storage analytics logging is enabled. + # * +:lease_id+ - String. Required if the blob has an active lease. To perform this operation on a blob with an + # active lease, specify the valid lease ID for this header. + # + # This operation also supports the use of conditional headers to commit the block list if a specified condition is met. + # For more information, see https://msdn.microsoft.com/en-us/library/azure/dd179371.aspx + # + # See http://msdn.microsoft.com/en-us/library/azure/dd179467.aspx + # + # Returns nil on success + def commit_blob_blocks(container, blob, block_list, options = {}) + query = { "comp" => "blocklist" } + StorageService.with_query query, "timeout", options[:timeout].to_s if options[:timeout] + + uri = blob_uri(container, blob, query) + + headers = {} + unless options.empty? + StorageService.with_header headers, "Content-MD5", options[:transactional_md5] + StorageService.with_header headers, "x-ms-blob-content-type", options[:content_type] + StorageService.with_header headers, "x-ms-blob-content-encoding", options[:content_encoding] + StorageService.with_header headers, "x-ms-blob-content-language", options[:content_language] + StorageService.with_header headers, "x-ms-blob-content-md5", options[:content_md5] + StorageService.with_header headers, "x-ms-blob-cache-control", options[:cache_control] + StorageService.with_header headers, "x-ms-blob-content-disposition", options[:content_disposition] + + StorageService.add_metadata_to_headers(options[:metadata], headers) + add_blob_conditional_headers(options, headers) + headers["x-ms-lease-id"] = options[:lease_id] if options[:lease_id] + end + headers["x-ms-blob-content-type"] = Default::CONTENT_TYPE_VALUE unless headers["x-ms-blob-content-type"] + body = Serialization.block_list_to_xml(block_list) + call(:put, uri, body, headers, options) + nil + end + + # Public: Retrieves the list of blocks that have been uploaded as part of a block blob. + # + # There are two block lists maintained for a blob: + # 1) Committed Block List: The list of blocks that have been successfully + # committed to a given blob with commitBlobBlocks. + # 2) Uncommitted Block List: The list of blocks that have been uploaded for a + # blob using Put Block (REST API), but that have not yet been committed. + # These blocks are stored in Microsoft Azure in association with a blob, but do + # not yet form part of the blob. + # + # ==== Attributes + # + # * +container+ - String. The container name. + # * +blob+ - String. The blob name. + # * +options+ - Hash. Optional parameters. + # + # ==== Options + # + # Accepted key/value pairs in options parameter are: + # * +:blocklist_type+ - Symbol. One of :all, :committed, :uncommitted. Defaults to :all (optional) + # * +:snapshot+ - String. An opaque DateTime value that specifies the blob snapshot to + # retrieve information from. (optional) + # * +:timeout+ - Integer. A timeout in seconds. + # * +:request_id+ - String. Provides a client-generated, opaque value with a 1 KB character limit that is recorded + # in the analytics logs when storage analytics logging is enabled. + # * +:location_mode+ - LocationMode. Specifies the location mode used to decide + # which location the request should be sent to. + # * +:lease_id+ - String. If this header is specified, the operation will be performed only if both of the + # following conditions are met: + # - The blob's lease is currently active. + # - The lease ID specified in the request matches that of the blob. + # If this header is specified and both of these conditions are not met, the request will fail + # and the operation will fail with status code 412 (Precondition Failed). + # + # See http://msdn.microsoft.com/en-us/library/azure/dd179400.aspx + # + # Returns a list of Azure::Storage::Entity::Blob::Block instances + def list_blob_blocks(container, blob, options = {}) + options[:blocklist_type] = options[:blocklist_type] || :all + + query = { "comp" => "blocklist" } + StorageService.with_query query, "snapshot", options[:snapshot] + StorageService.with_query query, "blocklisttype", options[:blocklist_type].to_s if options[:blocklist_type] + StorageService.with_query query, "timeout", options[:timeout].to_s if options[:timeout] + + headers = options[:lease_id] ? { "x-ms-lease-id" => options[:lease_id] } : {} + + options[:request_location_mode] = Azure::Storage::Common::RequestLocationMode::PRIMARY_OR_SECONDARY + uri = blob_uri(container, blob, query, options) + + response = call(:get, uri, nil, headers, options) + + Serialization.block_list_from_xml(response.body) + end + + # Public: Creates a new block blob or updates the content of an existing block blob. + # + # Updating an existing block blob overwrites any existing metadata on the blob + # Partial updates are not supported with create_block_blob the content of the + # existing blob is overwritten with the content of the new blob. To perform a + # partial update of the content of a block blob, use the create_block_list + # method. + # + # Note that the default content type is application/octet-stream. + # + # ==== Attributes + # + # * +container+ - String. The container name. + # * +blob+ - String. The blob name. + # * +content+ - IO or String. The content of the blob. + # * +options+ - Hash. Optional parameters. + # + # ==== Options + # + # Accepted key/value pairs in options parameter are: + # * +:transactional_md5+ - String. An MD5 hash of the blob content. This hash is used to verify the integrity of the blob during transport. + # When this header is specified, the storage service checks the hash that has arrived with the one that was sent. + # If the two hashes do not match, the operation will fail with error code 400 (Bad Request). + # * +:single_upload_threshold+ - Integer. Threshold in bytes for single upload, must be lower than 256MB or 256MB will be used. + # * +:content_length+ - Integer. Length of the content to upload, must be specified if 'content' does not implement 'size'. + # * +:content_type+ - String. Content type for the blob. Will be saved with blob. + # * +:content_encoding+ - String. Content encoding for the blob. Will be saved with blob. + # * +:content_language+ - String. Content language for the blob. Will be saved with blob. + # * +:content_md5+ - String. Content MD5 for the blob. Will be saved with blob. + # * +:cache_control+ - String. Cache control for the blob. Will be saved with blob. + # * +:content_disposition+ - String. Conveys additional information about how to process the response payload, + # and also can be used to attach additional metadata + # * +:metadata+ - Hash. Custom metadata values to store with the blob. + # * +:timeout+ - Integer. A timeout in seconds. + # * +:request_id+ - String. Provides a client-generated, opaque value with a 1 KB character limit that is recorded + # in the analytics logs when storage analytics logging is enabled. + # * +:if_modified_since+ - String. A DateTime value. Specify this conditional header to create a new blob + # only if the blob has been modified since the specified date/time. If the blob has not been modified, + # the Blob service returns status code 412 (Precondition Failed). + # * +:if_unmodified_since+ - String. A DateTime value. Specify this conditional header to create a new blob + # only if the blob has not been modified since the specified date/time. If the blob has been modified, + # the Blob service returns status code 412 (Precondition Failed). + # * +:if_match+ - String. An ETag value. Specify an ETag value for this conditional header to create a new blob + # only if the blob's ETag value matches the value specified. If the values do not match, + # the Blob service returns status code 412 (Precondition Failed). + # * +:if_none_match+ - String. An ETag value. Specify an ETag value for this conditional header to create a new blob + # only if the blob's ETag value does not match the value specified. If the values are identical, + # the Blob service returns status code 412 (Precondition Failed). + # * +:lease_id+ - String. Required if the blob has an active lease. To perform this operation on a blob with an active lease, + # specify the valid lease ID for this header. + # + # See http://msdn.microsoft.com/en-us/library/azure/dd179451.aspx + # + # Returns a Blob + alias create_block_blob_from_content create_block_blob + + # Protected: Creates a new block blob or updates the content of an existing block blob with single API call + # + # Updating an existing block blob overwrites any existing metadata on the blob + # Partial updates are not supported with create_block_blob the content of the + # existing blob is overwritten with the content of the new blob. To perform a + # partial update of the content of a block blob, use the create_block_list + # method. + # + # Note that the default content type is application/octet-stream. + # + # ==== Attributes + # + # * +container+ - String. The container name. + # * +blob+ - String. The blob name. + # * +content+ - IO or String. The content of the blob. + # * +options+ - Hash. Optional parameters. + # + # ==== Options + # + # Accepted key/value pairs in options parameter are: + # * +:transactional_md5+ - String. An MD5 hash of the blob content. This hash is used to verify the integrity of the blob during transport. + # When this header is specified, the storage service checks the hash that has arrived with the one that was sent. + # If the two hashes do not match, the operation will fail with error code 400 (Bad Request). + # * +:content_length+ - Integer. Length of the content to upload, must be specified if 'content' does not implement 'size'. + # * +:content_type+ - String. Content type for the blob. Will be saved with blob. + # * +:content_encoding+ - String. Content encoding for the blob. Will be saved with blob. + # * +:content_language+ - String. Content language for the blob. Will be saved with blob. + # * +:content_md5+ - String. Content MD5 for the blob. Will be saved with blob. + # * +:cache_control+ - String. Cache control for the blob. Will be saved with blob. + # * +:content_disposition+ - String. Conveys additional information about how to process the response payload, + # and also can be used to attach additional metadata + # * +:metadata+ - Hash. Custom metadata values to store with the blob. + # * +:timeout+ - Integer. A timeout in seconds. + # * +:request_id+ - String. Provides a client-generated, opaque value with a 1 KB character limit that is recorded + # in the analytics logs when storage analytics logging is enabled. + # * +:if_modified_since+ - String. A DateTime value. Specify this conditional header to create a new blob + # only if the blob has been modified since the specified date/time. If the blob has not been modified, + # the Blob service returns status code 412 (Precondition Failed). + # * +:if_unmodified_since+ - String. A DateTime value. Specify this conditional header to create a new blob + # only if the blob has not been modified since the specified date/time. If the blob has been modified, + # the Blob service returns status code 412 (Precondition Failed). + # * +:if_match+ - String. An ETag value. Specify an ETag value for this conditional header to create a new blob + # only if the blob's ETag value matches the value specified. If the values do not match, + # the Blob service returns status code 412 (Precondition Failed). + # * +:if_none_match+ - String. An ETag value. Specify an ETag value for this conditional header to create a new blob + # only if the blob's ETag value does not match the value specified. If the values are identical, + # the Blob service returns status code 412 (Precondition Failed). + # * +:lease_id+ - String. Required if the blob has an active lease. To perform this operation on a blob with an active lease, + # specify the valid lease ID for this header. + # + # See http://msdn.microsoft.com/en-us/library/azure/dd179451.aspx + # + # Returns a Blob + protected + def create_block_blob_single_put(container, blob, content, options = {}) + query = {} + StorageService.with_query query, "timeout", options[:timeout].to_s if options[:timeout] + + uri = blob_uri(container, blob, query) + + headers = {} + + # set x-ms-blob-type to BlockBlob + StorageService.with_header headers, "x-ms-blob-type", "BlockBlob" + + # set the rest of the optional headers + StorageService.with_header headers, "Content-MD5", options[:transactional_md5] + StorageService.with_header headers, "Content-Length", options[:content_length] + StorageService.with_header headers, "x-ms-blob-content-encoding", options[:content_encoding] + StorageService.with_header headers, "x-ms-blob-content-language", options[:content_language] + StorageService.with_header headers, "x-ms-blob-content-md5", options[:content_md5] + StorageService.with_header headers, "x-ms-blob-cache-control", options[:cache_control] + StorageService.with_header headers, "x-ms-blob-content-disposition", options[:content_disposition] + StorageService.with_header headers, "x-ms-lease-id", options[:lease_id] + + StorageService.add_metadata_to_headers options[:metadata], headers + add_blob_conditional_headers options, headers + headers["x-ms-blob-content-type"] = get_or_apply_content_type(content, options[:content_type]) + # call PutBlob + response = call(:put, uri, content, headers, options) + + result = Serialization.blob_from_headers(response.headers) + result.name = blob + result.metadata = options[:metadata] if options[:metadata] + + result + end + + # Protected: Creates a new block blob or updates the content of an existing block blob with multiple upload + # + # Updating an existing block blob overwrites any existing metadata on the blob + # Partial updates are not supported with create_block_blob the content of the + # existing blob is overwritten with the content of the new blob. To perform a + # partial update of the content of a block blob, use the create_block_list + # method. + # + # Note that the default content type is application/octet-stream. + # + # ==== Attributes + # + # * +container+ - String. The container name. + # * +blob+ - String. The blob name. + # * +content+ - IO or String. The content of the blob. + # * +size+ - Integer. The size of the content. + # * +options+ - Hash. Optional parameters. + # + # ==== Options + # + # Accepted key/value pairs in options parameter are: + # * +:content_type+ - String. Content type for the blob. Will be saved with blob. + # * +:content_encoding+ - String. Content encoding for the blob. Will be saved with blob. + # * +:content_language+ - String. Content language for the blob. Will be saved with blob. + # * +:content_md5+ - String. Content MD5 for the blob. Will be saved with blob. + # * +:cache_control+ - String. Cache control for the blob. Will be saved with blob. + # * +:content_disposition+ - String. Conveys additional information about how to process the response payload, + # and also can be used to attach additional metadata + # * +:metadata+ - Hash. Custom metadata values to store with the blob. + # * +:timeout+ - Integer. A timeout in seconds. + # * +:request_id+ - String. Provides a client-generated, opaque value with a 1 KB character limit that is recorded + # in the analytics logs when storage analytics logging is enabled. + # * +:lease_id+ - String. Required if the blob has an active lease. To perform this operation on a blob with an active lease, + # specify the valid lease ID for this header. + # + # See http://msdn.microsoft.com/en-us/library/azure/dd179451.aspx + # + # Returns a Blob + protected + def create_block_blob_multiple_put(container, blob, content, size, options = {}) + content_type = get_or_apply_content_type(content, options[:content_type]) + content = StringIO.new(content) if content.is_a? String + block_size = get_block_size(size) + # Get the number of blocks + block_count = (Float(size) / Float(block_size)).ceil + block_list = [] + for block_id in 0...block_count + id = block_id.to_s.rjust(6, "0") + put_blob_block(container, blob, id, content.read(block_size), timeout: options[:timeout], lease_id: options[:lease_id]) + block_list.push([id]) + end + + # Commit the blocks put + commit_options = {} + commit_options[:content_type] = content_type + commit_options[:content_encoding] = options[:content_encoding] if options[:content_encoding] + commit_options[:content_language] = options[:content_language] if options[:content_language] + commit_options[:content_md5] = options[:content_md5] if options[:content_md5] + commit_options[:cache_control] = options[:cache_control] if options[:cache_control] + commit_options[:content_disposition] = options[:content_disposition] if options[:content_disposition] + commit_options[:metadata] = options[:metadata] if options[:metadata] + commit_options[:timeout] = options[:timeout] if options[:timeout] + commit_options[:request_id] = options[:request_id] if options[:request_id] + commit_options[:lease_id] = options[:lease_id] if options[:lease_id] + + commit_blob_blocks(container, blob, block_list, commit_options) + + get_properties_options = {} + get_properties_options[:lease_id] = options[:lease_id] if options[:lease_id] + + # Get the blob properties + get_blob_properties(container, blob, get_properties_options) + end + + # Protected: Gets the single upload threshold according to user's preference + # + # ==== Attributes + # + # * +container+ - String. The container name. + # + # Returns an Integer + protected + def get_single_upload_threshold(userThreshold) + if userThreshold.nil? + BlobConstants::DEFAULT_SINGLE_BLOB_PUT_THRESHOLD_IN_BYTES + elsif userThreshold <= 0 + raise ArgumentError, "Single Upload Threshold should be positive number" + elsif userThreshold < BlobConstants::MAX_SINGLE_UPLOAD_BLOB_SIZE_IN_BYTES + userThreshold + else + BlobConstants::MAX_SINGLE_UPLOAD_BLOB_SIZE_IN_BYTES + end + end + + protected + def get_block_size(size) + if size > BlobConstants::MAX_BLOCK_BLOB_SIZE + raise ArgumentError, "Block blob size should be less than #{BlobConstants::MAX_BLOCK_BLOB_SIZE} bytes in size" + elsif (size / BlobConstants::MAX_BLOCK_COUNT) < BlobConstants::DEFAULT_WRITE_BLOCK_SIZE_IN_BYTES + BlobConstants::DEFAULT_WRITE_BLOCK_SIZE_IN_BYTES + else + BlobConstants::MAX_BLOCK_SIZE + end + end + end +end