net_fetcher.rb in omnibus-5.1.0

- old
+ new

@@ -22,12 +22,15 @@
   class NetFetcher < Fetcher
     # Use 7-zip to extract 7z/zip for Windows
     WIN_7Z_EXTENSIONS = %w(.7z .zip)
 
     # tar probably has compression scheme linked in, otherwise for tarballs
-    TAR_EXTENSIONS = %w(.tar .tar.gz .tgz .bz2 .tar.xz .txz)
+    COMPRESSED_TAR_EXTENSIONS = %w(.tar.gz .tgz tar.bz2 .tar.xz .txz .tar.lzma)
+    TAR_EXTENSIONS = COMPRESSED_TAR_EXTENSIONS + ['.tar']
 
+    ALL_EXTENSIONS = WIN_7Z_EXTENSIONS + TAR_EXTENSIONS
+
     # Digest types used for verifying file checksums
     DIGESTS = [:sha512, :sha256, :sha1, :md5]
 
     #
     # A fetch is required if the downloaded_file (such as a tarball) does not
@@ -49,25 +52,25 @@
     def version_guid
       "#{digest_type}:#{checksum}"
     end
 
     #
-    # Clean the project directory by removing the contents from disk.
+    # Clean the project directory if it exists and actually extract
+    # the downloaded file.
     #
     # @return [true, false]
     #   true if the project directory was removed, false otherwise
     #
     def clean
-      if File.exist?(project_dir)
+      needs_cleaning = File.exist?(project_dir)
+      if needs_cleaning
         log.info(log_key) { "Cleaning project directory `#{project_dir}'" }
         FileUtils.rm_rf(project_dir)
-        extract
-        true
-      else
-        extract
-        false
       end
+      create_required_directories
+      deploy
+      needs_cleaning
     end
 
     #
     # Fetch the given software definition. This method **always** fetches the
     # file, even if it already exists on disk! You should use {#fetch_required?}
@@ -79,17 +82,19 @@
       log.info(log_key) { "Downloading from `#{download_url}'" }
 
       create_required_directories
       download
       verify_checksum!
-      extract
     end
 
     #
     # The version for this item in the cache. This is the digest of downloaded
     # file and the URL where it was downloaded from.
     #
+    # This method is called *before* clean but *after* fetch. Do not ever
+    # use the contents of the project_dir here.
+    #
     # @return [String]
     #
     def version_for_cache
       "download_url:#{source[:url]}|#{digest_type}:#{checksum}"
     end
@@ -180,20 +185,23 @@
         downloaded_amount = [step, reported_total].min
         progress_bar.progress = downloaded_amount
       }
 
       file = open(download_url, options)
-      FileUtils.cp(file.path, downloaded_file)
+      # This is a temporary file. Close and flush it before attempting to copy
+      # it over.
       file.close
+      FileUtils.cp(file.path, downloaded_file)
+      file.unlink
     rescue SocketError,
            Errno::ECONNREFUSED,
            Errno::ECONNRESET,
            Errno::ENETUNREACH,
            Timeout::Error,
            OpenURI::HTTPError => e
       if fetcher_retries != 0
-        log.debug(log_key) { "Retrying failed download (#{fetcher_retries})..." }
+        log.info(log_key) { "Retrying failed download due to #{e} (#{fetcher_retries} retries left)..." }
         fetcher_retries -= 1
         retry
       else
         log.error(log_key) { "Download failed - #{e.class}!" }
         raise
@@ -203,32 +211,99 @@
     #
     # Extract the downloaded file, using the magical logic based off of the
     # ending file extension. In the rare event the file cannot be extracted, it
     # is copied over as a raw file.
     #
-    def extract
-      if command = extract_command
-        log.info(log_key) { "Extracting `#{downloaded_file}' to `#{Config.source_dir}'" }
-        shellout!(command)
+    def deploy
+      if downloaded_file.end_with?(*ALL_EXTENSIONS)
+        log.info(log_key) { "Extracting `#{safe_downloaded_file}' to `#{safe_project_dir}'" }
+        extract
       else
-        log.info(log_key) { "`#{downloaded_file}' is not an archive - copying to `#{project_dir}'" }
+        log.info(log_key) { "`#{safe_downloaded_file}' is not an archive - copying to `#{safe_project_dir}'" }
 
-        if File.directory?(project_dir)
+        if File.directory?(downloaded_file)
           # If the file itself was a directory, copy the whole thing over. This
           # seems unlikely, because I do not think it is a possible to download
           # a folder, but better safe than sorry.
-          FileUtils.cp_r(downloaded_file, project_dir)
+          FileUtils.cp_r("#{downloaded_file}/.", project_dir)
         else
           # In the more likely case that we got a "regular" file, we want that
-          # file to live **inside** the project directory.
-          FileUtils.mkdir_p(project_dir)
-          FileUtils.cp(downloaded_file, "#{project_dir}/")
+          # file to live **inside** the project directory. project_dir should already
+          # exist due to create_required_directories
+          FileUtils.cp(downloaded_file, project_dir)
         end
       end
     end
 
     #
+    # Extracts the downloaded archive file into project_dir.
+    #
+    # On windows, this is a fuster cluck and we allow users to specify the
+    # preferred extractor to be used. The default is to use tar. User overrides
+    # can be set in source[:extract] as:
+    #   :tar - use tar.exe and fail on errors (default strategy).
+    #   :seven_zip - use 7zip for all tar/compressed tar files on windows.
+    #   :lax_tar - use tar.exe on windows but ignore errors.
+    #
+    # Both 7z and bsdtar have issues on windows.
+    #
+    # 7z cannot extract and untar at the same time. You need to extract to a
+    # temporary location and then extract again into project_dir.
+    #
+    # 7z also doesn't handle symlinks well. A symlink to a non-existent
+    # location simply results in a text file with the target path written in
+    # it. It does this without throwing any errors.
+    #
+    # bsdtar will exit(1) if it is encounters symlinks on windows. So we can't
+    # use shellout! directly.
+    #
+    # bsdtar will also exit(1) and fail to overwrite files at the destination
+    # during extraction if a file already exists at the destination and is
+    # marked read-only. This used to be a problem when we weren't properly
+    # cleaning an existing project_dir. It should be less of a problem now...
+    # but who knows.
+    #
+    def extract
+      # Only used by tar
+      compression_switch = ''
+      compression_switch = 'z'        if downloaded_file.end_with?('gz')
+      compression_switch = '--lzma -' if downloaded_file.end_with?('lzma')
+      compression_switch = 'j'        if downloaded_file.end_with?('bz2')
+      compression_switch = 'J'        if downloaded_file.end_with?('xz')
+
+      if Ohai['platform'] == 'windows'
+        if downloaded_file.end_with?(*TAR_EXTENSIONS) && source[:extract] != :seven_zip
+          returns = [0]
+          returns << 1 if source[:extract] == :lax_tar
+
+          shellout!("tar.exe #{compression_switch}xf #{safe_downloaded_file} -C#{safe_project_dir}", returns: returns)
+        elsif downloaded_file.end_with?(*COMPRESSED_TAR_EXTENSIONS)
+          Dir.mktmpdir do |temp_dir|
+            log.debug(log_key) { "Temporarily extracting `#{safe_downloaded_file}' to `#{temp_dir}'" }
+
+            shellout!("7z.exe x #{safe_downloaded_file} -o#{windows_safe_path(temp_dir)} -r -y")
+
+            fname = File.basename(downloaded_file, File.extname(downloaded_file))
+            fname << ".tar" if downloaded_file.end_with?('tgz', 'txz')
+            next_file = windows_safe_path(File.join(temp_dir, fname))
+
+            log.debug(log_key) { "Temporarily extracting `#{next_file}' to `#{safe_project_dir}'" }
+            shellout!("7z.exe x #{next_file} -o#{safe_project_dir} -r -y")
+          end
+        else
+          shellout!("7z.exe x #{safe_downloaded_file} -o#{safe_project_dir} -r -y")
+        end
+      elsif downloaded_file.end_with?('.7z')
+        shellout!("7z x #{safe_downloaded_file} -o#{safe_project_dir} -r -y")
+      elsif downloaded_file.end_with?('.zip')
+        shellout!("unzip #{safe_downloaded_file} -d #{safe_project_dir}")
+      else
+        shellout!("#{tar} #{compression_switch}xf #{safe_downloaded_file} -C#{safe_project_dir}")
+      end
+    end
+
+    #
     # The digest type defined in the software definition
     #
     # @raise [ChecksumMissing]
     #   if the checksum does not exist
     #
@@ -240,11 +315,11 @@
       end
       raise ChecksumMissing.new(self)
     end
 
     #
-    # Verify the downloaded file has the correct checksum.#
+    # Verify the downloaded file has the correct checksum.
     #
     # @raise [ChecksumMismatch]
     #   if the checksum does not match
     #
     def verify_checksum!
@@ -256,29 +331,23 @@
       if expected != actual
         raise ChecksumMismatch.new(self, expected, actual)
       end
     end
 
+    def safe_project_dir
+      windows_safe_path(project_dir)
+    end
+
+    def safe_downloaded_file
+      windows_safe_path(downloaded_file)
+    end
+
     #
     # The command to use for extracting this piece of software.
     #
-    # @return [String, nil]
+    # @return [[String]]
     #
     def extract_command
-      if Ohai['platform'] == 'windows' && downloaded_file.end_with?(*WIN_7Z_EXTENSIONS)
-        "7z.exe x #{windows_safe_path(downloaded_file)} -o#{Config.source_dir} -r -y"
-      elsif Ohai['platform'] != 'windows' && downloaded_file.end_with?('.7z')
-        "7z x #{windows_safe_path(downloaded_file)} -o#{Config.source_dir} -r -y"
-      elsif Ohai['platform'] != 'windows' && downloaded_file.end_with?('.zip')
-        "unzip #{windows_safe_path(downloaded_file)} -d #{Config.source_dir}"
-      elsif downloaded_file.end_with?(*TAR_EXTENSIONS)
-        compression_switch = 'z' if downloaded_file.end_with?('gz')
-        compression_switch = 'j' if downloaded_file.end_with?('bz2')
-        compression_switch = 'J' if downloaded_file.end_with?('xz')
-        compression_switch = ''  if downloaded_file.end_with?('tar')
-
-        "#{tar} #{compression_switch}xf #{windows_safe_path(downloaded_file)} -C#{Config.source_dir}"
-      end
     end
 
     #
     # Primitively determine whether we should use gtar or tar to untar a file.
     # If gtar is present, we will use gtar (AIX). Otherwise, we fallback to tar.