lib/u3d/downloader.rb in u3d-0.9.3 vs lib/u3d/downloader.rb in u3d-0.9.4
- old
+ new
@@ -19,48 +19,117 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
## --- END LICENSE BLOCK ---
require 'net/http'
-require 'u3d/iniparser'
require 'u3d/utils'
+require 'u3d/download_validator'
module U3d
# Take care of downloading files and packages
+ # rubocop:disable ModuleLength
module Downloader
# Name of the directory for the package downloading
DOWNLOAD_DIRECTORY = 'Unity_Packages'.freeze
# Path to the directory for the package downloading
DOWNLOAD_PATH = "#{ENV['HOME']}/Downloads".freeze
# Regex to get the name of a package out of its file name
UNITY_MODULE_FILE_REGEX = %r{\/([\w\-_\.\+]+\.(?:pkg|exe|zip|sh|deb))}
class << self
- def hash_validation(expected: nil, actual: nil)
- if expected
- if expected != actual
- UI.verbose "Expected hash is #{expected}, file hash is #{actual}"
- UI.important 'File looks corrupted (wrong hash)'
- return false
+ # fetch modules and put them in local cache
+ def fetch_modules(definition, packages: [], download: nil)
+ if download
+ download_modules(definition, packages: packages)
+ else
+ local_files(definition, packages: packages)
+ end
+ end
+
+ # download packages
+ def download_modules(definition, packages: [])
+ files = []
+ validator, downloader = setup_os(definition.os)
+
+ packages.each do |package|
+ get_package(downloader, validator, package, definition, files)
+ end
+
+ # On Linux, make sure the files are executable
+ # FIXME: Move me to the LinuxInstaller
+ files.each { |f| U3dCore::CommandExecutor.execute(command: "chmod a+x #{f[1]}") } if definition.os == :linux
+
+ files
+ end
+
+ # find already downloaded packages
+ def local_files(definition, packages: [])
+ files = []
+ validator, downloader = setup_os(definition.os)
+
+ packages.each do |package|
+ path = downloader.destination_for(package, definition)
+ if File.file?(path)
+ if validator.validate(package, path, definition)
+ files << [package, path, definition[package]]
+ else
+ UI.important "File present at #{path} is not correct, will not be used. Skipping #{package}"
+ end
+ else
+ UI.error "No file has been downloaded for #{package}, or it has been moved from #{path}"
end
+ end
+
+ files
+ end
+
+ private #-----------------------------------------------------------------
+
+ def setup_os(os)
+ case os
+ when :linux
+ validator = LinuxValidator.new
+ downloader = Downloader::LinuxDownloader.new
+ when :mac
+ validator = MacValidator.new
+ downloader = Downloader::MacDownloader.new
+ when :win
+ validator = WindowsValidator.new
+ downloader = Downloader::WindowsDownloader.new
else
- UI.verbose 'No hash validation available. File is assumed correct but may not be.'
+ raise ArgumentError, "Operating system #{os.id2name} is not recognized"
end
- true
+ return validator, downloader
end
- def size_validation(expected: nil, actual: nil)
- if expected
- if expected != actual
- UI.verbose "Expected size is #{expected}, file size is #{actual}"
- UI.important 'File looks corrupted (wrong size)'
- return false
+ def get_package(downloader, validator, package, definition, files)
+ path = downloader.destination_for(package, definition)
+ url = downloader.url_for(package, definition)
+ if File.file?(path)
+ UI.verbose "Installer file for #{package} seems to be present at #{path}"
+ if validator.validate(package, path, definition)
+ UI.message "#{package.capitalize} is already downloaded"
+ files << [package, path, definition[package]]
+ return
+ else
+ extension = File.extname(path)
+ new_path = File.join(File.dirname(path), File.basename(path, extension) + '_CORRUPTED' + extension)
+ UI.important "File present at #{path} is not correct, it has been renamed to #{new_path}"
+ File.rename(path, new_path)
end
+ end
+
+ UI.header "Downloading #{package} version #{definition.version}"
+ UI.message 'Downloading from ' + url.to_s.cyan.underline
+ download_package(path, url, size: definition.size_in_kb(package))
+
+ if validator.validate(package, path, definition)
+ UI.success "Successfully downloaded #{package}."
+ files << [package, path, definition[package]]
else
- UI.verbose 'No size validation available. File is assumed correct but may not be.'
+ UI.error "Failed to download #{package}"
end
- true
end
def download_package(path, url, size: nil)
File.open(path, 'wb') do |f|
uri = URI(url)
@@ -77,11 +146,12 @@
started_at = Time.now.to_i - 1
response.read_body do |segment|
f.write(segment)
current += segment.length
# wait for Net::HTTP buffer on slow networks
- sleep 0.08 # adjust to reduce CPU
+ # FIXME revisits, this slows down download on fast network
+ # sleep 0.08 # adjust to reduce CPU
next unless UI.interactive?
next unless Time.now.to_f - last_print_update > 0.5
last_print_update = Time.now.to_f
if size
Utils.print_progress(current, size, started_at)
@@ -99,270 +169,48 @@
raise e
end
end
class MacDownloader
- class << self
- # Downloads all packages available for given version
- def download_all(version, cached_versions)
- if cached_versions[version].nil?
- UI.error "No version #{version} was found in cache. It might need updating."
- return nil
- end
- files = []
- ini_file = INIparser.load_ini(version, cached_versions)
- ini_file.keys.each do |k|
- result = download_specific(k, version, cached_versions)
- files << [k, result[0], result[1]] unless result.nil?
- end
- files
- end
+ def destination_for(package, definition)
+ dir = File.join(DOWNLOAD_PATH, DOWNLOAD_DIRECTORY, definition.version)
+ Utils.ensure_dir(dir)
+ file_name = UNITY_MODULE_FILE_REGEX.match(definition[package]['url'])[1]
- # Downloads a specific package for given version
- def download_specific(package, version, cached_versions)
- if cached_versions[version].nil?
- UI.error "No version #{version} was found in cache. It might need updating."
- return nil
- end
+ File.expand_path(file_name, dir)
+ end
- ini_file = INIparser.load_ini(version, cached_versions)
- if ini_file[package].empty?
- UI.error "No package \"#{package}\" was found for version #{version}."
- return nil
- end
-
- url = cached_versions[version]
- dir = File.join(DOWNLOAD_PATH, DOWNLOAD_DIRECTORY, version)
- Utils.ensure_dir(dir)
- return [get_package(package, ini_file, dir, url), ini_file[package]]
- end
-
- private #---------------------------------------------------------------
-
- def get_package(name, ini_file, main_dir, base_url)
- file_name = UNITY_MODULE_FILE_REGEX.match(ini_file[name]['url'])[1]
- file_path = File.expand_path(file_name, main_dir)
-
- # Check if file already exists and validate it
- if File.file?(file_path)
- if Downloader.size_validation(expected: ini_file[name]['size'], actual: File.size(file_path)) &&
- Downloader.hash_validation(expected: ini_file[name]['md5'], actual: Utils.hashfile(file_path))
- UI.important "#{name.capitalize} already downloaded at #{file_path}"
- return file_path
- else
- UI.verbose "Deleting existing file at #{file_path}"
- File.delete(file_path)
- end
- end
-
- # Download file
- url = base_url + ini_file[name]['url']
- UI.header "Downloading #{name}"
- UI.verbose 'Downloading from ' + url.to_s.cyan.underline
- Downloader.download_package(file_path, url, size: ini_file[name]['size'])
-
- # Validation download
- if Downloader.size_validation(expected: ini_file[name]['size'], actual: File.size(file_path)) &&
- Downloader.hash_validation(expected: ini_file[name]['md5'], actual: Utils.hashfile(file_path))
- UI.success "Successfully downloaded #{name}."
- else
- File.delete(file_path)
- raise 'Download failed: file is corrupted, deleting it.'
- end
-
- file_path
- end
-
- def all_local_files(version)
- files = []
- ini_file = INIparser.load_ini(version, {}, offline: true)
- ini_file.keys.each do |k|
- result = local_file(k, version)
- files << [k, result[0], result[1]] unless result.nil?
- end
- files
- end
-
- def local_file(package, version)
- ini_file = INIparser.load_ini(version, {}, offline: true)
- if ini_file[package].empty?
- UI.error "No package \"#{package}\" was found for version #{version}."
- return nil
- end
-
- dir = File.join(DOWNLOAD_PATH, DOWNLOAD_DIRECTORY, version)
- raise "Main directory #{dir} does not exist. Nothing has been downloaded for version #{version}" unless Dir.exist?(dir)
-
- file_name = UNITY_MODULE_FILE_REGEX.match(ini_file[package]['url'])[1]
- file_path = File.expand_path(file_name, dir)
-
- unless File.file?(file_path)
- UI.error "Package #{package} has not been downloaded"
- return nil
- end
-
- unless Downloader.size_validation(expected: ini_file[package]['size'], actual: File.size(file_path)) &&
- Downloader.hash_validation(expected: ini_file[package]['md5'], actual: Utils.hashfile(file_path))
- UI.error "File at #{file_path} is corrupted, deleting it"
- File.delete(file_path)
- return nil
- end
-
- return [file_path, ini_file[package]]
- end
+ def url_for(package, definition)
+ definition.url + definition[package]['url']
end
end
class LinuxDownloader
- class << self
- def download(version, cached_versions)
- if cached_versions[version].nil?
- UI.error "No version #{version} was found in cache. It might need updating."
- return nil
- end
- url = cached_versions[version]
- dir = File.join(DOWNLOAD_PATH, DOWNLOAD_DIRECTORY, version)
- Utils.ensure_dir(dir)
- file_name = UNITY_MODULE_FILE_REGEX.match(url)[1]
- file_path = File.expand_path(file_name, dir)
+ def destination_for(package, definition)
+ dir = File.join(DOWNLOAD_PATH, DOWNLOAD_DIRECTORY, definition.version)
+ Utils.ensure_dir(dir)
+ file_name = UNITY_MODULE_FILE_REGEX.match(definition[package]['url'])[1]
- # Check if file already exists
- # Note: without size or hash validation, the file is assumed to be correct
- if File.file?(file_path)
- UI.important "File already downloaded at #{file_path}"
- return file_path
- end
+ File.expand_path(file_name, dir)
+ end
- # Download file
- UI.header "Downloading Unity #{version}"
- UI.verbose 'Downloading from ' + url.to_s.cyan.underline
- Downloader.download_package(file_path, url)
- U3dCore::CommandExecutor.execute(command: "chmod a+x #{file_path}")
- file_path
- end
-
- def local_file(version)
- dir = File.join(DOWNLOAD_PATH, DOWNLOAD_DIRECTORY, version)
- raise "Main directory #{dir} does not exist. Nothing has been downloaded for version #{version}" unless Dir.exist?(dir)
- find_cmd = "find #{dir}/ -maxdepth 2 -name '*.sh'"
- files = U3dCore::CommandExecutor.execute(command: find_cmd).split("\n")
- return files[0] unless files.empty?
- raise 'No file has been downloaded'
- end
+ def url_for(_package, definition)
+ definition.url
end
end
class WindowsDownloader
- class << self
- def download_all(version, cached_versions)
- if cached_versions[version].nil?
- UI.error "No version #{version} was found in cache. It might need updating."
- return nil
- end
- files = []
- ini_file = INIparser.load_ini(version, cached_versions)
- ini_file.keys.each do |k|
- result = download_specific(k, version, cached_versions)
- files << [k, result[0], result[1]] unless result.nil?
- end
- files
- end
+ def destination_for(package, definition)
+ dir = File.join(DOWNLOAD_PATH, DOWNLOAD_DIRECTORY, definition.version)
+ Utils.ensure_dir(dir)
+ file_name = UNITY_MODULE_FILE_REGEX.match(definition[package]['url'])[1]
- # Downloads a specific package for given version
- def download_specific(package, version, cached_versions)
- if cached_versions[version].nil?
- UI.error "No version #{version} was found in cache. It might need updating."
- return nil
- end
+ File.expand_path(file_name, dir)
+ end
- ini_file = INIparser.load_ini(version, cached_versions)
- if ini_file[package].empty?
- UI.error "No package \"#{package}\" was found for version #{version}."
- return nil
- end
-
- url = cached_versions[version]
- dir = File.join(DOWNLOAD_PATH, DOWNLOAD_DIRECTORY, version)
- Utils.ensure_dir(dir)
- return [get_package(package, ini_file, dir, url), ini_file[package]]
- end
-
- def all_local_files(version)
- files = []
- ini_file = INIparser.load_ini(version, {}, offline: true)
- ini_file.keys.each do |k|
- result = local_file(k, version)
- files << [k, result[0], result[1]] unless result.nil?
- end
- files
- end
-
- def local_file(package, version)
- ini_file = INIparser.load_ini(version, {}, offline: true)
- if ini_file[package].empty?
- UI.error "No package \"#{package}\" was found for version #{version}."
- return nil
- end
-
- dir = File.join(DOWNLOAD_PATH, DOWNLOAD_DIRECTORY, version)
- raise "Main directory #{dir} does not exist. Nothing has been downloaded for version #{version}" unless Dir.exist?(dir)
-
- file_name = UNITY_MODULE_FILE_REGEX.match(ini_file[package]['url'])[1]
- file_path = File.expand_path(file_name, dir)
-
- unless File.file?(file_path)
- UI.error "Package #{package} has not been downloaded"
- return nil
- end
-
- rounded_size = (File.size(file_path).to_f / 1024).floor
- unless Downloader.size_validation(expected: ini_file[package]['size'], actual: rounded_size) &&
- Downloader.hash_validation(expected: ini_file[package]['md5'], actual: Utils.hashfile(file_path))
- UI.error "File at #{file_path} is corrupted, deleting it"
- File.delete(file_path)
- return nil
- end
-
- return [file_path, ini_file[package]]
- end
-
- private #---------------------------------------------------------------
-
- def get_package(name, ini_file, main_dir, base_url)
- file_name = UNITY_MODULE_FILE_REGEX.match(ini_file[name]['url'])[1]
- file_path = File.expand_path(file_name, main_dir)
-
- # Check if file already exists and validate it
- if File.file?(file_path)
- rounded_size = (File.size(file_path).to_f / 1024).floor
- if Downloader.size_validation(expected: ini_file[name]['size'], actual: rounded_size) &&
- Downloader.hash_validation(expected: ini_file[name]['md5'], actual: Utils.hashfile(file_path))
- UI.important "File already downloaded at #{file_path}"
- return file_path
- else
- UI.verbose 'Deleting existing file'
- File.delete(file_path)
- end
- end
-
- # Download file
- url = base_url + ini_file[name]['url']
- UI.header "Downloading #{name}"
- UI.verbose 'Downloading from ' + url.to_s.cyan.underline
- Downloader.download_package(file_path, url, size: ini_file[name]['size'] * 1024)
-
- # Validation download
- rounded_size = (File.size(file_path).to_f / 1024).floor
- if Downloader.size_validation(expected: ini_file[name]['size'], actual: rounded_size) &&
- Downloader.hash_validation(expected: ini_file[name]['md5'], actual: Utils.hashfile(file_path))
- UI.success "Successfully downloaded #{name}."
- else
- File.delete(file_path)
- raise 'Download failed: file is corrupted, deleting it.'
- end
-
- file_path
- end
+ def url_for(package, definition)
+ definition.url + definition[package]['url']
end
end
end
+ # rubocop:enable ModuleLength
end