lib/rubygems/source_index.rb in rubygems-update-0.9.4 vs lib/rubygems/source_index.rb in rubygems-update-0.9.5
- old
+ new
@@ -2,17 +2,16 @@
# Copyright 2006 by Chad Fowler, Rich Kilmer, Jim Weirich and others.
# All rights reserved.
# See LICENSE.txt for permissions.
#++
-require 'rubygems/user_interaction'
-require 'rubygems/remote_fetcher'
-require 'rubygems/digest/sha2'
-
require 'forwardable'
-require 'time'
+require 'rubygems'
+require 'rubygems/user_interaction'
+require 'rubygems/specification'
+
module Gem
# The SourceIndex object indexes all the gems available from a
# particular source (e.g. a list of gem directories, or a remote
# source). A SourceIndex maps a gem full name to a gem
@@ -28,12 +27,10 @@
include Enumerable
include Gem::UserInteraction
- INCREMENTAL_THRESHHOLD = 50
-
# Class Methods. -------------------------------------------------
class << self
include Gem::UserInteraction
# Factory method to construct a source index instance for a given
@@ -87,11 +84,11 @@
# return:: Specification instance or nil if an error occurs
#
def load_specification(file_name)
begin
spec_code = File.read(file_name).untaint
- gemspec = eval(spec_code)
+ gemspec = eval spec_code, binding, file_name
if gemspec.is_a?(Gem::Specification)
gemspec.loaded_from = file_name
return gemspec
end
alert_warning "File '#{file_name}' does not evaluate to a gem specification"
@@ -132,22 +129,28 @@
end
# Returns a Hash of name => Specification of the latest versions of each
# gem in this index.
def latest_specs
- thin = {}
+ result, latest = Hash.new { |h,k| h[k] = [] }, {}
- each do |full_name, spec|
+ self.each do |_, spec| # SourceIndex is not a hash, so we're stuck with each
name = spec.name
- if thin.has_key? name then
- thin[name] = spec if spec.version > thin[name].version
- else
- thin[name] = spec
+ curr_ver = spec.version
+ prev_ver = latest[name]
+
+ next unless prev_ver.nil? or curr_ver >= prev_ver
+
+ if prev_ver.nil? or curr_ver > prev_ver then
+ result[name].clear
+ latest[name] = curr_ver
end
+
+ result[name] << spec
end
- thin
+ result.values.flatten
end
# Add a gem specification to the source index.
def add_spec(gem_spec)
@gems[gem_spec.full_name] = gem_spec
@@ -157,14 +160,11 @@
def remove_spec(full_name)
@gems.delete(full_name)
end
# Iterate over the specifications in the source index.
- #
- # &block:: [yields gem.full_name, Gem::Specification]
- #
- def each(&block)
+ def each(&block) # :yields: gem.full_name, gem
@gems.each(&block)
end
# The gem specification given a full gem spec name.
def specification(full_name)
@@ -172,47 +172,75 @@
end
# The signature for the source index. Changes in the signature
# indicate a change in the index.
def index_signature
+ require 'rubygems/digest/sha2'
+
Gem::SHA256.new.hexdigest(@gems.keys.sort.join(',')).to_s
end
# The signature for the given gem specification.
def gem_signature(gem_full_name)
+ require 'rubygems/digest/sha2'
+
Gem::SHA256.new.hexdigest(@gems[gem_full_name].to_yaml).to_s
end
def_delegators :@gems, :size, :length
# Find a gem by an exact match on the short name.
- def find_name(gem_name, version_requirement=Version::Requirement.new(">= 0"))
+ def find_name(gem_name, version_requirement = Gem::Requirement.default)
search(/^#{gem_name}$/, version_requirement)
end
# Search for a gem by short name pattern and optional version
#
# gem_name::
# [String] a partial for the (short) name of the gem, or
# [Regex] a pattern to match against the short name
# version_requirement::
- # [String | default=Version::Requirement.new(">= 0")] version to
+ # [String | default=Gem::Requirement.default] version to
# find
# return::
# [Array] list of Gem::Specification objects in sorted (version)
# order. Empty if not found.
#
- def search(gem_pattern, version_requirement=Version::Requirement.new(">= 0"))
- gem_pattern = /#{ gem_pattern }/i if String === gem_pattern
- version_requirement = Gem::Version::Requirement.create(version_requirement)
- result = []
- @gems.each do |full_spec_name, spec|
- next unless spec.name =~ gem_pattern
- result << spec if version_requirement.satisfied_by?(spec.version)
+ def search(gem_pattern, platform_only_or_version_req = false)
+ version_requirement = nil
+ only_platform = false
+
+ case gem_pattern
+ when Regexp then
+ version_requirement = platform_only_or_version_req ||
+ Gem::Requirement.default
+ when Gem::Dependency then
+ only_platform = platform_only_or_version_req
+ version_requirement = gem_pattern.version_requirements
+ gem_pattern = gem_pattern.name.empty? ? // : /^#{gem_pattern.name}$/
+ else
+ version_requirement = platform_only_or_version_req ||
+ Gem::Requirement.default
+ gem_pattern = /#{gem_pattern}/i
end
- result = result.sort
- result
+
+ unless Gem::Requirement === version_requirement then
+ version_requirement = Gem::Requirement.create version_requirement
+ end
+
+ specs = @gems.values.select do |spec|
+ spec.name =~ gem_pattern and
+ version_requirement.satisfied_by? spec.version
+ end
+
+ if only_platform then
+ specs = specs.select do |spec|
+ Gem::Platform.match spec.platform
+ end
+ end
+
+ specs.sort_by { |s| s.sort_obj }
end
# Refresh the source index from the local file system.
#
# return:: Returns a pointer to itself.
@@ -222,81 +250,117 @@
end
# Returns an Array of Gem::Specifications that are not up to date.
#
def outdated
- remotes = Gem::SourceInfoCache.search(//)
+ dep = Gem::Dependency.new '', Gem::Requirement.default
+
+ remotes = Gem::SourceInfoCache.search dep, true
+
outdateds = []
- latest_specs.each do |_, local|
+
+ latest_specs.each do |local|
name = local.name
remote = remotes.select { |spec| spec.name == name }.
- sort_by { |spec| spec.version }.
+ sort_by { |spec| spec.version.to_ints }.
last
outdateds << name if remote and local.version < remote.version
end
+
outdateds
end
def update(source_uri)
use_incremental = false
begin
gem_names = fetch_quick_index source_uri
remove_extra gem_names
missing_gems = find_missing gem_names
- use_incremental = missing_gems.size <= INCREMENTAL_THRESHHOLD
+
+ return false if missing_gems.size.zero?
+
+ say "missing #{missing_gems.size} gems" if
+ missing_gems.size > 0 and Gem.configuration.really_verbose
+
+ use_incremental = missing_gems.size <= Gem.configuration.bulk_threshold
rescue Gem::OperationNotSupportedError => ex
+ alert_error "Falling back to bulk fetch: #{ex.message}" if
+ Gem.configuration.really_verbose
use_incremental = false
end
if use_incremental then
- update_with_missing source_uri, missing_gems
+ update_with_missing(source_uri, missing_gems)
else
- new_index = fetch_bulk_index source_uri
- @gems.replace new_index.gems
+ new_index = fetch_bulk_index(source_uri)
+ @gems.replace(new_index.gems)
end
- self
+ true
end
-
+
def ==(other) # :nodoc:
self.class === other and @gems == other.gems
end
+ def dump
+ Marshal.dump(self)
+ end
+
protected
attr_reader :gems
private
- # Convert the yamlized string spec into a real spec (actually, these are
- # hashes of specs.).
- def convert_specs(yaml_spec)
- YAML.load(reduce_specs(yaml_spec)) or
- raise "Didn't get a valid YAML document"
- end
-
def fetcher
+ require 'rubygems/remote_fetcher'
+
Gem::RemoteFetcher.fetcher
end
- def fetch_bulk_index(source_uri)
- say "Bulk updating Gem source index for: #{source_uri}"
+ def fetch_index_from(source_uri)
+ @fetch_error = nil
- begin
- yaml_spec = fetcher.fetch_path source_uri + '/yaml.Z'
- yaml_spec = unzip yaml_spec
- rescue
+ indexes = %W[
+ Marshal.#{Gem.marshal_version}.Z
+ Marshal.#{Gem.marshal_version}
+ yaml.Z
+ yaml
+ ]
+
+ indexes.each do |name|
+ spec_data = nil
begin
- yaml_spec = fetcher.fetch_path source_uri + '/yaml'
+ spec_data = fetcher.fetch_path("#{source_uri}/#{name}")
+ spec_data = unzip(spec_data) if name =~ /\.Z$/
+ if name =~ /Marshal/ then
+ return Marshal.load(spec_data)
+ else
+ return YAML.load(spec_data)
+ end
rescue => e
- raise Gem::RemoteSourceException,
- "Error fetching remote gem cache: #{e}"
+ if Gem.configuration.really_verbose then
+ alert_error "Unable to fetch #{name}: #{e.message}"
+ end
+ @fetch_error = e
end
end
+ nil
+ end
- convert_specs yaml_spec
+ def fetch_bulk_index(source_uri)
+ say "Bulk updating Gem source index for: #{source_uri}"
+
+ index = fetch_index_from(source_uri)
+ if index.nil? then
+ raise Gem::RemoteSourceException,
+ "Error fetching remote gem cache: #{@fetch_error}"
+ end
+ @fetch_error = nil
+ index
end
# Get the quick index needed for incremental updates.
def fetch_quick_index(source_uri)
zipped_index = fetcher.fetch_path source_uri + '/quick/index.rz'
@@ -311,30 +375,10 @@
spec_names.find_all { |full_name|
specification(full_name).nil?
}
end
- # This reduces the source spec in size so that YAML bugs with large data
- # sets will be dodged. Obviously this is a workaround, but it allows Gems
- # to continue to work until the YAML bug is fixed.
- def reduce_specs(yaml_spec)
- result = ""
- state = :copy
- yaml_spec.each do |line|
- if state == :copy && line =~ /^\s+files:\s*$/
- state = :skip
- result << line.sub(/$/, " []")
- elsif state == :skip
- if line !~ /^\s+-/
- state = :copy
- end
- end
- result << line if state == :copy
- end
- result
- end
-
def remove_extra(spec_names)
dictionary = spec_names.inject({}) { |h, k| h[k] = true; h }
each do |name, spec|
remove_spec name unless dictionary.include? name
end
@@ -344,23 +388,50 @@
def unzip(string)
require 'zlib'
Zlib::Inflate.inflate(string)
end
+ # Tries to fetch Marshal representation first, then YAML
+ def fetch_single_spec(source_uri, spec_name)
+ @fetch_error = nil
+ begin
+ marshal_uri = source_uri + "/quick/Marshal.#{Gem.marshal_version}/#{spec_name}.gemspec.rz"
+ zipped = fetcher.fetch_path marshal_uri
+ return Marshal.load(unzip(zipped))
+ rescue => ex
+ @fetch_error = ex
+ if Gem.configuration.really_verbose then
+ say "unable to fetch marshal gemspec #{marshal_uri}: #{ex.class} - #{ex}"
+ end
+ end
+
+ begin
+ yaml_uri = source_uri + "/quick/#{spec_name}.gemspec.rz"
+ zipped = fetcher.fetch_path yaml_uri
+ return YAML.load(unzip(zipped))
+ rescue => ex
+ @fetch_error = ex
+ if Gem.configuration.really_verbose then
+ say "unable to fetch YAML gemspec #{yaml_uri}: #{ex.class} - #{ex}"
+ end
+ end
+ nil
+ end
+
# Update the cached source index with the missing names.
def update_with_missing(source_uri, missing_names)
progress = ui.progress_reporter(missing_names.size,
- "Need to update #{missing_names.size} gems from #{source_uri}")
+ "Updating metadata for #{missing_names.size} gems from #{source_uri}")
missing_names.each do |spec_name|
- begin
- spec_uri = source_uri + "/quick/#{spec_name}.gemspec.rz"
- zipped_yaml = fetcher.fetch_path spec_uri
- gemspec = YAML.load unzip(zipped_yaml)
+ gemspec = fetch_single_spec(source_uri, spec_name)
+ if gemspec.nil? then
+ ui.say "Failed to download spec #{spec_name} from #{source_uri}:\n" \
+ "\t#{@fetch_error.message}"
+ else
add_spec gemspec
progress.updated spec_name
- rescue RuntimeError => ex
- ui.say "Failed to download spec for #{spec_name} from #{source_uri}"
end
+ @fetch_error = nil
end
progress.done
progress.count
end