require 'observer' module Unbreakable module DataStorage # Stores files to the filesystem. To configure: # # scraper.configure do |c| # c.datastore = Unbreakable::DataStorage::FileDataStore.new(scraper, # :decorators => [:timeout], # optional # :observers => [:log]) # optional # c.datastore.root_path = '/path/dir' # default '/var/tmp/unbreakable' # c.datastore.store_meta = true # default false # end class FileDataStore < Dragonfly::DataStorage::FileDataStore include Observable include Dragonfly::Loggable # Decorators should be able to add configuration variables. public_class_method :configurable_attr # Configure the datastore to overwrite files upon repeated download. # # scraper.configure do |c| # c.datastore.clobber = true # default false # end # # @return [Boolean, Proc, lambda] whether to overwrite files upon repeated # download configurable_attr :clobber, false # @param [Dragonfly::App] app # @param [Hash] opts # @option options [Module, Symbol, Array] :decorators # a module, the name of a decorator module, or an array of such # @option options [Class, Symbol, Array] :observers # a class, the name of an observer class, or an array of such def initialize(app, opts = {}) use_same_log_as(app) use_as_fallback_config(app) if opts[:decorators] opts[:decorators].each do |decorator| extend Symbol === decorator ? Unbreakable::Decorators.const_get(decorator.capitalize) : decorator end end if opts[:observers] opts[:observers].each do |observer| add_observer Symbol === observer ? Unbreakable::Observers.const_get(observer.capitalize).new(self) : observer.new(self) end end end # Stores a record in the datastore. This method does lazy evaluation of # the record's contents, e.g.: # # defer_store(:path => 'index.html') do # open('http://www.example.com/').read # end # # The +open+ method is called only if the record hasn't already been # downloaded or if the datastore has been configured to overwrite files # upon repeated download. # # @param [Hash] opts # @option opts [Hash] :meta any file metadata, e.g. bitrate # @option opts [String] :path the relative path at which to store the file # @param [Proc] block a block that yields the contents of the file # @raise [Dragonfly::DataStorage::UnableToStore] if permission is denied # @return [String] the relative path to the file # @see [Dragonfly::DataStorage::FileDataStore#store] def defer_store(opts = {}, &block) meta = opts[:meta] || {} relative_path = if opts[:path] opts[:path] else filename = meta[:name] || 'file' relative_path = relative_path_for(filename) end changed if empty?(relative_path) or clobber?(relative_path) begin path = absolute(relative_path) prepare_path(path) string = yield_block(relative_path, &block) Dragonfly::TempObject.new(string).to_file(path).close store_meta_data(path, meta) if store_meta notify_observers :store, relative_path, string relative(path) rescue InvalidRemoteFile => e log.error e.message rescue Errno::EACCES => e raise UnableToStore, e.message end else notify_observers :skip, relative_path end end # Returns all filenames matching a pattern, if given. # @param [String, Regexp] pattern a pattern to match filenames with # @return [Array] an array of matching filenames def records(pattern = nil) if pattern Dir[File.join(root_path, '**', pattern)] else Dir[File.join(root_path, '**', '*')] end.map do |absolute_path| relative absolute_path end end private # @param [String] relative_path the relative path to the file # @return [Boolean] whether the file is empty or non-existent def empty?(relative_path) path = absolute(relative_path) !File.exist?(path) || File.size(path).zero? end # @param [String] relative_path the relative path to the file # @return [Boolean] whether to overwrite any existing file def clobber?(relative_path) if clobber.respond_to? :call clobber.call(relative_path) else !!clobber end end # Yields a block. # @param [String] relative_path the relative path to the file # @return [String] the contents of the file def yield_block(relative_path) yield end end end end