Contents

module Wukong
  module Store
    class ChunkedFlatFileStore < Wukong::Store::FlatFileStore
      attr_accessor :filename_pattern, :chunk_monitor, :handle, :chunktime, :rootdir

      # Move to configliere
      Settings.define :chunk_file_pattern,   :default => ":rootdir/:date/:handle:timestamp-:pid.tsv",:description => "The pattern for chunked files."
      Settings.define :chunk_file_chunktime, :default => 4*60*60,:description => "The time interval to keep a chunk file open."
      Settings.define :chunk_file_rootdir,   :default => nil, :description => "The root directory for the chunked files."
      
      #Note that filemode is inherited from flat_file

      def initialize options={}
        # super wants a :filename in the options or it will fail. We need to get the initial filename
        # set up before we call super, so we need all of the parts of the pattern set up. 
        self.chunktime        = options[:chunktime] || Settings[:chunk_file_chunktime]
        self.rootdir          = options[:rootdir]   || Settings[:chunk_file_rootdir]
        self.handle           = options[:handle] 
        pattern               = options[:pattern] || Settings[:chunk_file_pattern]
        self.filename_pattern = FilenamePattern.new(pattern, :handle => handle, :rootdir => self.rootdir)
        options[:filename]    = filename_pattern.make() 
      
        super options

        Log.warn "You don't really want a chunk time this small: #{self.chunktime}" unless self.chunktime > 600
        self.chunk_monitor    = Wukong::PeriodicMonitor.new( :time => self.chunktime )
        self.mkdir!
      end

      def save *args
        result = super *args
        chunk_monitor.periodically do
          new_filename = filename_pattern.make()
          Log.info "Rotating chunked file #{filename} into #{new_filename}"
          self.close
          @filename = new_filename
          self.mkdir!
        end
        result
      end

    end
  end
end

Version data entries

1 entries across 1 versions & 1 rubygems

Version	Path
wukong-1.5.3	lib/wukong/store/chunked_flat_file_store.rb