Contents

module InstDataShipper
  module Destinations
    module Concerns
      module Chunking
        extend ActiveSupport::Concern

        DEFAULT_CHUNK_SIZE = 100_000

        def chunk_data(generator, chunk_size: nil, **kwargs)
          chunk_size ||= config&.dig(:params, :chunk_size) || DEFAULT_CHUNK_SIZE
          slice = 1

          btchr = CanvasSync::BatchProcessor.new(of: chunk_size) do |batch|
            yield batch, slice
            slice += 1
          end

          super(generator, **kwargs) do |row|
            btchr << row
          end

          btchr.flush
        end

        def group_key
          super.tap do |k|
            k[:chunk_size] = config&.dig(:params, :chunk_size) || DEFAULT_CHUNK_SIZE
          end
        end

      end
    end
  end
end

Version data entries

6 entries across 6 versions & 1 rubygems

Version	Path
inst_data_shipper-0.2.6	lib/inst_data_shipper/destinations/concerns/chunking.rb
inst_data_shipper-0.2.5	lib/inst_data_shipper/destinations/concerns/chunking.rb
inst_data_shipper-0.2.4	lib/inst_data_shipper/destinations/concerns/chunking.rb
inst_data_shipper-0.2.3	lib/inst_data_shipper/destinations/concerns/chunking.rb
inst_data_shipper-0.2.2	lib/inst_data_shipper/destinations/concerns/chunking.rb
inst_data_shipper-0.2.1	lib/inst_data_shipper/destinations/concerns/chunking.rb