Contents

#!/usr/bin/env ruby
$: << File.dirname(__FILE__)+'/../lib'
require 'wukong'

#
#
module CountKeys
  #
  class Mapper < Wukong::Streamer::Base
    attr_accessor :keys_count
    def initialize *args
      self.keys_count = {}
    end
    def process key, *args
      key.gsub!(/-.*/, '')  # kill off the slug
      self.keys_count[key] ||= 0
      self.keys_count[key]  += 1
    end
    def stream *args
      super *args
      self.keys_count.each do |key, count|
        emit [key, count].to_flat
      end
    end
  end
  # Identity Mapper
  class Reducer < Wukong::Streamer::AccumulatingReducer
    attr_accessor :key_count
    require 'active_support'
    require 'action_view/helpers/number_helper'; include ActionView::Helpers::NumberHelper

    # Override to look nice
    def formatted_count item, key_count
      key_count_str = number_with_delimiter(key_count.to_i, :delimiter => ',')
      "%-25s\t%12s" % [item, key_count_str]
    end
    def start!  *args
      self.key_count = 0
    end
    def accumulate key, count
      self.key_count += count.to_i
    end
    def finalize
      yield formatted_count(key, key_count)
    end
  end

  #
  class Script < Wukong::Script
    # There's just the one field
    def default_options
      super.merge :sort_fields => 1, :reduce_tasks => 1
    end
  end
end

CountKeys::Script.new(CountKeys::Mapper, CountKeys::Reducer).run

Version data entries

18 entries across 18 versions & 2 rubygems

Version	Path
mrflip-wukong-0.1.0	examples/count_keys_at_mapper.rb
wukong-1.5.4	examples/count_keys_at_mapper.rb
wukong-1.5.3	examples/count_keys_at_mapper.rb
wukong-1.5.2	examples/count_keys_at_mapper.rb
wukong-1.5.1	examples/count_keys_at_mapper.rb
wukong-1.5.0	examples/count_keys_at_mapper.rb
wukong-1.4.12	examples/count_keys_at_mapper.rb
wukong-1.4.11	examples/count_keys_at_mapper.rb
wukong-1.4.10	examples/count_keys_at_mapper.rb
wukong-1.4.9	examples/count_keys_at_mapper.rb
wukong-1.4.7	examples/count_keys_at_mapper.rb
wukong-1.4.6	examples/count_keys_at_mapper.rb
wukong-1.4.5	examples/count_keys_at_mapper.rb
wukong-1.4.2	examples/count_keys_at_mapper.rb
wukong-1.4.1	examples/count_keys_at_mapper.rb
wukong-1.4.0	examples/count_keys_at_mapper.rb
wukong-0.1.4	examples/count_keys_at_mapper.rb
wukong-0.1.1	examples/count_keys_at_mapper.rb