Contents

#!/usr/bin/env ruby
require 'rubygems'
require 'tokyocabinet' ; require 'tokyotyrant'
require 'trollop'
$: << File.dirname(__FILE__)+'/../../lib'; $: << File.dirname(__FILE__)
require 'wukong'
require 'monkeyshines'
require 'shorturl_request'
require 'shorturl_sequence'
require 'multiplex_shorturl_cache'

# Command Line options
opts = Trollop::options do
  opt :from_type,    'Class name for scrape store to load from',  :type => String
  opt :from,         'URI for scrape store to load from',  :type => String
  opt :handle,       "Handle for scrape", :type => String
  opt :log,          'File to store log', :type => String
end
Trollop::die :from_type unless opts[:from_type]

# ******************** Log ********************
periodic_log = Monkeyshines::Monitor::PeriodicLogger.new(:iters => 20_000, :time => 30)

# ******************** Load from flat file ********************
src_store_klass = Wukong.class_from_resource('Monkeyshines::Store::'+opts[:from_type])
src_store = src_store_klass.new(opts[:from], opts.merge(:filemode => 'r'))

# ******************** Store into read-thru cache ********************
RDB_PORTS = { 'tinyurl' => "localhost:10042", 'bitly' => "localhost:10043", 'other' => "localhost:10044" }
dest_uri = RDB_PORTS[opts[:handle]] or raise "Need a handle (bitly, tinyurl or other). got: #{handle}"
dest_store = Monkeyshines::Store::TyrantRdbKeyStore.new(dest_uri)
# dest_store = Monkeyshines::Store::MultiplexShorturlCache.new(RDB_PORTS)

# ******************** Dump ********************
src_store.each do |_, url, scat, *args|
  periodic_log.periodically{ [dest_store.size, url, scat, args] }
  dest_store.set_nr url, scat
end

#
# On a DB with 2M entries, this loads about 700/s
# You can optimize with something like
#   EXPECTED_MAX_KEYS = 20_000_000
#   store.db.optimize("bnum=#{2*EXPECTED_MAX_KEYS}#opts=l") # large (64-bit), 40M buckets
#

Version data entries

5 entries across 5 versions & 1 rubygems

Version	Path
monkeyshines-0.2.3	examples/shorturls/bulkload_shorturls.rb
monkeyshines-0.2.2	examples/shorturls/bulkload_shorturls.rb
monkeyshines-0.2.1	examples/shorturls/bulkload_shorturls.rb
monkeyshines-0.2.0	examples/shorturls/bulkload_shorturls.rb
monkeyshines-0.0.2	examples/shorturls/bulkload_shorturls.rb