Sha256: 18ba2ff3ead4234b095a16c7f5045bf0b58b4b36a6edc769d16424af473f4b43
Contents?: true
Size: 726 Bytes
Versions: 14
Compression:
Stored size: 726 Bytes
Contents
require 'wukong/widgets/sinks/hbase_record_sink.rb'
Wukong.chain(:friend_graph) do
tail(:scrapables) do
directory 'scrapables/ids-%{t:ymd}.tsv'
end
requester = decorator('tw_requester.rb') do
input :scrape_url, Url
output :raw_json_request, JsonString
config do
define :request_types, :default => [:follower_ids, :friend_ids], :doc => 'which requests to make: follower_ids, user_timeline, etc'
end
end
retriable_requester = retriable do
with :timeouts => [1,2,3]
on_failure :sleep
guest requester
end
tail(:scrapables)> retriable_requester > processor('tw_parse.rb') > hbase_record_sink
end
Wukong.processor(:tw_parse) do
def process
end
end
Version data entries
14 entries across 8 versions & 2 rubygems