Sha256: 086fa04559bf8900939674628f774da009b80f744674b3aac08bda638e22d0f0
Contents?: true
Size: 1.26 KB
Versions: 12
Compression:
Stored size: 1.26 KB
Contents
module Twitterscraper
module Proxy
PROXY_URL = 'https://free-proxy-list.net/'
class RetryExhausted < StandardError
end
class Pool
def initialize
@items = Proxy.get_proxies
@cur_index = 0
end
def sample
if @cur_index >= @items.size
reload
end
@cur_index += 1
@items[@cur_index - 1]
end
def size
@items.size
end
def empty?
@items.empty?
end
private
def reload
@items = Proxy.get_proxies
@cur_index = 0
end
end
module_function
def get_proxies(retries = 3)
response = Twitterscraper::Http.get(PROXY_URL)
html = Nokogiri::HTML(response)
table = html.xpath('//table[@id="proxylisttable"]').first
proxies = []
table.xpath('tbody/tr').each do |tr|
cells = tr.xpath('td')
ip, port, anonymity, https = [0, 1, 4, 6].map { |i| cells[i].text.strip }
next unless ['elite proxy', 'anonymous'].include?(anonymity)
next if https == 'no'
proxies << ip + ':' + port
end
proxies.shuffle
rescue => e
if (retries -= 1) > 0
retry
else
raise RetryExhausted.new(e.inspect)
end
end
end
end
Version data entries
12 entries across 12 versions & 1 rubygems