Sha256: f9d170e7542d718abcddb839a2d05a9932af11ff360c920c326481c2559a0ace
Contents?: true
Size: 1.43 KB
Versions: 3
Compression:
Stored size: 1.43 KB
Contents
class FeatureScraper attr_reader :name def initialize(url) @url = url @agent = Mechanize.new @agent.pluggable_parser['text/plain'] = GisScraper::JSONParser @layer = layer # hash of json @name, @pk, @max = name, pk, max # maxRecordCount - usually 1000 @form = form @loops = loops @threads = GisScraper.config[:threads] end def json_data data(0).merge({'features' => features(@threads)}).to_json end private def layer @agent.get(@url + '?f=pjson').json end def name @layer['name'] end def pk @layer['fields'].select { |f| f['type'] == 'esriFieldTypeOID' }[0]['name'] end def max @layer['maxRecordCount'].to_i end def form @agent.get(@url + '/query').forms.first end def count set_query_params @form.submit(@form.buttons[1]).json['count'].to_i end def set_query_params(loop_num = nil) @form.fields[0].value = where_text(loop_num) loop_num ? @form.radiobuttons[4].uncheck : @form.radiobuttons[4].check @form.fields[6].value = '*' @form.field_with(name: 'f').options[1].select # for JSON end def data(n) set_query_params(n) @form.submit(@form.buttons[1]).json end def features(t) Parallel.map(0...@loops, in_threads: t) { |n| data(n)['features'] }.flatten end def loops (count.to_f/@max).ceil end def where_text(n) n ? "#{pk} > #{n * @max} AND #{pk} <= #{(n + 1) * @max}" : "#{pk} > 0" end end
Version data entries
3 entries across 3 versions & 1 rubygems
Version | Path |
---|---|
gis_scraper-0.1.9.pre | lib/gis_scraper/feature_scraper.rb |
gis_scraper-0.1.8.pre | lib/gis_scraper/feature_scraper.rb |
gis_scraper-0.1.7.pre | lib/gis_scraper/feature_scraper.rb |