Contents

# frozen_string_literal: true

module Miteru
  class Feeds
    class UrlScan < Base
      #
      # @param [String] base_url
      #
      def initialize(base_url = "https://urlscan.io")
        super(base_url)

        @headers = {"api-key": api_key}
      end

      def urls
        search_with_pagination.flat_map do |json|
          (json["results"] || []).map { |result| result.dig("task", "url") }
        end.uniq
      end

      private

      def size
        10_000
      end

      # @return [<Type>] <description>
      #
      def api_key
        Miteru.config.urlscan_api_key
      end

      def q
        "task.method:automatic AND date:#{Miteru.config.urlscan_date_condition}"
      end

      #
      # @param [String, nil] search_after
      #
      # @return [Hash]
      #
      def search(search_after: nil)
        get_json("/api/v1/search/", params: {q:, size:, search_after:}.compact)
      end

      def search_with_pagination
        search_after = nil

        Enumerator.new do |y|
          loop do
            res = search(search_after:)

            y.yield res

            has_more = res["has_more"]
            break unless has_more

            search_after = res["results"].last["sort"].join(",")
          end
        end
      end
    end
  end
end

Version data entries

4 entries across 4 versions & 1 rubygems

Version	Path
miteru-2.0.3	lib/miteru/feeds/urlscan.rb
miteru-2.0.2	lib/miteru/feeds/urlscan.rb
miteru-2.0.1	lib/miteru/feeds/urlscan.rb
miteru-2.0.0	lib/miteru/feeds/urlscan.rb