Contents

# frozen_string_literal: true

require "spec_helpers"

describe Wayfarer::Middleware::Dedup, redis: true do
  let(:task) { build(:task) }
  subject(:dedup) { Wayfarer::Middleware::Dedup.new }

  before { task.metadata.staged_urls = SortedSet.new }

  describe "#call" do
    context "without staged URLs" do
      it "does not raise" do
        expect { dedup.call(task) }.not_to raise_error
      end
    end

    context "with unseen URL" do
      it "marks the URL as seen" do
        expect {
          dedup.call(task)
        }.to change { task.barrier.seen?(task.url) }.to(true)
      end

      it "yields" do
        expect { |spy| dedup.call(task, &spy) }.to yield_control
      end
    end

    context "with seen URL" do
      before { task.barrier.seen?(task.url) }

      it "does not yield" do
        expect { |spy| dedup.call(task, &spy) }.not_to yield_control
      end
    end

    context "with exception raised" do
      it "does not mark the URL as seen" do
        begin
          dedup.call(task) { raise }
        rescue StandardError
          nil
        end
        expect(task.barrier.seen?(task.url)).to be(false)
      end

      it "re-raises the exception" do
        expect {
          dedup.call(task) { raise }
        }.to raise_error(RuntimeError)
      end
    end

    describe "staged URL filtering" do
      let(:seen_urls) { %w[https://yahoo.com https://google.com] }
      let(:unseen_urls) { %w[https://w3c.org https://nasa.gov] }

      before do
        seen_urls.each do |url|
          task.barrier.seen?(url)
        end

        [*seen_urls, *unseen_urls].each do |url|
          task.metadata.staged_urls.add(url)
        end
      end

      it "filters seen staged URLs" do
        expect {
          dedup.call(task)
        }.to change { task.metadata.staged_urls }.to(SortedSet.new(unseen_urls))
      end
    end
  end
end

Version data entries

4 entries across 4 versions & 1 rubygems

Version	Path
wayfarer-0.4.3	spec/middleware/dedup_spec.rb
wayfarer-0.4.2	spec/middleware/dedup_spec.rb
wayfarer-0.4.1	spec/middleware/dedup_spec.rb
wayfarer-0.4.0	spec/middleware/dedup_spec.rb