s3_spec.rb in logstash-input-s3-3.4.0

- old
+ new

@@ -112,36 +112,50 @@
   end
 
   describe "#list_new_files" do
     before { allow_any_instance_of(Aws::S3::Bucket).to receive(:objects) { objects_list } }
 
-    let!(:present_object) { double(:key => 'this-should-be-present', :last_modified => Time.now, :content_length => 10) }
+    let!(:present_object) { double(:key => 'this-should-be-present', :last_modified => Time.now, :content_length => 10, :storage_class => 'STANDARD') }
+    let!(:archived_object) {double(:key => 'this-should-be-archived', :last_modified => Time.now, :content_length => 10, :storage_class => 'GLACIER') }
     let(:objects_list) {
       [
-        double(:key => 'exclude-this-file-1', :last_modified => Time.now - 2 * day, :content_length => 100),
-        double(:key => 'exclude/logstash', :last_modified => Time.now - 2 * day, :content_length => 50),
+        double(:key => 'exclude-this-file-1', :last_modified => Time.now - 2 * day, :content_length => 100, :storage_class => 'STANDARD'),
+        double(:key => 'exclude/logstash', :last_modified => Time.now - 2 * day, :content_length => 50, :storage_class => 'STANDARD'),
+        archived_object,
         present_object
       ]
     }
 
     it 'should allow user to exclude files from the s3 bucket' do
       plugin = LogStash::Inputs::S3.new(config.merge({ "exclude_pattern" => "^exclude" }))
       plugin.register
-      expect(plugin.list_new_files).to eq([present_object.key])
+
+      files = plugin.list_new_files
+      expect(files).to include(present_object.key)
+      expect(files).to_not include('exclude-this-file-1') # matches exclude pattern
+      expect(files).to_not include('exclude/logstash')    # matches exclude pattern
+      expect(files).to_not include(archived_object.key)   # archived
+      expect(files.size).to eq(1)
     end
 
     it 'should support not providing a exclude pattern' do
       plugin = LogStash::Inputs::S3.new(config)
       plugin.register
-      expect(plugin.list_new_files).to eq(objects_list.map(&:key))
+
+      files = plugin.list_new_files
+      expect(files).to include(present_object.key)
+      expect(files).to include('exclude-this-file-1')   # no exclude pattern given
+      expect(files).to include('exclude/logstash')      # no exclude pattern given
+      expect(files).to_not include(archived_object.key) # archived
+      expect(files.size).to eq(3)
     end
 
     context 'when all files are excluded from a bucket' do
       let(:objects_list) {
         [
-            double(:key => 'exclude-this-file-1', :last_modified => Time.now - 2 * day, :content_length => 100),
-            double(:key => 'exclude/logstash', :last_modified => Time.now - 2 * day, :content_length => 50),
+            double(:key => 'exclude-this-file-1', :last_modified => Time.now - 2 * day, :content_length => 100, :storage_class => 'STANDARD'),
+            double(:key => 'exclude/logstash', :last_modified => Time.now - 2 * day, :content_length => 50, :storage_class => 'STANDARD'),
         ]
       }
 
       it 'should not log that no files were found in the bucket' do
         plugin = LogStash::Inputs::S3.new(config.merge({ "exclude_pattern" => "^exclude" }))
@@ -166,37 +180,47 @@
     end
 
     context "If the bucket is the same as the backup bucket" do
       it 'should ignore files from the bucket if they match the backup prefix' do
         objects_list = [
-          double(:key => 'mybackup-log-1', :last_modified => Time.now, :content_length => 5),
+          double(:key => 'mybackup-log-1', :last_modified => Time.now, :content_length => 5, :storage_class => 'STANDARD'),
           present_object
         ]
 
         allow_any_instance_of(Aws::S3::Bucket).to receive(:objects) { objects_list }
 
         plugin = LogStash::Inputs::S3.new(config.merge({ 'backup_add_prefix' => 'mybackup',
                                                          'backup_to_bucket' => config['bucket']}))
         plugin.register
-        expect(plugin.list_new_files).to eq([present_object.key])
+
+        files = plugin.list_new_files
+        expect(files).to include(present_object.key)
+        expect(files).to_not include('mybackup-log-1') # matches backup prefix
+        expect(files.size).to eq(1)
       end
     end
 
     it 'should ignore files older than X' do
       plugin = LogStash::Inputs::S3.new(config.merge({ 'backup_add_prefix' => 'exclude-this-file'}))
 
-      expect_any_instance_of(LogStash::Inputs::S3::SinceDB::File).to receive(:read).exactly(objects_list.size) { Time.now - day }
+
+      allow_any_instance_of(LogStash::Inputs::S3::SinceDB::File).to receive(:read).and_return(Time.now - day)
       plugin.register
 
-      expect(plugin.list_new_files).to eq([present_object.key])
+      files = plugin.list_new_files
+      expect(files).to include(present_object.key)
+      expect(files).to_not include('exclude-this-file-1') # too old
+      expect(files).to_not include('exclude/logstash')    # too old
+      expect(files).to_not include(archived_object.key)   # archived
+      expect(files.size).to eq(1)
     end
 
     it 'should ignore file if the file match the prefix' do
         prefix = 'mysource/'
 
         objects_list = [
-          double(:key => prefix, :last_modified => Time.now, :content_length => 5),
+          double(:key => prefix, :last_modified => Time.now, :content_length => 5, :storage_class => 'STANDARD'),
           present_object
         ]
 
         allow_any_instance_of(Aws::S3::Bucket).to receive(:objects).with(:prefix => prefix) { objects_list }
 
@@ -205,13 +229,13 @@
         expect(plugin.list_new_files).to eq([present_object.key])
     end
 
     it 'should sort return object sorted by last_modification date with older first' do
       objects = [
-        double(:key => 'YESTERDAY', :last_modified => Time.now - day, :content_length => 5),
-        double(:key => 'TODAY', :last_modified => Time.now, :content_length => 5),
-        double(:key => 'TWO_DAYS_AGO', :last_modified => Time.now - 2 * day, :content_length => 5)
+        double(:key => 'YESTERDAY', :last_modified => Time.now - day, :content_length => 5, :storage_class => 'STANDARD'),
+        double(:key => 'TODAY', :last_modified => Time.now, :content_length => 5, :storage_class => 'STANDARD'),
+        double(:key => 'TWO_DAYS_AGO', :last_modified => Time.now - 2 * day, :content_length => 5, :storage_class => 'STANDARD')
       ]
 
       allow_any_instance_of(Aws::S3::Bucket).to receive(:objects) { objects }
 
 
@@ -313,11 +337,11 @@
     end
 
     %w(AccessDenied NoSuchKey).each do |error|
       context "when retrieving an object, #{error} is returned" do
         let(:objects) { [log] }
-        let(:log) { double(:key => 'uncompressed.log', :last_modified => Time.now - 2 * day, :content_length => 5) }
+        let(:log) { double(:key => 'uncompressed.log', :last_modified => Time.now - 2 * day, :content_length => 5, :storage_class => 'STANDARD') }
 
         let(:config) {
           {
               "access_key_id" => "1234",
               "secret_access_key" => "secret",
@@ -342,11 +366,11 @@
     end
   end
 
   context 'when working with logs' do
     let(:objects) { [log] }
-    let(:log) { double(:key => 'uncompressed.log', :last_modified => Time.now - 2 * day, :content_length => 5, :data => { "etag" => 'c2c966251da0bc3229d12c2642ba50a4' }) }
+    let(:log) { double(:key => 'uncompressed.log', :last_modified => Time.now - 2 * day, :content_length => 5, :data => { "etag" => 'c2c966251da0bc3229d12c2642ba50a4' }, :storage_class => 'STANDARD') }
     let(:data) { File.read(log_file) }
 
     before do
       Aws.config[:s3] = {
           stub_responses: {
@@ -387,26 +411,26 @@
 
       include_examples "generated events"
     end
 
     context "multiple compressed streams" do
-      let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day, :content_length => 5) }
+      let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day, :content_length => 5, :storage_class => 'STANDARD') }
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'multiple_compressed_streams.gz') }
 
       include_examples "generated events" do
         let(:events_to_process) { 16 }
       end
     end
       
     context 'compressed' do
-      let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day, :content_length => 5) }
+      let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day, :content_length => 5, :storage_class => 'STANDARD') }
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'compressed.log.gz') }
 
       include_examples "generated events"
     end
 
     context 'compressed with gzip extension' do
-      let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day, :content_length => 5) }
+      let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day, :content_length => 5, :storage_class => 'STANDARD') }
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'compressed.log.gzip') }
 
       include_examples "generated events"
     end
 