diff --git a/CHANGELOG.md b/CHANGELOG.md index 607c8ab..b906dce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.8.2 + - Refactor: read sincedb time once per bucket listing [#233](https://github.com/logstash-plugins/logstash-input-s3/pull/233) + ## 3.8.1 - Feat: cast true/false values for additional_settings [#232](https://github.com/logstash-plugins/logstash-input-s3/pull/232) diff --git a/lib/logstash/inputs/s3.rb b/lib/logstash/inputs/s3.rb index 3852d82..56e976a 100644 --- a/lib/logstash/inputs/s3.rb +++ b/lib/logstash/inputs/s3.rb @@ -139,6 +139,7 @@ def list_new_files objects = [] found = false current_time = Time.now + sincedb_time = sincedb.read begin @s3bucket.objects(:prefix => @prefix).each do |log| found = true @@ -147,7 +148,7 @@ def list_new_files @logger.debug('Ignoring', :key => log.key) elsif log.content_length <= 0 @logger.debug('Object Zero Length', :key => log.key) - elsif !sincedb.newer?(log.last_modified) + elsif log.last_modified <= sincedb_time @logger.debug('Object Not Modified', :key => log.key) elsif log.last_modified > (current_time - CUTOFF_SECOND).utc # file modified within last two seconds will be processed in next cycle @logger.debug('Object Modified After Cutoff Time', :key => log.key) @@ -464,10 +465,7 @@ def initialize(file) @sincedb_path = file end - def newer?(date) - date > read - end - + # @return [Time] def read if ::File.exists?(@sincedb_path) content = ::File.read(@sincedb_path).chomp.strip @@ -479,7 +477,7 @@ def read end def write(since = nil) - since = Time.now() if since.nil? + since = Time.now if since.nil? ::File.open(@sincedb_path, 'w') { |file| file.write(since.to_s) } end end diff --git a/logstash-input-s3.gemspec b/logstash-input-s3.gemspec index 021849a..08f55fe 100644 --- a/logstash-input-s3.gemspec +++ b/logstash-input-s3.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-s3' - s.version = '3.8.1' + s.version = '3.8.2' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files in a S3 bucket" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/inputs/s3_spec.rb b/spec/inputs/s3_spec.rb index 9d9eb4c..17b988e 100644 --- a/spec/inputs/s3_spec.rb +++ b/spec/inputs/s3_spec.rb @@ -195,6 +195,7 @@ it 'should log that no files were found in the bucket' do plugin = LogStash::Inputs::S3.new(config) plugin.register + allow(plugin.logger).to receive(:info).with(/Using the provided sincedb_path/, anything) expect(plugin.logger).to receive(:info).with(/No files found/, anything) expect(plugin.list_new_files).to be_empty end