From 6f2bfc16232dbcd08c8cdbadbf9a9718c6e3298f Mon Sep 17 00:00:00 2001 From: godway Date: Wed, 7 Apr 2021 11:15:05 +0900 Subject: [PATCH] Fix missing file because of cutoff time change cutoff time calculation part is using Time.now But if file list to check is too many, sometimes Time.now will be change seconds. example) previous loop : Time.now => 2021-04-06T20:13:59.996 current loop : Time.now => 2021-04-06T20:14:00.001 If two files have same modified second, first file is not process for next cycle. The other is processed and update sincedb. When next cycle, first file is not processed because first file's modified time is same sincedb. If current_time will be get before loop, there will be no missing files. --- lib/logstash/inputs/s3.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/logstash/inputs/s3.rb b/lib/logstash/inputs/s3.rb index b04bf74..b33f4ba 100644 --- a/lib/logstash/inputs/s3.rb +++ b/lib/logstash/inputs/s3.rb @@ -130,6 +130,7 @@ def run(queue) def list_new_files objects = [] found = false + current_time = Time.now begin @s3bucket.objects(:prefix => @prefix).each do |log| found = true @@ -140,7 +141,7 @@ def list_new_files @logger.debug('Object Zero Length', :key => log.key) elsif !sincedb.newer?(log.last_modified) @logger.debug('Object Not Modified', :key => log.key) - elsif log.last_modified > (Time.now - CUTOFF_SECOND).utc # file modified within last two seconds will be processed in next cycle + elsif log.last_modified > (current_time - CUTOFF_SECOND).utc # file modified within last two seconds will be processed in next cycle @logger.debug('Object Modified After Cutoff Time', :key => log.key) elsif (log.storage_class == 'GLACIER' || log.storage_class == 'DEEP_ARCHIVE') && !file_restored?(log.object) @logger.debug('Object Archived to Glacier', :key => log.key)