Prevent file opening for files reaching ignore_older with same offset (…

…#1649)

Files which are under ignore_older but kept the same offset, are not opened anymore.

See https://discuss.elastic.co/t/on-restart-harversters-are-opened-for-ignored-files/50038/4

CHANGELOG.asciidoc

@@ -65,6 +65,12 @@ https://github.com/elastic/beats/compare/v1.2.3...1.2[Check the HEAD diff]
 === Beats version 1.2.3
 https://github.com/elastic/beats/compare/v1.2.2...v1.2.3[View commits]
 
+==== Added
+
+*Filebeat*
+
+- Prevent file opening for files which reached ignore_older.
+
 ==== Bugfixes
 
 *Topbeat*

filebeat/crawler/prospector.go

@@ -358,6 +358,11 @@ func (p *Prospector) checkNewFile(newinfo *harvester.FileStat, file string, outp
 		p.ProspectorConfig.IgnoreOlderDuration != 0 &&
 		time.Since(newinfo.Fileinfo.ModTime()) > p.ProspectorConfig.IgnoreOlderDuration {
 
+		if oldState.offset == newinfo.Fileinfo.Size() {
+			logp.Debug("prospector", "File size of ignore_file didn't change. Nothing to do: %s", file)
+			return
+		}
+
 		logp.Debug("prospector", "Fetching old state of file to resume: %s", file)
 
 		// Are we resuming a dead file? We have to resume even if dead so we catch any old updates to the file
@@ -374,7 +379,7 @@ func (p *Prospector) checkNewFile(newinfo *harvester.FileStat, file string, outp
 				p.ProspectorConfig.IgnoreOlderDuration,
 				time.Since(newinfo.Fileinfo.ModTime()),
 				file)
-			newinfo.Skip(newinfo.Fileinfo.Size())
+			h.SetOffset(newinfo.Fileinfo.Size())
 		}
 		p.registrar.Persist <- h.GetState()
 	} else if previousFile, err := p.getPreviousFile(file, newinfo.Fileinfo); err == nil {

filebeat/harvester/harvester.go

@@ -106,7 +106,3 @@ func (fs *FileStat) Continue(old *FileStat) {
 		fs.Return = old.Return
 	}
 }
-
-func (fs *FileStat) Skip(returnOffset int64) {
-	fs.Return <- returnOffset
-}

filebeat/tests/system/test_registrar.py

@@ -395,10 +395,11 @@ def test_state_after_rotation_ignore_older(self):
         if os.name == "nt":
             # Under windows offset is +1 because of additional newline char
             assert data[os.path.abspath(testfile1)]["offset"] == 9
+            assert data[os.path.abspath(testfile2)]["offset"] == 8
         else:
             assert data[os.path.abspath(testfile1)]["offset"] == 8
+            assert data[os.path.abspath(testfile2)]["offset"] == 7
 
-        assert data[os.path.abspath(testfile2)]["offset"] == 0
 
         # Rotate files and remove old one
         os.rename(testfile2, testfile3)