in_tail: create stream_id by file inode(#4190)

plugins/in_tail/tail_file.c

@@ -795,6 +795,7 @@ int flb_tail_file_append(char *path, struct stat *st, int mode,
     size_t tag_len;
     struct flb_tail_file *file;
     struct stat lst;
+    flb_sds_t inode_str;
 
     if (!S_ISREG(st->st_mode)) {
         return -1;
@@ -882,18 +883,30 @@ int flb_tail_file_append(char *path, struct stat *st, int mode,
 
     /* Multiline core mode */
     if (ctx->ml_ctx) {
+        /*
+         * Create inode str to get stream_id.
+         *
+         * If stream_id is created by filename,
+         * it will be same after file rotation and it causes invalid destruction.
+         * https://github.com/fluent/fluent-bit/issues/4190
+         *
+         */
+        inode_str = flb_sds_create_size(64);
+        flb_sds_printf(&inode_str, "%"PRIu64, file->inode);
         /* Create a stream for this file */
         ret = flb_ml_stream_create(ctx->ml_ctx,
-                                   file->name, file->name_len,
+                                   inode_str, flb_sds_len(inode_str),
                                    ml_flush_callback, file,
                                    &stream_id);
         if (ret != 0) {
             flb_plg_error(ctx->ins,
                           "could not create multiline stream for file: %s",
-                          file->name);
+                          inode_str);
+            flb_sds_destroy(inode_str);
             goto error;
         }
         file->ml_stream_id = stream_id;
+        flb_sds_destroy(inode_str);
     }
 
     /* Local buffer */

tests/runtime_shell/in_tail/README.md

@@ -110,3 +110,17 @@ Test a link that gets a truncation and Fluent Bit properly use the new offset
 **Configuration File**
 
 ```conf/truncate_link.conf```
+
+### 6. Multiline Rotation
+
+**Unit**
+
+```test_multiline_rotation```
+
+**Description**
+
+Test a multiline rotation for issue 4190.
+
+**Configuration File**
+
+```conf/multiline_rotation.conf```

tests/runtime_shell/in_tail/conf/multiline_rotation.conf

@@ -0,0 +1,60 @@
+[SERVICE]
+    flush     1
+    daemon    off
+    log_level debug
+    log_file  ${TEST_DIR}/out.log
+
+[INPUT]
+    name             tail
+    tag              a
+    path             ${TEST_DIR}/a.log
+    db               ${TEST_DIR}/a.db
+    db.sync          full
+    multiline.parser cri
+    rotate_wait      5
+    refresh_interval 2
+
+[INPUT]
+    name             tail
+    tag              b
+    path             ${TEST_DIR}/b.log
+    db               ${TEST_DIR}/b.db
+    db.sync          full
+    multiline.parser cri
+    rotate_wait      5
+    refresh_interval 2
+
+[INPUT]
+    name             tail
+    tag              c
+    path             ${TEST_DIR}/c.log
+    db               ${TEST_DIR}/c.db
+    db.sync          full
+    multiline.parser cri
+    rotate_wait      5
+    refresh_interval 2
+
+[INPUT]
+    name             tail
+    tag              d
+    path             ${TEST_DIR}/d.log
+    db               ${TEST_DIR}/d.db
+    db.sync          full
+    multiline.parser cri
+    rotate_wait      5
+    refresh_interval 2
+
+[INPUT]
+    name             tail
+    tag              e
+    path             ${TEST_DIR}/e.log
+    db               ${TEST_DIR}/e.db
+    db.sync          full
+    multiline.parser cri
+    rotate_wait      5
+    refresh_interval 2
+
+[OUTPUT]
+    name   file
+    match  *
+    path   ${TEST_DIR}

tests/runtime_shell/in_tail/run_tests.sh

@@ -445,6 +445,86 @@ test_truncate_link() {
     kill -15 $FLB_PID
 }
 
+# 6. Multiline + rotation
+# ------------------
+# Run the logger tool that creates 5 different files, write 100000 messages to each one
+# while rotating at 256KB.
+#
+# This test for issue 4190
+#
+# Configuration file used: conf/multiline_rotation.conf
+
+test_multiline_rotation() {
+    # Helper function to check monitored files
+    sqlite_check()
+    {
+        # Incoming parameters:
+        #   $1: temporal directory to store data
+        #   $2: database file name
+        #   $3: Fluent Bit PID
+        #
+        # This function store the remaining monitored files listed in the database,
+        # we send the output to an .inodes for troubleshooting purposes if required
+        sqlite3 $1/$2 -batch \
+                ".headers off" ".width 20" "SELECT inode FROM in_tail_files" > \
+                $1/$2.inodes
+
+        rows=`cat $1/$2.inodes | wc -l | tr -d -C '[0-9]'`
+        if [ $rows != "1" ]; then
+            echo "> database file $1/$2 contains $rows rows, inodes:"
+            cat $1/$2.inodes
+            echo "> open files"
+            ls -l /proc/$3/fd/ | grep \\.log
+        else
+            echo "> database file $1/$2 is OK"
+        fi
+        ${_ASSERT_EQUALS_} "1" $rows
+    }
+
+    # Prepare test directory
+    export TEST_DIR=tmp_test
+    rm -rf $TEST_DIR
+    mkdir $TEST_DIR
+
+    # Create empty files so Fluent Bit will enqueue them on start
+    for logfile in a b c d e ; do
+        touch $TEST_DIR/$logfile.log
+    done
+
+    # Run Fluent Bit
+    $FLB_BIN -c conf/multiline_rotation.conf &
+    FLB_PID=$!
+    echo "Fluent Bit started, pid=$FLB_PID"
+
+    # Start the Logger: 5 files = 500000 log lines in total
+    python logger_file.py -l 100000 -s 256 -b 100 -d 0.1 \
+           -f $TEST_DIR/a.log \
+           -f $TEST_DIR/b.log \
+           -f $TEST_DIR/c.log \
+           -f $TEST_DIR/d.log \
+           -f $TEST_DIR/e.log
+
+    echo "Logger finished...wait 10 seconds"
+    sleep 10
+
+    # Count number of processed lines
+    write_lines=`cat $TEST_DIR/[abcdefghij].log* | wc -l`
+    read_lines=`cat $TEST_DIR/[abcdefghij] | wc -l`
+
+    echo "> write lines: $write_lines"
+    echo "> read lines : $read_lines"
+
+    # Check we processed same number of records
+    ${_ASSERT_EQUALS_} $write_lines $read_lines
+
+    # Validate our database files has only one remaining entry per database file
+    for logfile in a b c d e; do
+        sqlite_check $TEST_DIR "$logfile.db" $FLB_PID
+    done
+
+    # Stop Fluent Bit (SIGTERM)
+    kill -15 $FLB_PID
+}
 
 # Launch the tests
 . $FLB_RUN_TEST