tests: runtime_shell: add testcase for issue 4190

Signed-off-by: Takahiro Yamashita <[email protected]>

tests/runtime_shell/in_tail/README.md

@@ -110,3 +110,17 @@ Test a link that gets a truncation and Fluent Bit properly use the new offset
 **Configuration File**
 
 ```conf/truncate_link.conf```
+
+### 6. Multiline Rotation
+
+**Unit**
+
+```test_multiline_rotation```
+
+**Description**
+
+Test a multiline rotation for issue 4190.
+
+**Configuration File**
+
+```conf/multiline_rotation.conf```

tests/runtime_shell/in_tail/conf/multiline_rotation.conf

@@ -0,0 +1,60 @@
+[SERVICE]
+    flush     1
+    daemon    off
+    log_level debug
+    log_file  ${TEST_DIR}/out.log
+
+[INPUT]
+    name             tail
+    tag              a
+    path             ${TEST_DIR}/a.log
+    db               ${TEST_DIR}/a.db
+    db.sync          full
+    multiline.parser cri
+    rotate_wait      5
+    refresh_interval 2
+
+[INPUT]
+    name             tail
+    tag              b
+    path             ${TEST_DIR}/b.log
+    db               ${TEST_DIR}/b.db
+    db.sync          full
+    multiline.parser cri
+    rotate_wait      5
+    refresh_interval 2
+
+[INPUT]
+    name             tail
+    tag              c
+    path             ${TEST_DIR}/c.log
+    db               ${TEST_DIR}/c.db
+    db.sync          full
+    multiline.parser cri
+    rotate_wait      5
+    refresh_interval 2
+
+[INPUT]
+    name             tail
+    tag              d
+    path             ${TEST_DIR}/d.log
+    db               ${TEST_DIR}/d.db
+    db.sync          full
+    multiline.parser cri
+    rotate_wait      5
+    refresh_interval 2
+
+[INPUT]
+    name             tail
+    tag              e
+    path             ${TEST_DIR}/e.log
+    db               ${TEST_DIR}/e.db
+    db.sync          full
+    multiline.parser cri
+    rotate_wait      5
+    refresh_interval 2
+
+[OUTPUT]
+    name   file
+    match  *
+    path   ${TEST_DIR}

tests/runtime_shell/in_tail/run_tests.sh

@@ -445,6 +445,86 @@ test_truncate_link() {
     kill -15 $FLB_PID
 }
 
+# 6. Multiline + rotation
+# ------------------
+# Run the logger tool that creates 5 different files, write 100000 messages to each one
+# while rotating at 256KB.
+#
+# This test for issue 4190
+#
+# Configuration file used: conf/multiline_rotation.conf
+
+test_multiline_rotation() {
+    # Helper function to check monitored files
+    sqlite_check()
+    {
+        # Incoming parameters:
+        #   $1: temporal directory to store data
+        #   $2: database file name
+        #   $3: Fluent Bit PID
+        #
+        # This function store the remaining monitored files listed in the database,
+        # we send the output to an .inodes for troubleshooting purposes if required
+        sqlite3 $1/$2 -batch \
+                ".headers off" ".width 20" "SELECT inode FROM in_tail_files" > \
+                $1/$2.inodes
+
+        rows=`cat $1/$2.inodes | wc -l | tr -d -C '[0-9]'`
+        if [ $rows != "1" ]; then
+            echo "> database file $1/$2 contains $rows rows, inodes:"
+            cat $1/$2.inodes
+            echo "> open files"
+            ls -l /proc/$3/fd/ | grep \\.log
+        else
+            echo "> database file $1/$2 is OK"
+        fi
+        ${_ASSERT_EQUALS_} "1" $rows
+    }
+
+    # Prepare test directory
+    export TEST_DIR=tmp_test
+    rm -rf $TEST_DIR
+    mkdir $TEST_DIR
+
+    # Create empty files so Fluent Bit will enqueue them on start
+    for logfile in a b c d e ; do
+        touch $TEST_DIR/$logfile.log
+    done
+
+    # Run Fluent Bit
+    $FLB_BIN -c conf/multiline_rotation.conf &
+    FLB_PID=$!
+    echo "Fluent Bit started, pid=$FLB_PID"
+
+    # Start the Logger: 5 files = 500000 log lines in total
+    python logger_file.py -l 100000 -s 256 -b 100 -d 0.1 \
+           -f $TEST_DIR/a.log \
+           -f $TEST_DIR/b.log \
+           -f $TEST_DIR/c.log \
+           -f $TEST_DIR/d.log \
+           -f $TEST_DIR/e.log
+
+    echo "Logger finished...wait 10 seconds"
+    sleep 10
+
+    # Count number of processed lines
+    write_lines=`cat $TEST_DIR/[abcdefghij].log* | wc -l`
+    read_lines=`cat $TEST_DIR/[abcdefghij] | wc -l`
+
+    echo "> write lines: $write_lines"
+    echo "> read lines : $read_lines"
+
+    # Check we processed same number of records
+    ${_ASSERT_EQUALS_} $write_lines $read_lines
+
+    # Validate our database files has only one remaining entry per database file
+    for logfile in a b c d e; do
+        sqlite_check $TEST_DIR "$logfile.db" $FLB_PID
+    done
+
+    # Stop Fluent Bit (SIGTERM)
+    kill -15 $FLB_PID
+}
 
 # Launch the tests
 . $FLB_RUN_TEST