in_tail: new db.sync property (#396)

When tailing files in environments with high load, likely a database
file is required. By default the database engine (built-in SQLite3)
have a 'FULL' synchronization to disk which means: for every change
flush changes to disk. In some hardware this I/O have performance
penalties.

The following patch introduce a new configuration property called
db.sync which aims to expose the SQLite3 desired sync configuration,
the property accepts one of the following values:

- EXTRA: synchronous is like FULL with the addition that the directory
         containing a rollback journal is synced after that journal is
         unlinked to commit a transaction in DELETE mode. EXTRA
         provides additional durability if the commit is followed closely
         by a power loss.

- FULL (default): When synchronous is FULL (2), the SQLite database
         engine will use the Sync method of the VFS to ensure that all
         content is safely written to the disk surface prior to continuing.
         This ensures that an operating system crash or power failure
         will not corrupt the database. FULL synchronous is very safe,
         but it is also slower. FULL is the most commonly used
         synchronous setting when not in WAL mode.

- NORMAL: When synchronous is NORMAL, the SQLite database engine will
          still sync at the most critical moments, but less often than
          in FULL mode. There is a very small (though non-zero) chance
          that a power failure at just the wrong time could corrupt
          the database in NORMAL mode. But in practice, you are more
          likely to suffer a catastrophic disk failure or some other
          unrecoverable hardware fault.

- OFF:    With synchronous OFF (0), SQLite continues without syncing
          as soon as it has handed data off to the operating system.
          If the application running SQLite crashes, the data will
          be safe, but the database might become corrupted if the
          operating system crashes or the computer loses power before
          that data has been written to the disk surface. On the other
          hand, commits can be orders of magnitude faster with
          synchronous OFF.

note: params description copied from SQLite site

Signed-off-by: Eduardo Silva <[email protected]>

plugins/in_tail/tail_config.c

@@ -49,6 +49,7 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *i_ins,
     ctx->dynamic_tag = FLB_FALSE;
     ctx->ignore_older = 0;
     ctx->skip_long_lines = FLB_FALSE;
+    ctx->db_sync = -1;
 
     /* Create the channel manager */
     ret = pipe(ctx->ch_manager);
@@ -240,10 +241,30 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *i_ins,
     }
     i_ins->flags |= FLB_INPUT_DYN_TAG;
 
+    /* Database options (needs to be set before the context) */
+    tmp = flb_input_get_property("db.sync", i_ins);
+    if (tmp) {
+        if (strcasecmp(tmp, "extra") == 0) {
+            ctx->db_sync = 3;
+        }
+        else if (strcasecmp(tmp, "full") == 0) {
+            ctx->db_sync = 2;
+            }
+        else if (strcasecmp(tmp, "normal") == 0) {
+            ctx->db_sync = 1;
+        }
+        else if (strcasecmp(tmp, "off") == 0) {
+            ctx->db_sync = 0;
+        }
+        else {
+            flb_error("[in_tail] invalid database 'db.sync' value");
+        }
+    }
+
     /* Initialize database */
     tmp = flb_input_get_property("db", i_ins);
     if (tmp) {
-        ctx->db = flb_tail_db_open(tmp, i_ins, config);
+        ctx->db = flb_tail_db_open(tmp, i_ins, ctx, config);
         if (!ctx->db) {
             flb_error("[in_tail] could not open/create database");
         }

plugins/in_tail/tail_config.h

@@ -62,6 +62,7 @@ struct flb_tail_config {
 
     /* Database */
     struct flb_sqldb *db;
+    int db_sync;
 
     /* Parser / Format */
     struct flb_parser *parser;

plugins/in_tail/tail_db.c

@@ -35,9 +35,11 @@ struct query_status {
 /* Open or create database required by tail plugin */
 struct flb_sqldb *flb_tail_db_open(char *path,
                                    struct flb_input_instance *in,
+                                   struct flb_tail_config *ctx,
                                    struct flb_config *config)
 {
     int ret;
+    char tmp[64];
     struct flb_sqldb *db;
 
     /* Open/create the database */
@@ -54,6 +56,17 @@ struct flb_sqldb *flb_tail_db_open(char *path,
         return NULL;
     }
 
+    if (ctx->db_sync >= 0) {
+        snprintf(tmp, sizeof(tmp) - 1, SQL_PRAGMA_SYNC,
+                 ctx->db_sync);
+        ret = flb_sqldb_query(db, tmp, NULL, NULL);
+        if (ret != FLB_OK) {
+            flb_error("[in_tail:db] could not set pragma 'sync'");
+            flb_sqldb_close(db);
+            return NULL;
+        }
+    }
+
     return db;
 }
 

plugins/in_tail/tail_db.h

@@ -27,6 +27,7 @@
 
 struct flb_sqldb *flb_tail_db_open(char *path,
                                    struct flb_input_instance *in,
+                                   struct flb_tail_config *ctx,
                                    struct flb_config *config);
 
 int flb_tail_db_close(struct flb_sqldb *db);

plugins/in_tail/tail_sql.h

@@ -50,4 +50,6 @@
 #define SQL_ROTATE_FILE                         \
     "UPDATE in_tail_files set name='%s',rotated=1 WHERE id=%"PRId64";"
 
+#define SQL_PRAGMA_SYNC                         \
+    "PRAGMA synchronous=%i;"
 #endif