out_s3: support UUID in s3 key format

Signed-off-by: Zhonghui Hu <[email protected]>
PettitWesley · Mar 5, 2021 · 1b7e2ae · 1b7e2ae
1 parent 2f898b3
commit 1b7e2ae
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 6 deletions.
diff --git a/plugins/out_s3/s3.c b/plugins/out_s3/s3.c
@@ -405,6 +405,9 @@ static int cb_s3_init(struct flb_output_instance *ins,
             flb_plg_error(ctx->ins, "'s3_key_format' must start with a '/'");
             return -1;
         }
+        if (strstr((char *) tmp, "$UUID")) {
+            ctx->key_fmt_has_uuid = FLB_TRUE;
+        }
     }
 
     /* validate 'total_file_size' */
@@ -1033,7 +1036,7 @@ static int s3_put_object(struct flb_s3 *ctx, const char *tag, time_t create_time
     }
 
     len = strlen(s3_key);
-    if ((len + 16) <= 1024) {
+    if ((len + 16) <= 1024 && !ctx->key_fmt_has_uuid) {
         append_random = FLB_TRUE;
         len += 16;
     }

diff --git a/plugins/out_s3/s3.h b/plugins/out_s3/s3.h
@@ -136,6 +136,7 @@ struct flb_s3 {
 
     int timer_created;
     int timer_ms;
+    int key_fmt_has_uuid;
 
     struct flb_output_instance *ins;
 };

diff --git a/src/aws/flb_aws_util.c b/src/aws/flb_aws_util.c
@@ -38,11 +38,7 @@
 #define TAG_DESCRIPTOR "$TAG"
 #define MAX_TAG_PARTS 10
 #define S3_KEY_SIZE 1024
-
-#define TAG_PART_DESCRIPTOR "$TAG[%d]"
-#define TAG_DESCRIPTOR "$TAG"
-#define MAX_TAG_PARTS 10
-#define S3_KEY_SIZE 1024
+#define RANDOM_STRING "$UUID"
 
 struct flb_http_client *request_do(struct flb_aws_client *aws_client,
                                    int method, const char *uri,
@@ -693,6 +689,7 @@ flb_sds_t flb_get_s3_key(const char *format, time_t time, const char *tag, char
     int ret = 0;
     char *tag_token = NULL;
     char *key;
+    char *random_alphanumeric;
     int len;
     flb_sds_t tmp = NULL;
     flb_sds_t buf = NULL;
@@ -799,6 +796,28 @@ flb_sds_t flb_get_s3_key(const char *format, time_t time, const char *tag, char
     s3_key = tmp_key;
     tmp_key = NULL;
 
+    /* Find all occurences of $UUID and replace with a random string. */
+    random_alphanumeric = flb_sts_session_name();
+    if (!random_alphanumeric) {
+        goto error;
+    }
+    /* only use 8 chars of the random string */
+    random_alphanumeric[8] = '\0';
+    tmp_key = replace_uri_tokens(s3_key, RANDOM_STRING, random_alphanumeric);
+    if (!tmp_key) {
+        flb_free(random_alphanumeric);
+        goto error;
+    }
+
+    if(strlen(tmp_key) > S3_KEY_SIZE){
+        flb_warn("[s3_key] Object key length is longer than the 1024 character limit.");
+    }
+
+    flb_sds_destroy(s3_key);
+    s3_key = tmp_key;
+    tmp_key = NULL;
+    flb_free(random_alphanumeric);
+
     gmt = gmtime(&time);
 
     flb_sds_destroy(tmp);