From 29ecb55d0fbe468a33b1868da998707cd60246cb Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Wed, 4 Dec 2024 12:02:49 +0000 Subject: [PATCH 01/19] calyptia: generate machine id for fleet agents Signed-off-by: Patrick Stephens --- include/CMakeLists.txt | 6 + include/fluent-bit/flb_utils.h | 1 + plugins/custom_calyptia/calyptia.c | 191 +++++++++++++++++- plugins/custom_calyptia/calyptia.h | 12 ++ plugins/in_calyptia_fleet/in_calyptia_fleet.c | 95 ++------- plugins/out_calyptia/calyptia.c | 36 ++-- plugins/out_calyptia/calyptia.h | 26 +-- src/flb_utils.c | 52 +++++ 8 files changed, 289 insertions(+), 130 deletions(-) diff --git a/include/CMakeLists.txt b/include/CMakeLists.txt index 38f1b0dccd5..30ec68744a8 100644 --- a/include/CMakeLists.txt +++ b/include/CMakeLists.txt @@ -10,6 +10,12 @@ install(FILES ${headers} COMPONENT headers PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ) +file(GLOB headers "fluent-bit/calyptia/*.h") +install(FILES ${headers} + DESTINATION ${FLB_INSTALL_INCLUDEDIR}/fluent-bit + COMPONENT headers + PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ) + file(GLOB headers "fluent-bit/config_format/*.h") install(FILES ${headers} DESTINATION ${FLB_INSTALL_INCLUDEDIR}/fluent-bit/config_format/ diff --git a/include/fluent-bit/flb_utils.h b/include/fluent-bit/flb_utils.h index b1f06592278..a0719c53e50 100644 --- a/include/fluent-bit/flb_utils.h +++ b/include/fluent-bit/flb_utils.h @@ -74,5 +74,6 @@ int flb_utils_get_machine_id(char **out_id, size_t *out_size); void flb_utils_set_plugin_string_property(const char *name, flb_sds_t *field_storage, flb_sds_t new_value); +int flb_utils_mkdir(const char *dir, int perms); #endif diff --git a/plugins/custom_calyptia/calyptia.c b/plugins/custom_calyptia/calyptia.c index bfbb42f4767..44052812fb7 100644 --- a/plugins/custom_calyptia/calyptia.c +++ b/plugins/custom_calyptia/calyptia.c @@ -27,11 +27,16 @@ #include #include #include - +#include +#include +#include +#include #include #include "calyptia.h" +#define UUID_BUFFER_SIZE 38 /* Maximum length of UUID string + null terminator */ + /* * Check if the key belongs to a sensitive data field, if so report it. We never * share any sensitive data. @@ -217,16 +222,13 @@ int set_fleet_input_properties(struct calyptia *ctx, struct flb_input_instance * flb_input_set_property(fleet, "api_key", ctx->api_key); flb_input_set_property(fleet, "host", ctx->cloud_host); flb_input_set_property(fleet, "port", ctx->cloud_port); + flb_input_set_property(fleet, "config_dir", ctx->fleet_config_dir); /* Set TLS properties */ flb_input_set_property(fleet, "tls", ctx->cloud_tls == 1 ? "on" : "off"); flb_input_set_property(fleet, "tls.verify", ctx->cloud_tls_verify == 1 ? "on" : "off"); /* Optional configurations */ - if (ctx->fleet_config_dir) { - flb_input_set_property(fleet, "config_dir", ctx->fleet_config_dir); - } - if (ctx->fleet_max_http_buffer_size) { flb_input_set_property(fleet, "max_http_buffer_size", ctx->fleet_max_http_buffer_size); } @@ -376,15 +378,182 @@ static flb_sds_t sha256_to_hex(unsigned char *sha256) return hex; } +static flb_sds_t generate_base_agent_directory(struct calyptia *ctx, flb_sds_t *fleet_dir) +{ + flb_sds_t ret = NULL; + + if (ctx == NULL || fleet_dir == NULL) { + return NULL; + } + + if (*fleet_dir == NULL) { + *fleet_dir = flb_sds_create_size(CALYPTIA_MAX_DIR_SIZE); + if (*fleet_dir == NULL) { + return NULL; + } + } + + ret = flb_sds_printf(fleet_dir, "%s", ctx->fleet_config_dir); + if (ret == NULL) { + flb_sds_destroy(*fleet_dir); + return NULL; + } + + return ret; +} + +static flb_sds_t agent_config_filename(struct calyptia *ctx, char *fname) +{ + flb_sds_t cfgname = NULL; + flb_sds_t ret; + + if (ctx == NULL || fname == NULL) { + return NULL; + } + + if (generate_base_agent_directory(ctx, &cfgname) == NULL) { + return NULL; + } + + ret = flb_sds_printf(&cfgname, PATH_SEPARATOR "%s.conf", fname); + if (ret == NULL) { + flb_sds_destroy(cfgname); + return NULL; + } + + return cfgname; +} + +static char* generate_uuid() { + char* uuid = flb_malloc(UUID_BUFFER_SIZE); + if (!uuid) { + flb_errno(); + return NULL; + } + + /* create new UUID for fleet */ + if (flb_utils_uuid_v4_gen(uuid) != 0 || strlen(uuid) == 0) { + flb_free(uuid); + return NULL; + } + return uuid; +} + +static int write_uuid_to_file(flb_sds_t fleet_machine_id, char* uuid) { + int fd; + size_t uuid_len; + + if (fleet_machine_id == NULL || uuid == NULL) { + return FLB_FALSE; + } + + /* write uuid to file */ + fd = flb_open(fleet_machine_id, O_CREAT | O_WRONLY | O_TRUNC, 0666); + if (fd == -1) { + return FLB_FALSE; + } + + uuid_len = strlen(uuid); + + if (flb_write(fd, uuid, uuid_len) != uuid_len) { + flb_close(fd); + return FLB_FALSE; + } + + flb_close(fd); + return FLB_TRUE; +} + +static int create_agent_directory(struct calyptia *ctx) +{ + if( ctx == NULL ) { + return -1; + } + + /* If it exists just return */ + if (access(ctx->fleet_config_dir, F_OK) == 0) { + return 0; + } + + /* Create the directory if it does not exist */ + if (flb_utils_mkdir(ctx->fleet_config_dir, 0700) != 0) { + flb_plg_error(ctx->ins, "failed to create directory: %s", ctx->fleet_config_dir); + return -1; + } + + return 0; +} + static flb_sds_t get_machine_id(struct calyptia *ctx) { - int ret; - char *buf; - size_t blen; + int ret = -1; + char *buf = NULL; + size_t blen = 0; unsigned char sha256_buf[64] = {0}; +#if defined(FLB_SYSTEM_WINDOWS) /* retrieve raw machine id */ ret = flb_utils_get_machine_id(&buf, &blen); +#else + /* /etc/machine-id is not guaranteed to be unique so we generate one */ + flb_sds_t fleet_machine_id = NULL; + + /** ensure we have the directory created */ + if (create_agent_directory(ctx) != 0) { + return NULL; + } + + /** now get the agent filename */ + fleet_machine_id = machine_id_fleet_config_filename(ctx); + if (fleet_machine_id == NULL) { + flb_plg_error(ctx->ins, "unable to allocate machine id file"); + return NULL; + } + + /* check if the file exists first, if it does not we generate a UUID */ + if (flb_access(fleet_machine_id, F_OK) != 0) { + + /* create new UUID for fleet */ + buf = generate_uuid(); + if( buf == NULL ) { + flb_plg_error(ctx->ins, "failed to create uuid for fleet machine id"); + flb_sds_destroy(fleet_machine_id); + return NULL; + } + flb_plg_info(ctx->ins, "generated UUID for machine ID: %s", buf); + + /* write uuid to file */ + if (write_uuid_to_file(fleet_machine_id, buf ) != FLB_TRUE) { + flb_plg_error(ctx->ins, "failed to write fleet machine id file: %s", fleet_machine_id); + flb_free(buf); + flb_sds_destroy(fleet_machine_id); + return NULL; + } + + flb_free(buf); + buf = NULL; + + flb_plg_info(ctx->ins, "written machine ID to file: %s", fleet_machine_id); + } + + /* now check file exists (it always should) and read from it */ + if (flb_access(fleet_machine_id, F_OK) == 0) { + ret = flb_utils_read_file_wrapper(fleet_machine_id, &buf, &blen); + if (ret != 0) { + flb_plg_error(ctx->ins, "failed to read fleet machine id file: %s", fleet_machine_id); + flb_sds_destroy(fleet_machine_id); + return NULL; + } + flb_plg_info(ctx->ins, "read UUID (%s) from file: %s", buf, fleet_machine_id); + } + else { /* fall back to machine-id */ + flb_plg_warn(ctx->ins, "unable to get uuid from file (%s) so falling back to machine id", fleet_machine_id); + ret = flb_utils_get_machine_id(&buf, &blen); + } + + /* Clean up no longer required filename */ + flb_sds_destroy(fleet_machine_id); +#endif if (ret == -1) { flb_plg_error(ctx->ins, "could not obtain machine id"); @@ -520,13 +689,13 @@ static struct flb_config_map config_map[] = { }, { - FLB_CONFIG_MAP_STR, "calyptia_host", "cloud-api.calyptia.com", + FLB_CONFIG_MAP_STR, "calyptia_host", DEFAULT_CALYPTIA_HOST, 0, FLB_TRUE, offsetof(struct calyptia, cloud_host), "" }, { - FLB_CONFIG_MAP_STR, "calyptia_port", "443", + FLB_CONFIG_MAP_STR, "calyptia_port", DEFAULT_CALYPTIA_PORT, 0, FLB_TRUE, offsetof(struct calyptia, cloud_port), "" }, @@ -559,7 +728,7 @@ static struct flb_config_map config_map[] = { "Fleet id to be used when registering agent in a fleet" }, { - FLB_CONFIG_MAP_STR, "fleet.config_dir", NULL, + FLB_CONFIG_MAP_STR, "fleet.config_dir", FLEET_DEFAULT_CONFIG_DIR, 0, FLB_TRUE, offsetof(struct calyptia, fleet_config_dir), "Base path for the configuration directory." }, diff --git a/plugins/custom_calyptia/calyptia.h b/plugins/custom_calyptia/calyptia.h index b4313f51182..0a2e61abad6 100644 --- a/plugins/custom_calyptia/calyptia.h +++ b/plugins/custom_calyptia/calyptia.h @@ -57,4 +57,16 @@ struct calyptia { }; int set_fleet_input_properties(struct calyptia *ctx, struct flb_input_instance *fleet); +static flb_sds_t agent_config_filename(struct calyptia *ctx, char *fname); + +/* These are unique to the agent rather than the fleet */ +#define machine_id_fleet_config_filename(a) agent_config_filename((a), "machine-id") + +/* Function wrappers to enable mocking for unit test filesystem access */ +int (*flb_access)(const char *pathname, int mode) = access; +int (*flb_open)(const char *pathname, int flags, ...) = open; +ssize_t (*flb_write)(int fd, const void *buf, size_t count) = write; +int (*flb_close)(int fd) = close; +int (*flb_utils_read_file_wrapper)(char *path, char **out_buf, size_t *out_size) = flb_utils_read_file; + #endif /* FLB_CALYPTIA_H */ diff --git a/plugins/in_calyptia_fleet/in_calyptia_fleet.c b/plugins/in_calyptia_fleet/in_calyptia_fleet.c index 28e3fc8719c..5a423856c32 100644 --- a/plugins/in_calyptia_fleet/in_calyptia_fleet.c +++ b/plugins/in_calyptia_fleet/in_calyptia_fleet.c @@ -38,6 +38,9 @@ #include #include #include +#include + +#include /* Glob support */ #ifndef _MSC_VER @@ -50,26 +53,15 @@ #define PATH_MAX MAX_PATH #endif -#define CALYPTIA_H_PROJECT "X-Project-Token" -#define CALYPTIA_H_CTYPE "Content-Type" -#define CALYPTIA_H_CTYPE_JSON "application/json" +#define CALYPTIA_HEADERS_PROJECT "X-Project-Token" +#define CALYPTIA_HEADERS_CTYPE "Content-Type" +#define CALYPTIA_HEADERS_CTYPE_JSON "application/json" #define DEFAULT_INTERVAL_SEC "15" #define DEFAULT_INTERVAL_NSEC "0" #define DEFAULT_MAX_HTTP_BUFFER_SIZE "10485760" -#define CALYPTIA_HOST "cloud-api.calyptia.com" -#define CALYPTIA_PORT "443" - -#ifndef _WIN32 -#define PATH_SEPARATOR "/" -#define DEFAULT_CONFIG_DIR "/tmp/calyptia-fleet" -#else -#define DEFAULT_CONFIG_DIR NULL -#define PATH_SEPARATOR "\\" -#endif - struct flb_in_calyptia_fleet_config { /* Time interval check */ int interval_sec; @@ -920,7 +912,7 @@ static struct flb_http_client *fleet_http_do(struct flb_in_calyptia_fleet_config flb_http_buffer_size(client, ctx->max_http_buffer_size); flb_http_add_header(client, - CALYPTIA_H_PROJECT, sizeof(CALYPTIA_H_PROJECT) - 1, + CALYPTIA_HEADERS_PROJECT, sizeof(CALYPTIA_HEADERS_PROJECT) - 1, ctx->api_key, flb_sds_len(ctx->api_key)); ret = flb_http_do(client, &b_sent); @@ -1087,56 +1079,6 @@ static int get_calyptia_file(struct flb_in_calyptia_fleet_config *ctx, return ret; } -#ifdef FLB_SYSTEM_WINDOWS -#define _mkdir(a, b) mkdir(a) -#else -#define _mkdir(a, b) mkdir(a, b) -#endif - -/* recursively create directories, based on: - * https://stackoverflow.com/a/2336245 - * who found it at: - * http://nion.modprobe.de/blog/archives/357-Recursive-directory-creation.html - */ -static int __mkdir(const char *dir, int perms) { - char tmp[255]; - char *ptr = NULL; - size_t len; - int ret; - - ret = snprintf(tmp, sizeof(tmp),"%s",dir); - if (ret > sizeof(tmp)) { - flb_error("directory too long for __mkdir: %s", dir); - return -1; - } - - len = strlen(tmp); - - if (tmp[len - 1] == PATH_SEPARATOR[0]) { - tmp[len - 1] = 0; - } - -#ifndef FLB_SYSTEM_WINDOWS - for (ptr = tmp + 1; *ptr; ptr++) { -#else - for (ptr = tmp + 3; *ptr; ptr++) { -#endif - - if (*ptr == PATH_SEPARATOR[0]) { - *ptr = 0; - if (access(tmp, F_OK) != 0) { - ret = _mkdir(tmp, perms); - if (ret != 0) { - return ret; - } - } - *ptr = PATH_SEPARATOR[0]; - } - } - - return _mkdir(tmp, perms); -} - #ifndef _WIN32 static struct cfl_array *read_glob(const char *path) { @@ -1839,7 +1781,7 @@ static int get_calyptia_fleet_config(struct flb_in_calyptia_fleet_config *ctx) return -1; } - flb_sds_printf(&ctx->fleet_url, "/v1/fleets/%s/config?format=ini", ctx->fleet_id); + flb_sds_printf(&ctx->fleet_url, CALYPTIA_ENDPOINT_FLEET_CONFIG_INI, ctx->fleet_id); } if (ctx->fleet_files_url == NULL) { @@ -1849,7 +1791,7 @@ static int get_calyptia_fleet_config(struct flb_in_calyptia_fleet_config *ctx) return -1; } - flb_sds_printf(&ctx->fleet_files_url, "/v1/fleets/%s/files", ctx->fleet_id); + flb_sds_printf(&ctx->fleet_files_url, CALYPTIA_ENDPOINT_FLEET_FILES, ctx->fleet_id); } create_fleet_header(ctx); @@ -1920,7 +1862,7 @@ static int create_fleet_directory(struct flb_in_calyptia_fleet_config *ctx) flb_sds_t myfleetdir = NULL; if (access(ctx->config_dir, F_OK) != 0) { - if (__mkdir(ctx->config_dir, 0700) != 0) { + if (flb_utils_mkdir(ctx->config_dir, 0700) != 0) { return -1; } } @@ -1931,7 +1873,7 @@ static int create_fleet_directory(struct flb_in_calyptia_fleet_config *ctx) } if (access(myfleetdir, F_OK) != 0) { - if (__mkdir(myfleetdir, 0700) !=0) { + if (flb_utils_mkdir(myfleetdir, 0700) !=0) { return -1; } } @@ -1970,18 +1912,19 @@ static flb_sds_t fleet_gendir(struct flb_in_calyptia_fleet_config *ctx, time_t t static int fleet_mkdir(struct flb_in_calyptia_fleet_config *ctx, time_t timestamp) { + int ret = -1; flb_sds_t fleetcurdir; fleetcurdir = fleet_gendir(ctx, timestamp); - if (fleetcurdir == NULL) { - return -1; + if (fleetcurdir != NULL) { + if (flb_utils_mkdir(fleetcurdir, 0700) == 0) { + ret = 0; + } + flb_sds_destroy(fleetcurdir); } - __mkdir(fleetcurdir, 0700); - flb_sds_destroy(fleetcurdir); - - return 0; + return ret; } static int fleet_cur_chdir(struct flb_in_calyptia_fleet_config *ctx) @@ -2378,7 +2321,7 @@ static struct flb_config_map config_map[] = { "Calyptia Cloud API Key." }, { - FLB_CONFIG_MAP_STR, "config_dir", DEFAULT_CONFIG_DIR, + FLB_CONFIG_MAP_STR, "config_dir", FLEET_DEFAULT_CONFIG_DIR, 0, FLB_TRUE, offsetof(struct flb_in_calyptia_fleet_config, config_dir), "Base path for the configuration directory." }, diff --git a/plugins/out_calyptia/calyptia.c b/plugins/out_calyptia/calyptia.c index c16fd31d6ee..5f145075c1e 100644 --- a/plugins/out_calyptia/calyptia.c +++ b/plugins/out_calyptia/calyptia.c @@ -341,43 +341,43 @@ static int calyptia_http_do(struct flb_calyptia *ctx, struct flb_http_client *c, return FLB_ERROR; } flb_http_add_header(c, - CALYPTIA_H_CTYPE, sizeof(CALYPTIA_H_CTYPE) - 1, - CALYPTIA_H_CTYPE_JSON, sizeof(CALYPTIA_H_CTYPE_JSON) - 1); + CALYPTIA_HEADERS_CTYPE, sizeof(CALYPTIA_HEADERS_CTYPE) - 1, + CALYPTIA_HEADERS_CTYPE_JSON, sizeof(CALYPTIA_HEADERS_CTYPE_JSON) - 1); flb_http_add_header(c, - CALYPTIA_H_PROJECT, sizeof(CALYPTIA_H_PROJECT) - 1, + CALYPTIA_HEADERS_PROJECT, sizeof(CALYPTIA_HEADERS_PROJECT) - 1, ctx->api_key, flb_sds_len(ctx->api_key)); } else if (type == CALYPTIA_ACTION_PATCH) { flb_http_add_header(c, - CALYPTIA_H_CTYPE, sizeof(CALYPTIA_H_CTYPE) - 1, - CALYPTIA_H_CTYPE_JSON, sizeof(CALYPTIA_H_CTYPE_JSON) - 1); + CALYPTIA_HEADERS_CTYPE, sizeof(CALYPTIA_HEADERS_CTYPE) - 1, + CALYPTIA_HEADERS_CTYPE_JSON, sizeof(CALYPTIA_HEADERS_CTYPE_JSON) - 1); flb_http_add_header(c, - CALYPTIA_H_AGENT_TOKEN, - sizeof(CALYPTIA_H_AGENT_TOKEN) - 1, + CALYPTIA_HEADERS_AGENT_TOKEN, + sizeof(CALYPTIA_HEADERS_AGENT_TOKEN) - 1, ctx->agent_token, flb_sds_len(ctx->agent_token)); } else if (type == CALYPTIA_ACTION_METRICS) { flb_http_add_header(c, - CALYPTIA_H_CTYPE, sizeof(CALYPTIA_H_CTYPE) - 1, - CALYPTIA_H_CTYPE_MSGPACK, - sizeof(CALYPTIA_H_CTYPE_MSGPACK) - 1); + CALYPTIA_HEADERS_CTYPE, sizeof(CALYPTIA_HEADERS_CTYPE) - 1, + CALYPTIA_HEADERS_CTYPE_MSGPACK, + sizeof(CALYPTIA_HEADERS_CTYPE_MSGPACK) - 1); flb_http_add_header(c, - CALYPTIA_H_AGENT_TOKEN, - sizeof(CALYPTIA_H_AGENT_TOKEN) - 1, + CALYPTIA_HEADERS_AGENT_TOKEN, + sizeof(CALYPTIA_HEADERS_AGENT_TOKEN) - 1, ctx->agent_token, flb_sds_len(ctx->agent_token)); } #ifdef FLB_HAVE_CHUNK_TRACE else if (type == CALYPTIA_ACTION_TRACE) { flb_http_add_header(c, - CALYPTIA_H_CTYPE, sizeof(CALYPTIA_H_CTYPE) - 1, - CALYPTIA_H_CTYPE_JSON, sizeof(CALYPTIA_H_CTYPE_JSON) - 1); + CALYPTIA_HEADERS_CTYPE, sizeof(CALYPTIA_HEADERS_CTYPE) - 1, + CALYPTIA_HEADERS_CTYPE_JSON, sizeof(CALYPTIA_HEADERS_CTYPE_JSON) - 1); flb_http_add_header(c, - CALYPTIA_H_AGENT_TOKEN, - sizeof(CALYPTIA_H_AGENT_TOKEN) - 1, + CALYPTIA_HEADERS_AGENT_TOKEN, + sizeof(CALYPTIA_HEADERS_AGENT_TOKEN) - 1, ctx->agent_token, flb_sds_len(ctx->agent_token)); } #endif @@ -1059,13 +1059,13 @@ static void cb_calyptia_flush(struct flb_event_chunk *event_chunk, /* Configuration properties map */ static struct flb_config_map config_map[] = { { - FLB_CONFIG_MAP_STR, "cloud_host", CALYPTIA_HOST, + FLB_CONFIG_MAP_STR, "cloud_host", DEFAULT_CALYPTIA_HOST, 0, FLB_TRUE, offsetof(struct flb_calyptia, cloud_host), "", }, { - FLB_CONFIG_MAP_INT, "cloud_port", CALYPTIA_PORT, + FLB_CONFIG_MAP_INT, "cloud_port", DEFAULT_CALYPTIA_PORT, 0, FLB_TRUE, offsetof(struct flb_calyptia, cloud_port), "", }, diff --git a/plugins/out_calyptia/calyptia.h b/plugins/out_calyptia/calyptia.h index 0532e4a2b01..4da215d2206 100644 --- a/plugins/out_calyptia/calyptia.h +++ b/plugins/out_calyptia/calyptia.h @@ -25,31 +25,7 @@ #include #include -/* End point */ -#define CALYPTIA_HOST "cloud-api.calyptia.com" -#define CALYPTIA_PORT "443" - -/* HTTP action types */ -#define CALYPTIA_ACTION_REGISTER 0 -#define CALYPTIA_ACTION_PATCH 1 -#define CALYPTIA_ACTION_METRICS 2 -#define CALYPTIA_ACTION_TRACE 3 - -/* Endpoints */ -#define CALYPTIA_ENDPOINT_CREATE "/v1/agents" -#define CALYPTIA_ENDPOINT_PATCH "/v1/agents/%s" -#define CALYPTIA_ENDPOINT_METRICS "/v1/agents/%s/metrics" -#define CALYPTIA_ENDPOINT_TRACE "/v1/traces/%s" - -/* Storage */ -#define CALYPTIA_SESSION_FILE "session.CALYPTIA" - -/* Headers */ -#define CALYPTIA_H_PROJECT "X-Project-Token" -#define CALYPTIA_H_AGENT_TOKEN "X-Agent-Token" -#define CALYPTIA_H_CTYPE "Content-Type" -#define CALYPTIA_H_CTYPE_JSON "application/json" -#define CALYPTIA_H_CTYPE_MSGPACK "application/x-msgpack" +#include struct flb_calyptia { /* config map */ diff --git a/src/flb_utils.c b/src/flb_utils.c index 211632260e3..3aa4412086b 100644 --- a/src/flb_utils.c +++ b/src/flb_utils.c @@ -37,6 +37,8 @@ #include #include +#include + #ifdef FLB_HAVE_AWS_ERROR_REPORTER #include @@ -1636,3 +1638,53 @@ void flb_utils_set_plugin_string_property(const char *name, *field_storage = new_value; } + +#ifdef FLB_SYSTEM_WINDOWS +#define _mkdir(a, b) mkdir(a) +#else +#define _mkdir(a, b) mkdir(a, b) +#endif + +/* recursively create directories, based on: + * https://stackoverflow.com/a/2336245 + * who found it at: + * http://nion.modprobe.de/blog/archives/357-Recursive-directory-creation.html + */ +int flb_utils_mkdir(const char *dir, int perms) { + char tmp[CALYPTIA_MAX_DIR_SIZE]; + char *ptr = NULL; + size_t len; + int ret; + + ret = snprintf(tmp, sizeof(tmp),"%s",dir); + if (ret < 0 || ret >= sizeof(tmp)) { + flb_error("directory too long for flb_utils_mkdir: %s", dir); + return -1; + } + + len = strlen(tmp); + /* len == ret but verifying index is valid */ + if ( len > 0 && tmp[len - 1] == PATH_SEPARATOR[0]) { + tmp[len - 1] = 0; + } + +#ifndef FLB_SYSTEM_WINDOWS + for (ptr = tmp + 1; *ptr; ptr++) { +#else + for (ptr = tmp + 3; *ptr; ptr++) { +#endif + + if (*ptr == PATH_SEPARATOR[0]) { + *ptr = 0; + if (access(tmp, F_OK) != 0) { + ret = _mkdir(tmp, perms); + if (ret != 0) { + return ret; + } + } + *ptr = PATH_SEPARATOR[0]; + } + } + + return _mkdir(tmp, perms); +} From 3576d25d534b57a5631b3840257fc40364761919 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Wed, 4 Dec 2024 13:29:10 +0000 Subject: [PATCH 02/19] calyptia: add missing include Signed-off-by: Patrick Stephens --- plugins/custom_calyptia/calyptia.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/custom_calyptia/calyptia.c b/plugins/custom_calyptia/calyptia.c index 44052812fb7..dfa19a556ed 100644 --- a/plugins/custom_calyptia/calyptia.c +++ b/plugins/custom_calyptia/calyptia.c @@ -33,6 +33,8 @@ #include #include +#include + #include "calyptia.h" #define UUID_BUFFER_SIZE 38 /* Maximum length of UUID string + null terminator */ From 8e260f6a01b32db9f97dcbbe62e09a016eba3ae5 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Wed, 4 Dec 2024 13:34:27 +0000 Subject: [PATCH 03/19] calyptia: add missing include Signed-off-by: Patrick Stephens --- .../fluent-bit/calyptia/calyptia_constants.h | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 include/fluent-bit/calyptia/calyptia_constants.h diff --git a/include/fluent-bit/calyptia/calyptia_constants.h b/include/fluent-bit/calyptia/calyptia_constants.h new file mode 100644 index 00000000000..f2271b87a67 --- /dev/null +++ b/include/fluent-bit/calyptia/calyptia_constants.h @@ -0,0 +1,68 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef FLB_CALYPTIA_CONSTANTS_H +#define FLB_CALYPTIA_CONSTANTS_H + +/* End point */ +#define DEFAULT_CALYPTIA_HOST "cloud-api.calyptia.com" +#define DEFAULT_CALYPTIA_PORT "443" + +/* HTTP action types */ +#define CALYPTIA_ACTION_REGISTER 0 +#define CALYPTIA_ACTION_PATCH 1 +#define CALYPTIA_ACTION_METRICS 2 +#define CALYPTIA_ACTION_TRACE 3 + +/* Endpoints */ +#define CALYPTIA_ENDPOINT_CREATE "/v1/agents" +#define CALYPTIA_ENDPOINT_PATCH "/v1/agents/%s" +#define CALYPTIA_ENDPOINT_METRICS "/v1/agents/%s/metrics" +#define CALYPTIA_ENDPOINT_TRACE "/v1/traces/%s" + +#define CALYPTIA_ENDPOINT_FLEETS "/v1/fleets" +#define CALYPTIA_ENDPOINT_FLEET_CONFIG_INI "/v1/fleets/%s/config?format=ini" +#define CALYPTIA_ENDPOINT_FLEET_FILES "/v1/fleets/%s/files" + +/* Storage */ +#define CALYPTIA_SESSION_FILE "session.CALYPTIA" + +/* Headers */ +#define CALYPTIA_HEADERS_PROJECT "X-Project-Token" +#define CALYPTIA_HEADERS_AGENT_TOKEN "X-Agent-Token" +#define CALYPTIA_HEADERS_CTYPE "Content-Type" +#define CALYPTIA_HEADERS_CTYPE_JSON "application/json" +#define CALYPTIA_HEADERS_CTYPE_MSGPACK "application/x-msgpack" + +#ifndef FLB_SYSTEM_WINDOWS +#define FLEET_DEFAULT_CONFIG_DIR "/tmp/calyptia-fleet" +#else +#define FLEET_DEFAULT_CONFIG_DIR NULL +#endif + +#ifndef PATH_SEPARATOR +#ifndef FLB_SYSTEM_WINDOWS +#define PATH_SEPARATOR "/" +#else +#define PATH_SEPARATOR "\\" +#endif +#endif /* PATH_SEPARATOR */ + +#define CALYPTIA_MAX_DIR_SIZE 4096 + +#endif /* FLB_CALYPTIA_CONSTANTS_H */ From de4af2990778293fcef82628ba99fcee6f52a622 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Wed, 4 Dec 2024 13:54:54 +0000 Subject: [PATCH 04/19] tests: resolve conflicting definitions Signed-off-by: Patrick Stephens --- tests/runtime/custom_calyptia_input_test.c | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/runtime/custom_calyptia_input_test.c b/tests/runtime/custom_calyptia_input_test.c index ffc774c83d9..c5fb199b77d 100644 --- a/tests/runtime/custom_calyptia_input_test.c +++ b/tests/runtime/custom_calyptia_input_test.c @@ -1,7 +1,6 @@ #include #include #include -#include #include #include #include From 3e45940d814d3f28e877d5d4d5dfcf82993ab509 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Wed, 4 Dec 2024 14:04:58 +0000 Subject: [PATCH 05/19] calyptia: resolve missing bool type Signed-off-by: Patrick Stephens --- plugins/custom_calyptia/calyptia.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/custom_calyptia/calyptia.h b/plugins/custom_calyptia/calyptia.h index 0a2e61abad6..ed54552a4cf 100644 --- a/plugins/custom_calyptia/calyptia.h +++ b/plugins/custom_calyptia/calyptia.h @@ -20,6 +20,8 @@ #ifndef FLB_CALYPTIA_H #define FLB_CALYPTIA_H +#include + struct calyptia { /* config map options */ flb_sds_t api_key; From b309cf3becd752de0508864ef6af143f63dffa2f Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Wed, 4 Dec 2024 14:16:03 +0000 Subject: [PATCH 06/19] calyptia: resolve missing header Signed-off-by: Patrick Stephens --- plugins/custom_calyptia/calyptia.h | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/custom_calyptia/calyptia.h b/plugins/custom_calyptia/calyptia.h index ed54552a4cf..6f8e14c63df 100644 --- a/plugins/custom_calyptia/calyptia.h +++ b/plugins/custom_calyptia/calyptia.h @@ -20,6 +20,7 @@ #ifndef FLB_CALYPTIA_H #define FLB_CALYPTIA_H +#include #include struct calyptia { From 89ac5a91c0934a27fd5c17723ec041805d00245e Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Wed, 4 Dec 2024 14:30:05 +0000 Subject: [PATCH 07/19] calyptia: resolve duplicate symbols for tests Signed-off-by: Patrick Stephens --- plugins/custom_calyptia/calyptia.c | 7 +++++++ plugins/custom_calyptia/calyptia.h | 8 -------- tests/runtime/custom_calyptia_input_test.c | 2 ++ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/plugins/custom_calyptia/calyptia.c b/plugins/custom_calyptia/calyptia.c index dfa19a556ed..f6f81160078 100644 --- a/plugins/custom_calyptia/calyptia.c +++ b/plugins/custom_calyptia/calyptia.c @@ -39,6 +39,13 @@ #define UUID_BUFFER_SIZE 38 /* Maximum length of UUID string + null terminator */ +/* Function wrappers to enable mocking for unit test filesystem access */ +int (*flb_access)(const char *pathname, int mode) = access; +int (*flb_open)(const char *pathname, int flags, ...) = open; +ssize_t (*flb_write)(int fd, const void *buf, size_t count) = write; +int (*flb_close)(int fd) = close; +int (*flb_utils_read_file_wrapper)(char *path, char **out_buf, size_t *out_size) = flb_utils_read_file; + /* * Check if the key belongs to a sensitive data field, if so report it. We never * share any sensitive data. diff --git a/plugins/custom_calyptia/calyptia.h b/plugins/custom_calyptia/calyptia.h index 6f8e14c63df..22937c157ec 100644 --- a/plugins/custom_calyptia/calyptia.h +++ b/plugins/custom_calyptia/calyptia.h @@ -20,7 +20,6 @@ #ifndef FLB_CALYPTIA_H #define FLB_CALYPTIA_H -#include #include struct calyptia { @@ -65,11 +64,4 @@ static flb_sds_t agent_config_filename(struct calyptia *ctx, char *fname); /* These are unique to the agent rather than the fleet */ #define machine_id_fleet_config_filename(a) agent_config_filename((a), "machine-id") -/* Function wrappers to enable mocking for unit test filesystem access */ -int (*flb_access)(const char *pathname, int mode) = access; -int (*flb_open)(const char *pathname, int flags, ...) = open; -ssize_t (*flb_write)(int fd, const void *buf, size_t count) = write; -int (*flb_close)(int fd) = close; -int (*flb_utils_read_file_wrapper)(char *path, char **out_buf, size_t *out_size) = flb_utils_read_file; - #endif /* FLB_CALYPTIA_H */ diff --git a/tests/runtime/custom_calyptia_input_test.c b/tests/runtime/custom_calyptia_input_test.c index c5fb199b77d..068189bb042 100644 --- a/tests/runtime/custom_calyptia_input_test.c +++ b/tests/runtime/custom_calyptia_input_test.c @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include "flb_tests_runtime.h" #include "../../plugins/custom_calyptia/calyptia.h" From 353364ebc9e21b10d5a471a07ce6678fecd1c618 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Wed, 4 Dec 2024 14:49:39 +0000 Subject: [PATCH 08/19] calyptia: resolve function pointer usage Signed-off-by: Patrick Stephens --- plugins/custom_calyptia/calyptia.h | 7 +++++++ tests/runtime/custom_calyptia_input_test.c | 1 + 2 files changed, 8 insertions(+) diff --git a/plugins/custom_calyptia/calyptia.h b/plugins/custom_calyptia/calyptia.h index 22937c157ec..534136b6b31 100644 --- a/plugins/custom_calyptia/calyptia.h +++ b/plugins/custom_calyptia/calyptia.h @@ -64,4 +64,11 @@ static flb_sds_t agent_config_filename(struct calyptia *ctx, char *fname); /* These are unique to the agent rather than the fleet */ #define machine_id_fleet_config_filename(a) agent_config_filename((a), "machine-id") +/* Function wrappers to enable mocking for unit test filesystem access */ +int (*flb_access)(const char *pathname, int mode); +int (*flb_open)(const char *pathname, int flags, ...); +ssize_t (*flb_write)(int fd, const void *buf, size_t count); +int (*flb_close)(int fd); +int (*flb_utils_read_file_wrapper)(char *path, char **out_buf, size_t *out_size); + #endif /* FLB_CALYPTIA_H */ diff --git a/tests/runtime/custom_calyptia_input_test.c b/tests/runtime/custom_calyptia_input_test.c index 068189bb042..a09fe1c7c7d 100644 --- a/tests/runtime/custom_calyptia_input_test.c +++ b/tests/runtime/custom_calyptia_input_test.c @@ -6,6 +6,7 @@ #include #include #include +#include #include "flb_tests_runtime.h" #include "../../plugins/custom_calyptia/calyptia.h" From 80220bc7d173349536a7b56580a54adbb4dbd2f9 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Wed, 4 Dec 2024 14:56:36 +0000 Subject: [PATCH 09/19] calyptia: update codeowners Signed-off-by: Patrick Stephens --- CODEOWNERS | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index 2b5b404a281..d52de3f298b 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -91,6 +91,8 @@ # Devcontainer /.devcontainer @patrick-stephens @niedbalski @edsiper -# Calytia Fleet -/plugins/custom_calyptia/ @pwhelan -/plugins/in_calyptia_fleet/ @pwhelan +# Calyptia Fleet +/include/fluent-bit/calyptia/ @pwhelan @patrick-stephens @niedbalski +/plugins/custom_calyptia/ @pwhelan @patrick-stephens @niedbalski +/plugins/custom_calyptia/ @pwhelan @patrick-stephens @niedbalski +/plugins/out_calyptia/ @pwhelan @patrick-stephens @niedbalski From 7f2e333fbf8f269e94dd1e3f2d902d94b495d862 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 5 Dec 2024 16:37:14 +0900 Subject: [PATCH 10/19] custom_calyptia: tests: Add missing declarations To link those missing declaration functions, we need to define for stub in the custom_calyptia testing files. Signed-off-by: Hiroshi Hatake --- tests/runtime/custom_calyptia_input_test.c | 7 +++++++ tests/runtime/custom_calyptia_test.c | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/tests/runtime/custom_calyptia_input_test.c b/tests/runtime/custom_calyptia_input_test.c index a09fe1c7c7d..bbeb285a11c 100644 --- a/tests/runtime/custom_calyptia_input_test.c +++ b/tests/runtime/custom_calyptia_input_test.c @@ -10,6 +10,13 @@ #include "flb_tests_runtime.h" #include "../../plugins/custom_calyptia/calyptia.h" +const char *flb_input_get_property(const char *key, + struct flb_input_instance *ins); +struct flb_input_instance *flb_input_new(struct flb_config *config, + const char *input, void *data, + int public_only); +void flb_input_instance_destroy(struct flb_input_instance *ins); + /* Test context structure */ struct test_context { struct calyptia *ctx; diff --git a/tests/runtime/custom_calyptia_test.c b/tests/runtime/custom_calyptia_test.c index 7a13fa48dd1..00d65fd3a36 100644 --- a/tests/runtime/custom_calyptia_test.c +++ b/tests/runtime/custom_calyptia_test.c @@ -5,6 +5,12 @@ #include #include "flb_tests_runtime.h" +const char *flb_input_get_property(const char *key, + struct flb_input_instance *ins); +struct flb_input_instance *flb_input_new(struct flb_config *config, + const char *input, void *data, + int public_only); + flb_sds_t custom_calyptia_pipeline_config_get(struct flb_config *ctx); void flb_custom_calyptia_pipeline_config_get_test() From 89c2ad08751e148edcb95d3c565d21b2da59b2ab Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 5 Dec 2024 17:11:40 +0900 Subject: [PATCH 11/19] custom_calyptia: Remove duplicated function declarations Signed-off-by: Hiroshi Hatake --- plugins/custom_calyptia/calyptia.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/plugins/custom_calyptia/calyptia.h b/plugins/custom_calyptia/calyptia.h index 534136b6b31..22937c157ec 100644 --- a/plugins/custom_calyptia/calyptia.h +++ b/plugins/custom_calyptia/calyptia.h @@ -64,11 +64,4 @@ static flb_sds_t agent_config_filename(struct calyptia *ctx, char *fname); /* These are unique to the agent rather than the fleet */ #define machine_id_fleet_config_filename(a) agent_config_filename((a), "machine-id") -/* Function wrappers to enable mocking for unit test filesystem access */ -int (*flb_access)(const char *pathname, int mode); -int (*flb_open)(const char *pathname, int flags, ...); -ssize_t (*flb_write)(int fd, const void *buf, size_t count); -int (*flb_close)(int fd); -int (*flb_utils_read_file_wrapper)(char *path, char **out_buf, size_t *out_size); - #endif /* FLB_CALYPTIA_H */ From 594767e80d6d039caafb005e9f923ad6d0825ac1 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Thu, 5 Dec 2024 12:26:56 +0000 Subject: [PATCH 12/19] tests: add calyptia machine id generation tests Signed-off-by: Patrick Stephens --- plugins/custom_calyptia/calyptia.h | 1 + tests/runtime/custom_calyptia_input_test.c | 89 ++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/plugins/custom_calyptia/calyptia.h b/plugins/custom_calyptia/calyptia.h index 22937c157ec..bffbd6473f6 100644 --- a/plugins/custom_calyptia/calyptia.h +++ b/plugins/custom_calyptia/calyptia.h @@ -60,6 +60,7 @@ struct calyptia { int set_fleet_input_properties(struct calyptia *ctx, struct flb_input_instance *fleet); static flb_sds_t agent_config_filename(struct calyptia *ctx, char *fname); +static flb_sds_t get_machine_id(struct calyptia *ctx); /* These are unique to the agent rather than the fleet */ #define machine_id_fleet_config_filename(a) agent_config_filename((a), "machine-id") diff --git a/tests/runtime/custom_calyptia_input_test.c b/tests/runtime/custom_calyptia_input_test.c index bbeb285a11c..1b6f0de463c 100644 --- a/tests/runtime/custom_calyptia_input_test.c +++ b/tests/runtime/custom_calyptia_input_test.c @@ -174,8 +174,97 @@ void test_set_fleet_input_properties() cleanup_test_context(t_ctx); } +static struct test_context * update_config_dir(struct test_context * t_ctx, const char* new_config_dir) { + if (!t_ctx || !new_config_dir) { + return NULL; + } + + if (t_ctx->ctx) { + if (t_ctx->ctx->fleet_config_dir) { + flb_free(t_ctx->ctx->fleet_config_dir); + } + t_ctx->ctx->fleet_config_dir = flb_strdup(new_config_dir); + return t_ctx; + } + + cleanup_test_context(t_ctx); + return NULL; +} + +static void test_calyptia_machine_id_generation() { + struct test_context *t_ctx = init_test_context(); + TEST_CHECK(t_ctx != NULL); + + /* Set config directory to default */ + t_ctx = update_config_dir(FLEET_DEFAULT_CONFIG_DIR) + TEST_CHECK(t_ctx != NULL); + + /* Test setting properties */ + int ret = set_fleet_input_properties(t_ctx->ctx, t_ctx->fleet); + TEST_CHECK(ret == 0); + + /* Verify properties were set correctly */ + const char *value; + + /* Check config_dir */ + value = flb_input_get_property("config_dir", t_ctx->fleet); + TEST_CHECK(value != NULL); + TEST_MSG("config_dir expected=%s got=%s", FLEET_DEFAULT_CONFIG_DIR, value); + TEST_CHECK(value && strcmp(value, FLEET_DEFAULT_CONFIG_DIR) == 0); + + /** + * Initial generation should create a new UUID + * Subsequent generation should reuse the previous one + * Repeat with custom directory to confirm that works too + */ + value = machine_id_fleet_config_filename(t_ctx->ctx); + const char* expectedValue = flb_sds_printf(expectedValue, "%s/machine-id.conf", FLEET_DEFAULT_CONFIG_DIR); + TEST_CHECK(value != NULL); + TEST_CHECK(expectedValue != NULL); + TEST_MSG("machine_id filename expected=%s got=%s", expectedValue, value); + TEST_CHECK(value && strcmp(value, expectedValue) == 0); + + /* generate a new machine ID and verify it is not null then store for later use */ + const char* machine_id = get_machine_id(t_ctx->ctx); + TEST_CHECK(machine_id != NULL); + + /* repeat to confirm existing UUID is maintained */ + value = get_machine_id(t_ctx->ctx); + TEST_CHECK(value != NULL); + TEST_MSG("machine_id changed, expected=%s got=%s", machine_id, value); + TEST_CHECK(value && strcmp(value, machine_id) == 0); + + /* repeat with new config directory */ + t_ctx = update_config_dir("/test/config/fleet") + TEST_CHECK(t_ctx != NULL); + + /* check we use the new directory for the filename */ + value = machine_id_fleet_config_filename(t_ctx->ctx); + expectedValue = flb_sds_printf(expectedValue, "/test/config/fleet/machine-id.conf"); + TEST_CHECK(value != NULL); + TEST_CHECK(expectedValue != NULL); + TEST_MSG("machine_id filename expected=%s got=%s", expectedValue, value); + TEST_CHECK(value && strcmp(value, expectedValue) == 0); + + /* check we generate a new value */ + value = get_machine_id(t_ctx->ctx); + TEST_CHECK(value != NULL); + TEST_MSG("machine_id did not change, expected!=%s got=%s", machine_id, value); + TEST_CHECK(value && strcmp(value, machine_id) != 0); + + machine_id = value; + /* repeat to confirm existing UUID is maintained */ + value = get_machine_id(t_ctx->ctx); + TEST_CHECK(value != NULL); + TEST_MSG("machine_id changed, expected=%s got=%s", machine_id, value); + TEST_CHECK(value && strcmp(value, machine_id) == 0); + + cleanup_test_context(t_ctx); +} + /* Define test list */ TEST_LIST = { {"set_fleet_input_properties", test_set_fleet_input_properties}, + {"machine_id_generation", test_calyptia_machine_id_generation}, {NULL, NULL} }; \ No newline at end of file From 51433a632081424af069fd49dd929bb19537207b Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Thu, 5 Dec 2024 12:42:24 +0000 Subject: [PATCH 13/19] tests: add missing header Signed-off-by: Patrick Stephens --- tests/runtime/custom_calyptia_input_test.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/runtime/custom_calyptia_input_test.c b/tests/runtime/custom_calyptia_input_test.c index 1b6f0de463c..9bf27d22368 100644 --- a/tests/runtime/custom_calyptia_input_test.c +++ b/tests/runtime/custom_calyptia_input_test.c @@ -7,6 +7,7 @@ #include #include #include +#include #include "flb_tests_runtime.h" #include "../../plugins/custom_calyptia/calyptia.h" From 277eb52abb7848c42a830e504f431de003755ff9 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Thu, 5 Dec 2024 12:45:38 +0000 Subject: [PATCH 14/19] tests: fix typos Signed-off-by: Patrick Stephens --- tests/runtime/custom_calyptia_input_test.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/runtime/custom_calyptia_input_test.c b/tests/runtime/custom_calyptia_input_test.c index 9bf27d22368..1eb4a4152f8 100644 --- a/tests/runtime/custom_calyptia_input_test.c +++ b/tests/runtime/custom_calyptia_input_test.c @@ -197,7 +197,7 @@ static void test_calyptia_machine_id_generation() { TEST_CHECK(t_ctx != NULL); /* Set config directory to default */ - t_ctx = update_config_dir(FLEET_DEFAULT_CONFIG_DIR) + t_ctx = update_config_dir(t_ctx, FLEET_DEFAULT_CONFIG_DIR); TEST_CHECK(t_ctx != NULL); /* Test setting properties */ @@ -236,7 +236,7 @@ static void test_calyptia_machine_id_generation() { TEST_CHECK(value && strcmp(value, machine_id) == 0); /* repeat with new config directory */ - t_ctx = update_config_dir("/test/config/fleet") + t_ctx = update_config_dir(t_ctx, "/test/config/fleet"); TEST_CHECK(t_ctx != NULL); /* check we use the new directory for the filename */ From b21fc3ac1cef0ed16729917e60eae2b881931065 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Thu, 5 Dec 2024 13:25:26 +0000 Subject: [PATCH 15/19] tests: fix typos Signed-off-by: Patrick Stephens --- plugins/custom_calyptia/calyptia.c | 4 ++-- plugins/custom_calyptia/calyptia.h | 4 ++-- tests/runtime/custom_calyptia_input_test.c | 9 ++++++--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/plugins/custom_calyptia/calyptia.c b/plugins/custom_calyptia/calyptia.c index f6f81160078..dba68a25f10 100644 --- a/plugins/custom_calyptia/calyptia.c +++ b/plugins/custom_calyptia/calyptia.c @@ -411,7 +411,7 @@ static flb_sds_t generate_base_agent_directory(struct calyptia *ctx, flb_sds_t * return ret; } -static flb_sds_t agent_config_filename(struct calyptia *ctx, char *fname) +flb_sds_t agent_config_filename(struct calyptia *ctx, char *fname) { flb_sds_t cfgname = NULL; flb_sds_t ret; @@ -493,7 +493,7 @@ static int create_agent_directory(struct calyptia *ctx) return 0; } -static flb_sds_t get_machine_id(struct calyptia *ctx) +flb_sds_t get_machine_id(struct calyptia *ctx) { int ret = -1; char *buf = NULL; diff --git a/plugins/custom_calyptia/calyptia.h b/plugins/custom_calyptia/calyptia.h index bffbd6473f6..5da14d80ace 100644 --- a/plugins/custom_calyptia/calyptia.h +++ b/plugins/custom_calyptia/calyptia.h @@ -59,8 +59,8 @@ struct calyptia { }; int set_fleet_input_properties(struct calyptia *ctx, struct flb_input_instance *fleet); -static flb_sds_t agent_config_filename(struct calyptia *ctx, char *fname); -static flb_sds_t get_machine_id(struct calyptia *ctx); +flb_sds_t agent_config_filename(struct calyptia *ctx, char *fname); +flb_sds_t get_machine_id(struct calyptia *ctx); /* These are unique to the agent rather than the fleet */ #define machine_id_fleet_config_filename(a) agent_config_filename((a), "machine-id") diff --git a/tests/runtime/custom_calyptia_input_test.c b/tests/runtime/custom_calyptia_input_test.c index 1eb4a4152f8..427a73fef27 100644 --- a/tests/runtime/custom_calyptia_input_test.c +++ b/tests/runtime/custom_calyptia_input_test.c @@ -18,6 +18,9 @@ struct flb_input_instance *flb_input_new(struct flb_config *config, int public_only); void flb_input_instance_destroy(struct flb_input_instance *ins); +flb_sds_t agent_config_filename(struct calyptia *ctx, char *fname); +flb_sds_t get_machine_id(struct calyptia *ctx); + /* Test context structure */ struct test_context { struct calyptia *ctx; @@ -205,7 +208,7 @@ static void test_calyptia_machine_id_generation() { TEST_CHECK(ret == 0); /* Verify properties were set correctly */ - const char *value; + const char *value, *expectedValue, *machine_id; /* Check config_dir */ value = flb_input_get_property("config_dir", t_ctx->fleet); @@ -219,14 +222,14 @@ static void test_calyptia_machine_id_generation() { * Repeat with custom directory to confirm that works too */ value = machine_id_fleet_config_filename(t_ctx->ctx); - const char* expectedValue = flb_sds_printf(expectedValue, "%s/machine-id.conf", FLEET_DEFAULT_CONFIG_DIR); + expectedValue = flb_sds_printf(expectedValue, "%s/machine-id.conf", FLEET_DEFAULT_CONFIG_DIR); TEST_CHECK(value != NULL); TEST_CHECK(expectedValue != NULL); TEST_MSG("machine_id filename expected=%s got=%s", expectedValue, value); TEST_CHECK(value && strcmp(value, expectedValue) == 0); /* generate a new machine ID and verify it is not null then store for later use */ - const char* machine_id = get_machine_id(t_ctx->ctx); + machine_id = get_machine_id(t_ctx->ctx); TEST_CHECK(machine_id != NULL); /* repeat to confirm existing UUID is maintained */ From 7092fb53f2bdaac08314e1785902ca0cb54aa8f9 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Thu, 5 Dec 2024 13:58:17 +0000 Subject: [PATCH 16/19] tests: fix memory issues Signed-off-by: Patrick Stephens --- lib/cprofiles/include/cprofiles/cprof_info.h | 5 +-- tests/runtime/custom_calyptia_input_test.c | 46 ++++++++++++-------- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/lib/cprofiles/include/cprofiles/cprof_info.h b/lib/cprofiles/include/cprofiles/cprof_info.h index d0e26fcfbfe..10f06c905cd 100644 --- a/lib/cprofiles/include/cprofiles/cprof_info.h +++ b/lib/cprofiles/include/cprofiles/cprof_info.h @@ -20,12 +20,9 @@ #ifndef CPROF_INFO_H #define CPROF_INFO_H -#define CPROF_SOURCE_DIR "/Users/leonardo/Work/Calyptia/fluent-bit" +#define CPROF_SOURCE_DIR "/src/fluent-bit" /* General flags set by /CMakeLists.txt */ -#ifndef CPROF_HAVE_SANITIZE_ADDRESS -#define CPROF_HAVE_SANITIZE_ADDRESS -#endif #ifndef CPROF_HAVE_TIMESPEC_GET #define CPROF_HAVE_TIMESPEC_GET #endif diff --git a/tests/runtime/custom_calyptia_input_test.c b/tests/runtime/custom_calyptia_input_test.c index 427a73fef27..5a0fc0e8436 100644 --- a/tests/runtime/custom_calyptia_input_test.c +++ b/tests/runtime/custom_calyptia_input_test.c @@ -208,7 +208,7 @@ static void test_calyptia_machine_id_generation() { TEST_CHECK(ret == 0); /* Verify properties were set correctly */ - const char *value, *expectedValue, *machine_id; + const char *value; /* Check config_dir */ value = flb_input_get_property("config_dir", t_ctx->fleet); @@ -221,48 +221,60 @@ static void test_calyptia_machine_id_generation() { * Subsequent generation should reuse the previous one * Repeat with custom directory to confirm that works too */ + char expectedValue[CALYPTIA_MAX_DIR_SIZE]; + ret = sprintf(expectedValue, "%s/machine-id.conf", FLEET_DEFAULT_CONFIG_DIR); + TEST_CHECK(ret > 0); + value = machine_id_fleet_config_filename(t_ctx->ctx); - expectedValue = flb_sds_printf(expectedValue, "%s/machine-id.conf", FLEET_DEFAULT_CONFIG_DIR); TEST_CHECK(value != NULL); TEST_CHECK(expectedValue != NULL); TEST_MSG("machine_id filename expected=%s got=%s", expectedValue, value); TEST_CHECK(value && strcmp(value, expectedValue) == 0); /* generate a new machine ID and verify it is not null then store for later use */ - machine_id = get_machine_id(t_ctx->ctx); + flb_sds_t machine_id = get_machine_id(t_ctx->ctx); TEST_CHECK(machine_id != NULL); /* repeat to confirm existing UUID is maintained */ - value = get_machine_id(t_ctx->ctx); - TEST_CHECK(value != NULL); - TEST_MSG("machine_id changed, expected=%s got=%s", machine_id, value); - TEST_CHECK(value && strcmp(value, machine_id) == 0); + flb_sds_t new_machine_id = get_machine_id(t_ctx->ctx); + TEST_CHECK(new_machine_id != NULL); + TEST_MSG("machine_id changed, expected=%s got=%s", machine_id, new_machine_id); + TEST_CHECK(value && strcmp(new_machine_id, machine_id) == 0); + flb_free(new_machine_id); + new_machine_id = NULL; /* repeat with new config directory */ t_ctx = update_config_dir(t_ctx, "/test/config/fleet"); TEST_CHECK(t_ctx != NULL); /* check we use the new directory for the filename */ + ret = sprintf(expectedValue, "/test/config/fleet/machine-id.conf"); + TEST_CHECK(ret > 0); + value = machine_id_fleet_config_filename(t_ctx->ctx); - expectedValue = flb_sds_printf(expectedValue, "/test/config/fleet/machine-id.conf"); TEST_CHECK(value != NULL); TEST_CHECK(expectedValue != NULL); TEST_MSG("machine_id filename expected=%s got=%s", expectedValue, value); TEST_CHECK(value && strcmp(value, expectedValue) == 0); /* check we generate a new value */ - value = get_machine_id(t_ctx->ctx); - TEST_CHECK(value != NULL); - TEST_MSG("machine_id did not change, expected!=%s got=%s", machine_id, value); - TEST_CHECK(value && strcmp(value, machine_id) != 0); + new_machine_id = get_machine_id(t_ctx->ctx); + TEST_CHECK(new_machine_id != NULL); + TEST_MSG("machine_id did not change, expected!=%s got=%s", machine_id, new_machine_id); + TEST_CHECK(new_machine_id && strcmp(new_machine_id, machine_id) != 0); + + flb_free(machine_id); + machine_id = new_machine_id; + new_machine_id = NULL; - machine_id = value; /* repeat to confirm existing UUID is maintained */ - value = get_machine_id(t_ctx->ctx); - TEST_CHECK(value != NULL); - TEST_MSG("machine_id changed, expected=%s got=%s", machine_id, value); - TEST_CHECK(value && strcmp(value, machine_id) == 0); + new_machine_id = get_machine_id(t_ctx->ctx); + TEST_CHECK(new_machine_id != NULL); + TEST_MSG("machine_id changed, expected=%s got=%s", machine_id, new_machine_id); + TEST_CHECK(new_machine_id && strcmp(new_machine_id, machine_id) == 0); + flb_free(new_machine_id); + flb_free(machine_id); cleanup_test_context(t_ctx); } From cb0a847d7d5d965cc8b13eb7789d1714f77df006 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Thu, 5 Dec 2024 14:06:42 +0000 Subject: [PATCH 17/19] tests: fix location Signed-off-by: Patrick Stephens --- tests/runtime/custom_calyptia_input_test.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/runtime/custom_calyptia_input_test.c b/tests/runtime/custom_calyptia_input_test.c index 5a0fc0e8436..d79e81f34db 100644 --- a/tests/runtime/custom_calyptia_input_test.c +++ b/tests/runtime/custom_calyptia_input_test.c @@ -244,11 +244,11 @@ static void test_calyptia_machine_id_generation() { new_machine_id = NULL; /* repeat with new config directory */ - t_ctx = update_config_dir(t_ctx, "/test/config/fleet"); + t_ctx = update_config_dir(t_ctx, "/tmp/config/fleet"); TEST_CHECK(t_ctx != NULL); /* check we use the new directory for the filename */ - ret = sprintf(expectedValue, "/test/config/fleet/machine-id.conf"); + ret = sprintf(expectedValue, "/tmp/config/fleet/machine-id.conf"); TEST_CHECK(ret > 0); value = machine_id_fleet_config_filename(t_ctx->ctx); From 750528220233f37c4c93217e6e72b73f6eeb872c Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Thu, 5 Dec 2024 14:22:41 +0000 Subject: [PATCH 18/19] tests: fix destroy usage Signed-off-by: Patrick Stephens --- Vagrantfile | 2 +- tests/runtime/custom_calyptia_input_test.c | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 7f74824cb34..b18ea9795ee 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -28,7 +28,7 @@ Vagrant.configure("2") do |config| # Main build apt-get install --yes build-essential cmake dh-make git make openssl pkg-config tar # Dependencies - apt-get install --yes libssl3 libssl-dev libsasl2-dev pkg-config libsystemd-dev zlib1g-dev libpq-dev postgresql-server-dev-all flex bison libyaml-dev libpq5 + apt-get install --yes libssl3 libssl-dev libsasl2-dev pkg-config libsystemd-dev zlib1g-dev libpq-dev postgresql-server-dev-all flex bison libyaml-dev libpq5 libbpf-dev # Debug apt-get install --yes gdb valgrind diff --git a/tests/runtime/custom_calyptia_input_test.c b/tests/runtime/custom_calyptia_input_test.c index d79e81f34db..73a3872c7d6 100644 --- a/tests/runtime/custom_calyptia_input_test.c +++ b/tests/runtime/custom_calyptia_input_test.c @@ -240,8 +240,7 @@ static void test_calyptia_machine_id_generation() { TEST_CHECK(new_machine_id != NULL); TEST_MSG("machine_id changed, expected=%s got=%s", machine_id, new_machine_id); TEST_CHECK(value && strcmp(new_machine_id, machine_id) == 0); - flb_free(new_machine_id); - new_machine_id = NULL; + flb_sds_destroy(new_machine_id); /* repeat with new config directory */ t_ctx = update_config_dir(t_ctx, "/tmp/config/fleet"); @@ -263,9 +262,8 @@ static void test_calyptia_machine_id_generation() { TEST_MSG("machine_id did not change, expected!=%s got=%s", machine_id, new_machine_id); TEST_CHECK(new_machine_id && strcmp(new_machine_id, machine_id) != 0); - flb_free(machine_id); + flb_sds_destroy(machine_id); machine_id = new_machine_id; - new_machine_id = NULL; /* repeat to confirm existing UUID is maintained */ new_machine_id = get_machine_id(t_ctx->ctx); @@ -273,8 +271,8 @@ static void test_calyptia_machine_id_generation() { TEST_MSG("machine_id changed, expected=%s got=%s", machine_id, new_machine_id); TEST_CHECK(new_machine_id && strcmp(new_machine_id, machine_id) == 0); - flb_free(new_machine_id); - flb_free(machine_id); + flb_sds_destroy(new_machine_id); + flb_sds_destroy(machine_id); cleanup_test_context(t_ctx); } From d4e14713a07931989f89fd8f8d38194255ee63ae Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Thu, 5 Dec 2024 15:24:25 +0000 Subject: [PATCH 19/19] tests: fix destroy usage Signed-off-by: Patrick Stephens --- tests/runtime/custom_calyptia_input_test.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/runtime/custom_calyptia_input_test.c b/tests/runtime/custom_calyptia_input_test.c index 73a3872c7d6..5cf023afa76 100644 --- a/tests/runtime/custom_calyptia_input_test.c +++ b/tests/runtime/custom_calyptia_input_test.c @@ -225,11 +225,11 @@ static void test_calyptia_machine_id_generation() { ret = sprintf(expectedValue, "%s/machine-id.conf", FLEET_DEFAULT_CONFIG_DIR); TEST_CHECK(ret > 0); - value = machine_id_fleet_config_filename(t_ctx->ctx); - TEST_CHECK(value != NULL); - TEST_CHECK(expectedValue != NULL); - TEST_MSG("machine_id filename expected=%s got=%s", expectedValue, value); - TEST_CHECK(value && strcmp(value, expectedValue) == 0); + flb_sds_t filename = machine_id_fleet_config_filename(t_ctx->ctx); + TEST_CHECK(filename != NULL); + TEST_MSG("machine_id filename expected=%s got=%s", expectedValue, filename); + TEST_CHECK(filename && strcmp(filename, expectedValue) == 0); + flb_sds_destroy(filename); /* generate a new machine ID and verify it is not null then store for later use */ flb_sds_t machine_id = get_machine_id(t_ctx->ctx); @@ -250,11 +250,11 @@ static void test_calyptia_machine_id_generation() { ret = sprintf(expectedValue, "/tmp/config/fleet/machine-id.conf"); TEST_CHECK(ret > 0); - value = machine_id_fleet_config_filename(t_ctx->ctx); - TEST_CHECK(value != NULL); - TEST_CHECK(expectedValue != NULL); - TEST_MSG("machine_id filename expected=%s got=%s", expectedValue, value); - TEST_CHECK(value && strcmp(value, expectedValue) == 0); + filename = machine_id_fleet_config_filename(t_ctx->ctx); + TEST_CHECK(filename != NULL); + TEST_MSG("machine_id filename expected=%s got=%s", expectedValue, filename); + TEST_CHECK(filename && strcmp(filename, expectedValue) == 0); + flb_sds_destroy(filename); /* check we generate a new value */ new_machine_id = get_machine_id(t_ctx->ctx);