diff --git a/auto/help b/auto/help index 6a6aee19e..948547623 100644 --- a/auto/help +++ b/auto/help @@ -52,6 +52,8 @@ cat << END --njs enable njs library usage + --otel enable otel library usage + --debug enable debug logging --fuzz=ENGINE enable fuzz testing diff --git a/auto/make b/auto/make index f21a2dfc5..e7d29ba30 100644 --- a/auto/make +++ b/auto/make @@ -7,6 +7,11 @@ $echo "creating $NXT_MAKEFILE" +if [ $NXT_OTEL = "NO" ]; then + NXT_OTEL_LIB_LOC= + NXT_OTEL_BUILD_FLAG= + NXT_OTEL_LIB_DIR= +fi cat << END > $NXT_MAKEFILE @@ -138,14 +143,14 @@ cat << END >> $NXT_MAKEFILE libnxt: $NXT_BUILD_DIR/lib/$NXT_LIB_SHARED $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC -$NXT_BUILD_DIR/lib/$NXT_LIB_SHARED: \$(NXT_LIB_OBJS) +$NXT_BUILD_DIR/lib/$NXT_LIB_SHARED: \$(NXT_LIB_OBJS) $NXT_OTEL_LIB_LOC \$(PP_LD) \$@ \$(v)\$(NXT_SHARED_LOCAL_LINK) -o \$@ \$(NXT_LIB_OBJS) \\ - $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS + $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS $NXT_OTEL_LIB_LOC -$NXT_BUILD_DIR/lib/$NXT_LIB_STATIC: \$(NXT_LIB_OBJS) +$NXT_BUILD_DIR/lib/$NXT_LIB_STATIC: \$(NXT_LIB_OBJS) $NXT_OTEL_LIB_LOC \$(PP_AR) \$@ - \$(v)$NXT_STATIC_LINK \$@ \$(NXT_LIB_OBJS) + \$(v)$NXT_STATIC_LINK \$@ \$(NXT_LIB_OBJS) $NXT_OTEL_LIB_LOC $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC: \$(NXT_LIB_UNIT_OBJS) \\ $NXT_BUILD_DIR/share/pkgconfig/unit.pc \\ @@ -359,11 +364,11 @@ $echo >> $NXT_MAKEFILE cat << END >> $NXT_MAKEFILE $NXT_BUILD_DIR/sbin/$NXT_DAEMON: $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \\ - \$(NXT_OBJS) + \$(NXT_OBJS) $NXT_OTEL_LIB_LOC \$(PP_LD) \$@ \$(v)\$(NXT_EXEC_LINK) -o \$@ \$(CFLAGS) \\ \$(NXT_OBJS) $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \\ - $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS + $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS $NXT_OTEL_LIB_LOC END @@ -535,10 +540,6 @@ cat << END > Makefile include $NXT_MAKEFILE -.PHONY: clean -clean: - rm -rf $NXT_BUILD_DIR *.dSYM Makefile - .PHONY: help help: @echo "Variables to control make/build behaviour:" @@ -551,4 +552,22 @@ help: @echo @echo " Variables can be combined." +.PHONY: clean +clean: + rm -rf $NXT_BUILD_DIR *.dSYM Makefile +END + +if [ $NXT_OTEL = YES ]; then + cat << END >> Makefile + cd "$NXT_OTEL_LIB_DIR" && cargo clean +END + + cat << END >> $NXT_MAKEFILE + +$NXT_OTEL_LIB_LOC: + cd src/otel/ && \ + cargo build $NXT_OTEL_BUILD_FLAG && \ + cd ../../ + END +fi diff --git a/auto/options b/auto/options index 5be1ebe18..7aa7a73a9 100644 --- a/auto/options +++ b/auto/options @@ -27,6 +27,7 @@ NXT_CYASSL=NO NXT_POLARSSL=NO NXT_NJS=NO +NXT_OTEL=NO NXT_TEST_BUILD_EPOLL=NO NXT_TEST_BUILD_EVENTPORT=NO @@ -112,6 +113,7 @@ do --polarssl) NXT_POLARSSL=YES ;; --njs) NXT_NJS=YES ;; + --otel) NXT_OTEL=YES ;; --test-build-epoll) NXT_TEST_BUILD_EPOLL=YES ;; --test-build-eventport) NXT_TEST_BUILD_EVENTPORT=YES ;; diff --git a/auto/otel b/auto/otel new file mode 100644 index 000000000..8f6fb8944 --- /dev/null +++ b/auto/otel @@ -0,0 +1,27 @@ + +# Copyright (C) NGINX, Inc. + +if [ $NXT_DEBUG = YES ]; then + NXT_OTEL_LIB_DIR=src/otel/ + NXT_OTEL_LIB_LOC=src/otel/target/debug/libotel.a + NXT_OTEL_BUILD_FLAG="" +else + NXT_OTEL_LIB_DIR=src/otel/ + NXT_OTEL_LIB_LOC=src/otel/target/release/libotel.a + NXT_OTEL_BUILD_FLAG="--release" +fi + +if [ $NXT_OTEL ]; then + NXT_OTEL_LIBS="-lssl -lcrypto" + if [ $(which pkgconf) ]; then + NXT_OTEL_LIBS="$(pkgconf openssl --cflags --libs)" + elif [ $(which pkg-config) ]; then + NXT_OTEL_LIBS="$(pkg-config openssl --cflags --libs)" + fi + NXT_LIB_AUX_LIBS="$NXT_LIB_AUX_LIBS $NXT_OTEL_LIBS" + cat << END >> $NXT_AUTO_CONFIG_H +#ifndef NXT_HAVE_OTEL +#define NXT_HAVE_OTEL 1 +#endif +END +fi diff --git a/auto/sources b/auto/sources index dfabf7cf2..027403970 100644 --- a/auto/sources +++ b/auto/sources @@ -127,6 +127,10 @@ if [ "$NXT_NJS" != "NO" ]; then NXT_LIB_SRCS="$NXT_LIB_SRCS src/nxt_js.c src/nxt_http_js.c src/nxt_script.c" fi +if [ "$NXT_OTEL" != "NO" ]; then + NXT_LIB_SRCS="$NXT_LIB_SRCS src/nxt_otel.c" +fi + NXT_LIB_EPOLL_SRCS="src/nxt_epoll_engine.c" NXT_LIB_KQUEUE_SRCS="src/nxt_kqueue_engine.c" NXT_LIB_EVENTPORT_SRCS="src/nxt_eventport_engine.c" diff --git a/auto/summary b/auto/summary index b6caee6c6..eba88be4c 100644 --- a/auto/summary +++ b/auto/summary @@ -30,6 +30,7 @@ Unit configuration summary: TLS support: ............... $NXT_OPENSSL Regex support: ............. $NXT_REGEX njs support: ............... $NXT_NJS + otel support: .............. $NXT_OTEL process isolation: ......... $NXT_ISOLATION cgroupv2: .................. $NXT_HAVE_CGROUP diff --git a/configure b/configure index 6929d41da..d8eb1ea3b 100755 --- a/configure +++ b/configure @@ -179,6 +179,10 @@ if [ $NXT_NJS != NO ]; then . auto/njs fi +if [ $NXT_OTEL != NO ]; then + . auto/otel +fi + . auto/make . auto/fuzzing . auto/summary diff --git a/src/nxt_http.h b/src/nxt_http.h index 5369c8e16..bc76d67eb 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -9,6 +9,9 @@ #include +#if (NXT_HAVE_OTEL) +#include +#endif typedef enum { NXT_HTTP_UNSET = -1, @@ -190,6 +193,10 @@ struct nxt_http_request_s { nxt_http_response_t resp; +#if (NXT_HAVE_OTEL) + nxt_otel_state_t *otel; +#endif + nxt_http_status_t status:16; uint8_t log_route; /* 1 bit */ diff --git a/src/nxt_http_error.c b/src/nxt_http_error.c index 370b12dbc..be5aef101 100644 --- a/src/nxt_http_error.c +++ b/src/nxt_http_error.c @@ -8,6 +8,11 @@ #include +#if (NXT_HAVE_OTEL) +#include +#endif + + static void nxt_http_request_send_error_body(nxt_task_t *task, void *r, void *data); @@ -55,6 +60,10 @@ nxt_http_request_error(nxt_task_t *task, nxt_http_request_t *r, r->resp.content_length = NULL; r->resp.content_length_n = NXT_HTTP_ERROR_LEN; +#if (NXT_HAVE_OTEL) + nxt_otel_request_error_path(task, r); +#endif + r->state = &nxt_http_request_send_error_body_state; nxt_http_request_header_send(task, r, diff --git a/src/nxt_http_request.c b/src/nxt_http_request.c index a7e9ff69a..d32226991 100644 --- a/src/nxt_http_request.c +++ b/src/nxt_http_request.c @@ -283,7 +283,15 @@ nxt_http_request_create(nxt_task_t *task) task->thread->engine->requests_cnt++; r->tstr_cache.var.pool = mp; - +#if (NXT_HAVE_OTEL) + if (nxt_otel_rs_is_init()) { + r->otel = nxt_mp_zget(r->mem_pool, sizeof(nxt_otel_state_t)); + if (r->otel == NULL) { + goto fail; + } + r->otel->status = NXT_OTEL_INIT_STATE; + } +#endif return r; fail: diff --git a/src/nxt_otel.c b/src/nxt_otel.c new file mode 100644 index 000000000..5595e8210 --- /dev/null +++ b/src/nxt_otel.c @@ -0,0 +1,403 @@ + +/* + * Copyright (C) F5, Inc. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#define NXT_OTEL_TRACEPARENT_LEN 55 +#define NXT_OTEL_BODY_SIZE_TAG "body size" +#define NXT_OTEL_METHOD_TAG "method" +#define NXT_OTEL_PATH_TAG "path" +#define NXT_OTEL_STATUS_CODE_TAG "status" + + +void +nxt_otel_state_transition(nxt_otel_state_t *state, nxt_otel_status_t status) +{ + if (status == NXT_OTEL_ERROR_STATE || state->status != NXT_OTEL_ERROR_STATE) { + state->status = status; + } +} + + +void +nxt_otel_propagate_header(nxt_task_t *task, nxt_http_request_t *r) +{ + u_char *traceval; + nxt_str_t traceparent_name, traceparent; + nxt_http_field_t *f; + + traceval = nxt_mp_zalloc(r->mem_pool, NXT_OTEL_TRACEPARENT_LEN + 1); + if (traceval == NULL) { + /* let it go blank here. + * span still gets populated and sent + * but data is not propagated to peer or app. + */ + nxt_log(task, NXT_LOG_ERR, + "couldnt allocate traceparent header. span will not propagate"); + return; + } + + if (r->otel->trace_id != NULL) { + // copy in the pre-existing traceparent for the response + sprintf((char *) traceval, "%s-%s-%s-%s", + (char *) r->otel->version, + (char *) r->otel->trace_id, + (char *) r->otel->parent_id, + (char *) r->otel->trace_flags); + + // if we didnt inherit a trace id then we need to add the + // traceparent header to the request + } else if (r->otel->trace_id == NULL) { + nxt_otel_rs_copy_traceparent(traceval, r->otel->trace); + f = nxt_list_add(r->fields); + if (f == NULL) { + return; + } + + nxt_http_field_name_set(f, "traceparent"); + f->value = traceval; + f->value_length = nxt_strlen(traceval); + traceparent_name = (nxt_str_t){ + .start = f->name, + .length = f->name_length, + }; + traceparent = (nxt_str_t){ + .start = f->value, + .length = f->value_length, + }; + nxt_otel_rs_add_event_to_trace(r->otel->trace, + &traceparent_name, + &traceparent); + + // potentially nxt_http_request_error called before headers finished parsing + } else { + nxt_log(task, NXT_LOG_DEBUG, + "not propagating tracing headers for missing trace"); + return; + } + + f = nxt_list_add(r->resp.fields); + if (f == NULL) { + nxt_log(task, NXT_LOG_ERR, + "couldnt allocate traceparent header in response"); + return; + } + + nxt_http_field_name_set(f, "traceparent"); + f->value = traceval; + f->value_length = nxt_strlen(traceval); +} + + +void +nxt_otel_span_add_headers(nxt_task_t *task, nxt_http_request_t *r) +{ + nxt_str_t method_name, path_name; + nxt_http_field_t *cur; + + nxt_log(task, NXT_LOG_DEBUG, "adding headers to trace"); + + if (r->otel == NULL || r->otel->trace == NULL) + { + nxt_log(task, NXT_LOG_ERR, "no trace to add events to!"); + nxt_otel_state_transition(r->otel, NXT_OTEL_ERROR_STATE); + return; + } + + nxt_list_each(cur, r->fields) { + nxt_str_t name, val; + name = (nxt_str_t){ + .start = cur->name, + .length = cur->name_length, + }; + + val = (nxt_str_t){ + .start = cur->value, + .length = cur->value_length, + }; + + nxt_otel_rs_add_event_to_trace(r->otel->trace, &name, &val); + } nxt_list_loop; + + method_name = (nxt_str_t){ + .start = (u_char *) NXT_OTEL_METHOD_TAG, + .length = nxt_length(NXT_OTEL_METHOD_TAG) + 1, + }; + nxt_otel_rs_add_event_to_trace(r->otel->trace, &method_name, r->method); + + path_name = (nxt_str_t){ + .start = (u_char *) NXT_OTEL_PATH_TAG, + .length = nxt_length(NXT_OTEL_PATH_TAG) + 1, + }; + nxt_otel_rs_add_event_to_trace(r->otel->trace, &path_name, r->path); + nxt_otel_propagate_header(task, r); + + nxt_otel_state_transition(r->otel, NXT_OTEL_BODY_STATE); +} + + +void +nxt_otel_span_add_body(nxt_http_request_t *r) +{ + size_t body_size, buf_size; + u_char *body_buf, *body_size_buf; + nxt_str_t body_key, body_val; + nxt_int_t cur; + + if (r->body != NULL) { + body_size = nxt_buf_used_size(r->body); + } else { + body_size = 0; + } + + buf_size = 1; // first digit + if (body_size != 0) { + buf_size += log10(body_size); // subsequent digits + } + buf_size += 1; // \0 + buf_size += nxt_length(NXT_OTEL_BODY_SIZE_TAG); + buf_size += 1; // \0 + + body_buf = nxt_mp_zalloc(r->mem_pool, buf_size); + if (body_buf == NULL) { + return; + } + + cur = sprintf((char *) body_buf, "%lu", body_size); + if (cur < 0) { + return; + } + + // (already was zero-alloc'ed) + //body_buf[cur] = '\0'; + cur += 1; + body_size_buf = body_buf + cur; + + nxt_cpystr(body_buf + cur, (const u_char *) NXT_OTEL_BODY_SIZE_TAG); + // (already was zero-alloc'ed) + //body_buf[cur] = '\0'; + + body_key = (nxt_str_t){ + .start = body_size_buf, + .length = nxt_length(body_size_buf), + }; + body_val = (nxt_str_t){ + .start = body_buf, + .length = nxt_length(body_buf), + }; + + nxt_otel_rs_add_event_to_trace(r->otel->trace, &body_key, &body_val); + nxt_otel_state_transition(r->otel, NXT_OTEL_COLLECT_STATE); +} + + +void +nxt_otel_span_add_status(nxt_task_t *task, nxt_http_request_t *r) { + u_char *status_buf; + nxt_str_t status_key, status_val; + + // dont bother logging an unset status + if (r->status == 0) { + return; + } + + // add specific 3 character status to span + status_buf = nxt_mp_zalloc(r->mem_pool, 4); + if (status_buf == NULL) { + return; + } + + sprintf((char *) status_buf, "%i", r->status); + + // set up event + status_key = (nxt_str_t){ + .start = (u_char *) NXT_OTEL_STATUS_CODE_TAG, + .length = nxt_length(NXT_OTEL_STATUS_CODE_TAG), + }; + status_val = (nxt_str_t){ + .start = status_buf, + .length = 4, + }; + nxt_otel_rs_add_event_to_trace(r->otel->trace, &status_key, &status_val); +} + + +void +nxt_otel_span_collect(nxt_task_t *task, nxt_http_request_t *r) +{ + if (r->otel->trace == NULL) { + nxt_log(task, NXT_LOG_ERR, "otel error: no trace to send!"); + nxt_otel_state_transition(r->otel, NXT_OTEL_ERROR_STATE); + return; + } + + nxt_otel_span_add_status(task, r); + nxt_otel_state_transition(r->otel, NXT_OTEL_UNINIT_STATE); + nxt_otel_rs_send_trace(r->otel->trace); + + r->otel->trace = NULL; +} + + +void +nxt_otel_error(nxt_task_t *task, nxt_http_request_t *r) +{ + // purposefully not using state transition helper + r->otel->status = NXT_OTEL_UNINIT_STATE; + nxt_log(task, NXT_LOG_ERR, "otel error condition"); + + /* assumable at time of writing that there is no + * r->otel->trace to leak. This state is only set + * in cases where trace fails to generate or is missing + */ +} + + +void +nxt_otel_trace_and_span_init(nxt_task_t *task, nxt_http_request_t *r) +{ + r->otel->trace = + nxt_otel_rs_get_or_create_trace(r->otel->trace_id); + if (r->otel->trace == NULL) { + nxt_log(task, NXT_LOG_ERR, "error generating otel span"); + nxt_otel_state_transition(r->otel, NXT_OTEL_ERROR_STATE); + return; + } + + nxt_otel_state_transition(r->otel, NXT_OTEL_HEADER_STATE); +} + + +void +nxt_otel_test_and_call_state(nxt_task_t *task, nxt_http_request_t *r) +{ + if (r == NULL || r->otel == NULL) { + return; + } + + switch (r->otel->status) { + case NXT_OTEL_UNINIT_STATE: + return; + case NXT_OTEL_INIT_STATE: + nxt_otel_trace_and_span_init(task, r); + break; + case NXT_OTEL_HEADER_STATE: + nxt_otel_span_add_headers(task, r); + break; + case NXT_OTEL_BODY_STATE: + nxt_otel_span_add_body(r); + break; + case NXT_OTEL_COLLECT_STATE: + nxt_otel_span_collect(task, r); + break; + case NXT_OTEL_ERROR_STATE: + nxt_otel_error(task, r); + break; + } +} + + +// called in nxt_http_request_error +void +nxt_otel_request_error_path(nxt_task_t *task, nxt_http_request_t *r) { + if (r->otel->trace == NULL) { + return; + } + + // response headers have been cleared + nxt_otel_propagate_header(task, r); + + // collect span immediately + if (r->otel) { + nxt_otel_state_transition(r->otel, NXT_OTEL_COLLECT_STATE); + } + nxt_otel_test_and_call_state(task, r); +} + + +nxt_int_t +nxt_otel_parse_traceparent(void *ctx, nxt_http_field_t *field, uintptr_t data) +{ + char *copy; + nxt_http_request_t *r; + + /* For information on parsing the traceparent header: + * https://www.w3.org/TR/trace-context/#traceparent-header + * A summary of the traceparent header value format follows: + * Traceparent: "$a-$b-$c-$d" + * a. version (2 hex digits) (ff is forbidden) + * b. trace_id (32 hex digits) (all zeroes forbidden) + * c. parent_id (16 hex digits) (all zeroes forbidden) + * d. flags (2 hex digits) + */ + + r = ctx; + if (field->value_length != NXT_OTEL_TRACEPARENT_LEN) { + goto error_state; + } + + /* strsep is destructive so we make a copy of the field + */ + copy = nxt_mp_zalloc(r->mem_pool, field->value_length + 1); + if (copy == NULL) { + goto error_state; + } + memcpy(copy, field->value, field->value_length); + + r->otel->version = (u_char *) strsep(©, "-"); + r->otel->trace_id = (u_char *) strsep(©, "-"); + r->otel->parent_id = (u_char *) strsep(©, "-"); + r->otel->trace_flags = (u_char *) strsep(©, "-"); + + if (r->otel->version == NULL || + r->otel->trace_id == NULL || + r->otel->parent_id == NULL || + r->otel->trace_flags == NULL) + { + goto error_state; + } + + return NXT_OK; + + error_state: + nxt_otel_state_transition(r->otel, NXT_OTEL_ERROR_STATE); + return NXT_ERROR; +} + + +nxt_int_t +nxt_otel_parse_tracestate(void *ctx, nxt_http_field_t *field, uintptr_t data) +{ + nxt_str_t s; + nxt_http_field_t *f; + nxt_http_request_t *r; + + s.length = field->value_length; + s.start = field->value; + r = ctx; + r->otel->trace_state = s; + + // maybe someday this should get sent down into the otel lib + // when we can figure out what to do with it at least + + f = nxt_list_add(r->resp.fields); + if (f != NULL) { + *f = *field; + } + + return NXT_OK; +} diff --git a/src/nxt_otel.h b/src/nxt_otel.h new file mode 100644 index 000000000..5a60dc04d --- /dev/null +++ b/src/nxt_otel.h @@ -0,0 +1,80 @@ +/* + * Copyright (C) F5, Inc. + */ + +#ifndef _NXT_OTEL_H_INCLUDED_ +#define _NXT_OTEL_H_INCLUDED_ + +#include +#include + +// forward declared +struct nxt_http_field_t; +struct nxt_conf_validation_t; +struct nxt_conf_value_t; +struct nxt_http_request_t; + +extern void nxt_otel_rs_send_trace(void *trace); +extern void * nxt_otel_rs_get_or_create_trace(u_char *trace_id); +extern void nxt_otel_rs_init(void (*log_callback)(u_char *log_string), + const nxt_str_t *endpoint, + const nxt_str_t *protocol, + double sample_fraction, + double batch_size); +extern void nxt_otel_rs_copy_traceparent(u_char *buffer, void *span); +extern void nxt_otel_rs_add_event_to_trace(void *trace, + nxt_str_t *key, + nxt_str_t *val); +extern uint8_t nxt_otel_rs_is_init(void); +extern void nxt_otel_rs_uninit(void); + + +/* nxt_otel_status_t + * more efficient than a single handler state struct + */ +typedef enum { + NXT_OTEL_UNINIT_STATE = 0, + NXT_OTEL_INIT_STATE, + NXT_OTEL_HEADER_STATE, + NXT_OTEL_BODY_STATE, + NXT_OTEL_COLLECT_STATE, + NXT_OTEL_ERROR_STATE, +} nxt_otel_status_t; + +/* nxt_otel_state_t + * cache of trace data needed per request and + * includes indicator as to current flow state + */ +typedef struct { + u_char *trace_id; + u_char *version; + u_char *parent_id; + u_char *trace_flags; + void *trace; + nxt_otel_status_t status; + nxt_str_t trace_state; +} nxt_otel_state_t; + + +nxt_int_t nxt_otel_parse_traceparent(void *ctx, + nxt_http_field_t *field, + uintptr_t data); +nxt_int_t nxt_otel_parse_tracestate(void *ctx, + nxt_http_field_t *field, + uintptr_t data); + +void nxt_otel_span_add_headers(nxt_task_t *task, nxt_http_request_t *r); +void nxt_otel_span_add_body(nxt_http_request_t *r); +void nxt_otel_span_add_status(nxt_task_t *task, nxt_http_request_t *r); +void nxt_otel_span_collect(nxt_task_t *task, nxt_http_request_t *r); + +void nxt_otel_error(nxt_task_t *task, nxt_http_request_t *r); +void nxt_otel_test_and_call_state(nxt_task_t *task, nxt_http_request_t *r); +void nxt_otel_state_transition(nxt_otel_state_t *state, + nxt_otel_status_t status); + +void nxt_otel_trace_and_span_init(nxt_task_t *task, nxt_http_request_t *r); +void nxt_otel_propagate_header(nxt_task_t *task, nxt_http_request_t *r); +void nxt_otel_request_error_path(nxt_task_t *task, nxt_http_request_t *r); + +#endif // _NXT_OTEL_H_INCLUDED_