Nginx attributes feature implementation

.vscode/settings.json

@@ -0,0 +1,60 @@
+{
+    "files.associations": {
+        "array": "cpp",
+        "atomic": "cpp",
+        "bit": "cpp",
+        "*.tcc": "cpp",
+        "cctype": "cpp",
+        "chrono": "cpp",
+        "clocale": "cpp",
+        "cmath": "cpp",
+        "condition_variable": "cpp",
+        "cstdarg": "cpp",
+        "cstddef": "cpp",
+        "cstdint": "cpp",
+        "cstdio": "cpp",
+        "cstdlib": "cpp",
+        "cstring": "cpp",
+        "ctime": "cpp",
+        "cwchar": "cpp",
+        "cwctype": "cpp",
+        "deque": "cpp",
+        "list": "cpp",
+        "map": "cpp",
+        "set": "cpp",
+        "unordered_map": "cpp",
+        "unordered_set": "cpp",
+        "vector": "cpp",
+        "exception": "cpp",
+        "algorithm": "cpp",
+        "functional": "cpp",
+        "iterator": "cpp",
+        "memory": "cpp",
+        "memory_resource": "cpp",
+        "numeric": "cpp",
+        "optional": "cpp",
+        "random": "cpp",
+        "ratio": "cpp",
+        "string": "cpp",
+        "string_view": "cpp",
+        "system_error": "cpp",
+        "tuple": "cpp",
+        "type_traits": "cpp",
+        "utility": "cpp",
+        "fstream": "cpp",
+        "initializer_list": "cpp",
+        "iomanip": "cpp",
+        "iosfwd": "cpp",
+        "iostream": "cpp",
+        "istream": "cpp",
+        "limits": "cpp",
+        "mutex": "cpp",
+        "new": "cpp",
+        "ostream": "cpp",
+        "sstream": "cpp",
+        "stdexcept": "cpp",
+        "streambuf": "cpp",
+        "thread": "cpp",
+        "typeinfo": "cpp"
+    }
+}

instrumentation/otel-webserver-module/build.gradle

@@ -8,7 +8,8 @@ apply plugin: 'de.undercouch.download'
 
 def coverageEnabled = System.getenv('enableCoverage')
 def target_system = System.getProperty('targetSystem') ?: 'centos6'
-def buildType = System.getProperty('buildType') ?: 'release'
+// def buildType = System.getProperty('buildType') ?: 'release'
+def buildType = 'debug'
 
 project.ext {
     // To do a 32-bit build, pass "-Dos.arch=x86"

instrumentation/otel-webserver-module/docker-compose.yml

@@ -83,6 +83,8 @@ services:
             - ubuntu20.04_nginx
         ports:
             - "8016:80"
+        volumes:
+            - /home/ubuntu/code/aryanOTEL/:/code/
         depends_on:
             - otel-collector
 

instrumentation/otel-webserver-module/include/core/api/Payload.h

@@ -87,11 +87,13 @@ class RequestPayload
     std::unordered_map<std::string, std::string>& get_request_headers() {
     	return request_headers;
     }
+
 };
 
 struct ResponsePayload
 {
 	std::unordered_map<std::string, std::string> response_headers;
+	std::unordered_map<std::string, std::string> otel_attributes;
 	unsigned int status_code{sdkwrapper::kStatusCodeInit};
 };
 

instrumentation/otel-webserver-module/include/core/api/opentelemetry_ngx_api.h

@@ -45,6 +45,7 @@ typedef struct {
     const char* http_post_param;
     const char* request_method;
     const char* client_ip;
+
     http_headers* propagation_headers;
     http_headers* request_headers;
 
@@ -55,6 +56,8 @@ typedef struct {
 typedef struct {
     http_headers* response_headers;
     int response_headers_count;
+    http_headers* otel_attributes;
+    int otel_attributes_count;
 
     unsigned int status_code;
 }response_payload;

instrumentation/otel-webserver-module/include/core/sdkwrapper/SdkConstants.h

@@ -39,6 +39,7 @@ const std::string kAttrNETHostPort            = "net.host.port";
 const std::string kAttrRequestProtocol        = "request_protocol";
 const std::string kHTTPFlavor1_0                 = "1.0";
 const std::string kHTTPFlavor1_1                = "1.1";
+const std::string kOtelAttributes             = "otel.attribute.";
 
 
 constexpr int HTTP_ERROR_1XX = 100;

instrumentation/otel-webserver-module/src/core/api/RequestProcessingEngine.cpp

@@ -82,6 +82,7 @@ OTEL_SDK_STATUS_CODE RequestProcessingEngine::startRequest(
                 std::string(itr->first);
         keyValueMap[key] = itr->second;
     }
+
     auto span = m_sdkWrapper->CreateSpan(spanName, sdkwrapper::SpanKind::SERVER, keyValueMap, payload->get_http_headers());
 
     LOG4CXX_TRACE(mLogger, "Span started for context: [" << wscontext
@@ -157,6 +158,11 @@ OTEL_SDK_STATUS_CODE RequestProcessingEngine::endRequest(
             rootSpan->AddAttribute(key, itr->second);
         }
 
+        for (auto itr = payload->otel_attributes.begin(); itr != payload->otel_attributes.end(); itr++) {
+            std::string key = std::string(kOtelAttributes) + std::string(itr->first);
+            rootSpan->AddAttribute(key, itr->second);
+        }
+
         rootSpan->AddAttribute(kAttrHTTPStatusCode, payload->status_code);
     }
 

instrumentation/otel-webserver-module/src/core/api/opentelemetry_ngx_api.cpp

@@ -119,13 +119,17 @@ OTEL_SDK_STATUS_CODE endRequest(OTEL_SDK_HANDLE_REQ req_handle_key, const char*
     if (payload != NULL) {
         for (int i = 0; i < payload->response_headers_count; i++) {
             std::string key(payload->response_headers[i].name);
-            if (responseHeadersToCapture.find(key)
-            != responseHeadersToCapture.end()) {
+            if (responseHeadersToCapture.find(key) != responseHeadersToCapture.end()) {
                 responsePayload->response_headers[key]
                     = payload->response_headers[i].value;
             }
         }
         responsePayload->status_code = payload->status_code;
+
+        for(int i=0; i < payload->otel_attributes_count; i++){
+            std::string key(payload->otel_attributes[i].name);
+            responsePayload->otel_attributes[key] = payload->otel_attributes[i].value;
+        }
     }
 
     res = wsAgent.endRequest(req_handle_key, errMsg, responsePayload.get());

instrumentation/otel-webserver-module/src/nginx/ngx_http_opentelemetry_module.c

@@ -383,10 +383,19 @@ static ngx_command_t ngx_http_opentelemetry_commands[] = {
       NGX_HTTP_LOC_CONF_OFFSET,
       offsetof(ngx_http_opentelemetry_loc_conf_t, nginxTrustIncomingSpans),
       NULL},
-
-    ngx_null_command	/* command termination */
+
+    { ngx_string("NginxAttributes"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_1MORE,
+      ngx_otel_attributes_set,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      0,
+      NULL},
+
+    /* command termination */
+    ngx_null_command
 };
 
+
 /* The module context. */
 static ngx_http_module_t ngx_http_opentelemetry_module_ctx = {
     NULL,						/* preconfiguration */
@@ -454,6 +463,7 @@ static char* ngx_http_opentelemetry_merge_loc_conf(ngx_conf_t *cf, void *parent,
     ngx_http_opentelemetry_loc_conf_t *prev = parent;
     ngx_http_opentelemetry_loc_conf_t *conf = child;
     ngx_otel_set_global_context(prev);
+    ngx_otel_set_attributes(prev, conf);
 
     ngx_conf_merge_value(conf->nginxModuleEnabled, prev->nginxModuleEnabled, 1);
     ngx_conf_merge_value(conf->nginxModuleReportAllInstrumentedModules, prev->nginxModuleReportAllInstrumentedModules, 0);
@@ -673,6 +683,34 @@ static void ngx_http_opentelemetry_exit_worker(ngx_cycle_t *cycle)
         ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "mod_opentelemetry: ngx_http_opentelemetry_exit_worker: Exiting Nginx Worker for process with PID: %s**********", worker_conf->pid);
     }
 }
+static char* ngx_otel_attributes_set(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
+    ngx_http_opentelemetry_loc_conf_t * my_conf=(ngx_http_opentelemetry_loc_conf_t *)conf;
+
+    ngx_str_t *value = cf->args->elts;
+
+    ngx_array_t *arr;
+    ngx_str_t   *elt;
+    ngx_int_t arg_count = cf->args->nelts; 
+
+    arr = ngx_array_create(cf->pool, arg_count, sizeof(ngx_str_t));
+
+    if (arr == NULL) {
+        return NGX_CONF_ERROR;
+    }
+
+    // Add elements to the array
+    for (ngx_int_t i = 1; i < arg_count; i++) {
+        elt = ngx_array_push(arr);
+        if (elt == NULL) {
+            return NGX_CONF_ERROR;
+        }
+        ngx_str_set(elt, value[i].data);
+        elt->len = ngx_strlen(value[i].data);
+    }
+    my_conf->nginxAttributes = arr;
+    return NGX_CONF_OK;
+
+}
 
 static char* ngx_otel_context_set(ngx_conf_t *cf, ngx_command_t *cmd, void *conf){
     ngx_str_t* value;
@@ -704,6 +742,20 @@ static void ngx_otel_set_global_context(ngx_http_opentelemetry_loc_conf_t * prev
     }
 }
 
+static void ngx_otel_set_attributes(ngx_http_opentelemetry_loc_conf_t * prev, ngx_http_opentelemetry_loc_conf_t * conf)
+{
+    if (conf->nginxAttributes && (conf->nginxAttributes->nelts) > 0) {
+        return;
+    }
+    if (prev->nginxAttributes && (prev->nginxAttributes->nelts) > 0) {
+        if((conf->nginxAttributes) == NULL)
+        {
+            conf->nginxAttributes = prev->nginxAttributes;
+        }
+    }
+    return;
+}
+
 /*
     Begin a new interaction
 */
@@ -737,6 +789,7 @@ static OTEL_SDK_STATUS_CODE otel_startInteraction(ngx_http_request_t* r, const c
         int ix = 0;
         res = startModuleInteraction((void*)ctx->otel_req_handle_key, module_name, "", resolveBackends, propagationHeaders, &ix);
 
+
         if (OTEL_ISSUCCESS(res))
         {
             removeUnwantedHeader(r);
@@ -892,6 +945,39 @@ static ngx_uint_t otel_getErrorCode(ngx_http_request_t* r)
     else return 0;
 }
 
+static void resolve_attributes_variables(ngx_http_request_t* r)
+{
+    ngx_http_opentelemetry_loc_conf_t *conf = ngx_http_get_module_loc_conf(r, ngx_http_opentelemetry_module);
+
+    for (ngx_uint_t j = 0; j < conf->nginxAttributes->nelts; j++) {
+
+        void *element = conf->nginxAttributes->elts + (j * conf->nginxAttributes->size);
+        ngx_str_t var_name = (((ngx_str_t *)(conf->nginxAttributes->elts))[j]);
+        ngx_uint_t           key; // The variable's hashed key.
+        ngx_http_variable_value_t  *value; // Pointer to the value object.
+
+        if(var_name.data[0] == '$'){
+            // Get the hashed key.
+            ngx_str_t new_var_name = var_name;
+            new_var_name.data++;
+            new_var_name.len--;
+            key = ngx_hash_key(new_var_name.data, new_var_name.len);
+
+            // Get the variable.
+            value = ngx_http_get_variable(r, &new_var_name, key);
+
+            if (value == NULL || value->not_found) {
+                // Variable was not found.
+            } else {
+                ngx_str_t * ngx_str = (ngx_str_t *)(element);
+                ngx_str->data = value->data;
+                ngx_str->len = ngx_strlen(value);
+                // Variable was found, `value` now contains the value.
+            }
+        }
+    }
+}
+
 static ngx_flag_t ngx_initialize_opentelemetry(ngx_http_request_t *r)
 {
     // check to see if we have already been initialized
@@ -1274,7 +1360,7 @@ static ngx_int_t ngx_http_otel_realip_handler(ngx_http_request_t *r){
 
     otel_startInteraction(r, "ngx_http_realip_module");
     ngx_int_t rvalue = h[NGX_HTTP_REALIP_MODULE_INDEX](r);
-    otel_stopInteraction(r, "ngx_http_realip_module", OTEL_SDK_NO_HANDLE);
+    otel_stopInteraction(r, "ngx_http_realip_module", OTEL_SDK_NO_HANDLE);    
 
     return rvalue;
 }
@@ -1555,6 +1641,7 @@ static void removeUnwantedHeader(ngx_http_request_t* r)
   }
 }
 
+
 static void fillRequestPayload(request_payload* req_payload, ngx_http_request_t* r){
     ngx_list_part_t  *part;
     ngx_table_elt_t  *header;
@@ -1636,6 +1723,7 @@ static void fillRequestPayload(request_payload* req_payload, ngx_http_request_t*
 
     ngx_http_opentelemetry_loc_conf_t *conf =
       ngx_http_get_module_loc_conf(r, ngx_http_opentelemetry_module);
+
     part = &r->headers_in.headers.part;
     header = (ngx_table_elt_t*)part->elts;
     nelts = part->nelts;
@@ -1669,6 +1757,7 @@ static void fillRequestPayload(request_payload* req_payload, ngx_http_request_t*
     }
     req_payload->propagation_count = propagation_headers_idx;
     req_payload->request_headers_count = request_headers_idx;
+
 }
 
 static void fillResponsePayload(response_payload* res_payload, ngx_http_request_t* r)
@@ -1689,6 +1778,31 @@ static void fillResponsePayload(response_payload* res_payload, ngx_http_request_
     res_payload->response_headers = ngx_pcalloc(r->pool, nelts * sizeof(http_headers));
     ngx_uint_t headers_count = 0;
 
+     ngx_http_opentelemetry_loc_conf_t *conf = ngx_http_get_module_loc_conf(r, ngx_http_opentelemetry_module);
+
+    res_payload->otel_attributes = ngx_pcalloc(r->pool, ((nelts+1)/3) * sizeof(http_headers));
+    int otel_attributes_idx=0;
+
+    resolve_attributes_variables(r);
+    for (ngx_uint_t j = 0, isKey = 1, isValue = 0; j < conf->nginxAttributes->nelts; j++) {
+        const char* data = (const char*)(((ngx_str_t *)(conf->nginxAttributes->elts))[j]).data;
+        if(strcmp(data, ",") == 0){
+            otel_attributes_idx++;
+            continue;
+        }
+        else if(isKey){
+            res_payload->otel_attributes[otel_attributes_idx].name = data; 
+        }
+        else{
+            res_payload->otel_attributes[otel_attributes_idx].value = data;
+        } 
+        isKey=!isKey;
+        isValue=!isValue;
+    }
+    res_payload->otel_attributes_count = otel_attributes_idx+1;
+
+
+
     for (ngx_uint_t j = 0; j < nelts; j++) {
         h = &header[j];
 

instrumentation/otel-webserver-module/src/nginx/ngx_http_opentelemetry_module.h

@@ -104,6 +104,7 @@ typedef struct {
     ngx_str_t   nginxModuleResponseHeaders;
     ngx_str_t   nginxModuleOtelExporterOtlpHeaders;
     ngx_flag_t  nginxTrustIncomingSpans;
+    ngx_array_t  *nginxAttributes;
 
 } ngx_http_opentelemetry_loc_conf_t;
 
@@ -154,7 +155,9 @@ static void otel_payload_decorator(ngx_http_request_t* r, OTEL_SDK_ENV_RECORD* p
 static ngx_flag_t otel_requestHasErrors(ngx_http_request_t* r);
 static ngx_uint_t otel_getErrorCode(ngx_http_request_t* r);
 static char* ngx_otel_context_set(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);
+static char* ngx_otel_attributes_set(ngx_conf_t* cf, ngx_command_t*, void* conf);
 static void ngx_otel_set_global_context(ngx_http_opentelemetry_loc_conf_t * prev);
+static void ngx_otel_set_attributes(ngx_http_opentelemetry_loc_conf_t * prev, ngx_http_opentelemetry_loc_conf_t * conf);
 static void removeUnwantedHeader(ngx_http_request_t* r);
 /*
     Module specific handler