Log redacted header names only and allow to turn them off (Azure#38501)

* add http log options to only log allowed header keys and drop REDACTED ones, log redacted headers as a list of names, without the value by default

sdk/core/azure-core/src/main/java/com/azure/core/http/policy/HttpLogDetailLevel.java

@@ -23,6 +23,7 @@ public enum HttpLogDetailLevel {
 
     /**
      * Logs everything in BASIC, plus all the request and response headers.
+     * Values of the headers will be logged only for allowed headers. See {@link HttpLogOptions#getAllowedHeaderNames()}.
      */
     HEADERS,
 
@@ -35,6 +36,7 @@ public enum HttpLogDetailLevel {
 
     /**
      * Logs everything in HEADERS and BODY.
+     * Values of the headers will be logged only for allowed headers. See {@link HttpLogOptions#getAllowedHeaderNames()}.
      */
     BODY_AND_HEADERS;
 

sdk/core/azure-core/src/main/java/com/azure/core/http/policy/HttpLogOptions.java

@@ -23,6 +23,7 @@ public class HttpLogOptions {
     private Set<String> allowedHeaderNames;
     private Set<String> allowedQueryParamNames;
     private boolean prettyPrintBody;
+    private boolean disableRedactedHeaderLogging;
 
     private HttpRequestLogger requestLogger;
     private HttpResponseLogger responseLogger;
@@ -279,4 +280,28 @@ public HttpLogOptions setResponseLogger(HttpResponseLogger responseLogger) {
         this.responseLogger = responseLogger;
         return this;
     }
+
+    /**
+     * Sets the flag that controls if header names which value is redacted should be logged.
+     * <p>
+     * Applies only if logging request and response headers is enabled. See {@link HttpLogOptions#setLogLevel(HttpLogDetailLevel)} for details.
+     * Defaults to `false` - redacted header names are logged.
+     *
+     * @param disableRedactedHeaderLogging If true, redacted header names are not logged.
+     * Otherwise, they are logged as a comma separated list under `redactedHeaders` property.
+     * @return The updated HttpLogOptions object.
+     */
+    public HttpLogOptions disableRedactedHeaderLogging(boolean disableRedactedHeaderLogging) {
+        this.disableRedactedHeaderLogging = disableRedactedHeaderLogging;
+        return this;
+    }
+
+    /**
+     * Gets the flag that controls if header names with redacted values should be logged.
+     *
+     * @return true if header names with redacted values should be logged.
+     */
+    public boolean isRedactedHeaderLoggingDisabled() {
+        return disableRedactedHeaderLogging;
+    }
 }

sdk/core/azure-core/src/main/java/com/azure/core/http/policy/HttpLoggingPolicy.java

@@ -73,7 +73,7 @@ public class HttpLoggingPolicy implements HttpPipelinePolicy {
     private final Set<String> allowedHeaderNames;
     private final Set<String> allowedQueryParameterNames;
     private final boolean prettyPrintBody;
-
+    private final boolean disableRedactedHeaderLogging;
     private final HttpRequestLogger requestLogger;
     private final HttpResponseLogger responseLogger;
 
@@ -102,6 +102,7 @@ public HttpLoggingPolicy(HttpLogOptions httpLogOptions) {
                 .map(queryParamName -> queryParamName.toLowerCase(Locale.ROOT))
                 .collect(Collectors.toSet());
             this.prettyPrintBody = false;
+            this.disableRedactedHeaderLogging = false;
 
             this.requestLogger = new DefaultHttpRequestLogger();
             this.responseLogger = new DefaultHttpResponseLogger();
@@ -116,6 +117,7 @@ public HttpLoggingPolicy(HttpLogOptions httpLogOptions) {
                 .map(queryParamName -> queryParamName.toLowerCase(Locale.ROOT))
                 .collect(Collectors.toSet());
             this.prettyPrintBody = httpLogOptions.isPrettyPrintBody();
+            this.disableRedactedHeaderLogging = httpLogOptions.isRedactedHeaderLoggingDisabled();
 
             this.requestLogger = (httpLogOptions.getRequestLogger() == null)
                 ? new DefaultHttpRequestLogger()
@@ -237,7 +239,7 @@ private void log(LogLevel logLevel, ClientLogger logger, HttpRequestLoggingConte
             }
 
             if (httpLogDetailLevel.shouldLogHeaders() && logger.canLogAtLevel(LogLevel.INFORMATIONAL)) {
-                addHeadersToLogMessage(allowedHeaderNames, request.getHeaders(), logBuilder);
+                addHeadersToLogMessage(allowedHeaderNames, request.getHeaders(), logBuilder, disableRedactedHeaderLogging);
             }
 
             if (request.getBody() == null) {
@@ -326,7 +328,7 @@ public Mono<HttpResponse> logResponse(ClientLogger logger, HttpResponseLoggingCo
 
         private void logHeaders(ClientLogger logger, HttpResponse response, LoggingEventBuilder logBuilder) {
             if (httpLogDetailLevel.shouldLogHeaders() && logger.canLogAtLevel(LogLevel.INFORMATIONAL)) {
-                addHeadersToLogMessage(allowedHeaderNames, response.getHeaders(), logBuilder);
+                addHeadersToLogMessage(allowedHeaderNames, response.getHeaders(), logBuilder, disableRedactedHeaderLogging);
             }
         }
 
@@ -413,10 +415,25 @@ private static String getRedactedUrl(URL url, Set<String> allowedQueryParameterN
      * @param logLevel Log level the environment is configured to use.
      */
     private static void addHeadersToLogMessage(Set<String> allowedHeaderNames, HttpHeaders headers,
-        LoggingEventBuilder logBuilder) {
+        LoggingEventBuilder logBuilder, boolean disableRedactedHeaderLogging) {
+
+        final StringBuilder redactedHeaders = new StringBuilder();
+
         // The raw header map uses keys that are already lower-cased.
-        HttpHeadersAccessHelper.getRawHeaderMap(headers).forEach((key, value) -> logBuilder.addKeyValue(value.getName(),
-            allowedHeaderNames.contains(key) ? value.getValue() : REDACTED_PLACEHOLDER));
+        HttpHeadersAccessHelper.getRawHeaderMap(headers).forEach((key, value) -> {
+            if (allowedHeaderNames.contains(key)) {
+                logBuilder.addKeyValue(value.getName(), value.getValue());
+            } else if (!disableRedactedHeaderLogging) {
+                if (redactedHeaders.length() > 0) {
+                    redactedHeaders.append(',');
+                }
+                redactedHeaders.append(value.getName());
+            }
+        });
+
+        if (redactedHeaders.length() > 0) {
+            logBuilder.addKeyValue("redactedHeaders", redactedHeaders.toString());
+        }
     }
 
     /*