Ability to specify log category levels through separate options

Closes keycloak#34957

Co-authored-by: Steve Hawkins <[email protected]>
Signed-off-by: Václav Muzikář <[email protected]>

docs/documentation/release_notes/topics/26_1_0.adoc

@@ -61,6 +61,12 @@ See the Javadoc for a detailed description.
 In this release, admin events might hold additional details about the context when the event is fired. When upgrading you should
 expect the database schema being updated to add a new column `DETAILS_JSON` to the `ADMIN_EVENT_ENTITY` table.
 
+= Individual options for category-specific log levels
+
+It is now possible to set category-specific log levels as individual `log-level-category` options.
+
+For more details, see the https://www.keycloak.org/server/logging#_configuring_levels_as_individual_options[Logging guide].
+
 = Infinispan default XML configuration location
 
 Previous releases ignored any change  to `conf/cache-ispn.xml` if the `--cache-config-file` option was not provided.

docs/guides/server/logging.adoc

@@ -69,6 +69,25 @@ This example sets the following log levels:
 * The hibernate log level in general is set to debug.
 * To keep SQL abstract syntax trees from creating verbose log output, the specific subcategory `org.hibernate.hql.internal.ast` is set to info. As a result, the SQL abstract syntax trees are omitted instead of appearing at the `debug` level.
 
+==== Configuring levels as individual options
+When configuring category-specific log levels, you can also set the log levels as individual `log-level-<category>` options instead of using the `log-level` option for that.
+This is useful when you want to set the log levels for selected categories without overwriting the previously set `log-level` option.
+
+.Example
+If you start the server as:
+
+<@kc.start parameters="--log-level=\"INFO,org.hibernate:debug\""/>
+
+you can then set an environmental variable `KC_LOG_LEVEL_ORG_KEYCLOAK=trace` to change the log level for the `org.keycloak` category.
+
+The `log-level-<category>` options take precedence over `log-level`. This allows you to override what was set in the `log-level` option.
+For instance if you set `KC_LOG_LEVEL_ORG_HIBERNATE=trace` for the CLI example above, the  `org.hibernate` category will use the `trace` level instead of `debug`.
+
+Bear in mind that when using the environmental variables, the category name must be in uppercase and the dots must be replaced with underscores.
+When using other config sources, the category name must be specified "as is", for example:
+
+<@kc.start parameters="--log-level=\"INFO,org.hibernate:debug\" --log-level-org.keycloak=trace"/>
+
 == Enabling log handlers
 To enable log handlers, enter the following command:
 

quarkus/config-api/src/main/java/org/keycloak/config/LoggingOptions.java

@@ -60,6 +60,12 @@ public String toString() {
             .description("The log level of the root category or a comma-separated list of individual categories and their levels. For the root category, you don't need to specify a category.")
             .build();
 
+    public static final Option<Level> LOG_LEVEL_CATEGORY = new OptionBuilder<>("log-level-<category>", Level.class)
+            .category(OptionCategory.LOGGING)
+            .description("The log level of a category. Takes precedence over the 'log-level' option.")
+            .caseInsensitiveExpectedValues(true)
+            .build();
+
     public enum Output {
         DEFAULT,
         JSON;

quarkus/config-api/src/main/java/org/keycloak/config/Option.java

@@ -2,9 +2,12 @@
 
 import java.util.List;
 import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 public class Option<T> {
+    public static final Pattern WILDCARD_PLACEHOLDER_PATTERN = Pattern.compile("<.+>");
 
     private final Class<T> type;
     private final String key;

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/cli/Picocli.java

@@ -47,6 +47,7 @@
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
 import org.keycloak.common.profile.ProfileException;
 import org.keycloak.config.DeprecatedMetadata;
@@ -101,16 +102,12 @@ private static class IncludeOptions {
     }
 
     private ExecutionExceptionHandler errorHandler = new ExecutionExceptionHandler();
+    private Set<PropertyMapper<?>> allowedMappers;
+    private List<String> unrecognizedArgs = new ArrayList<>();
 
     public void parseAndRun(List<String> cliArgs) {
         // perform two passes over the cli args. First without option validation to determine the current command, then with option validation enabled
-        CommandLine cmd = createCommandLine(spec -> spec
-                .addUnmatchedArgsBinding(CommandLine.Model.UnmatchedArgsBinding.forStringArrayConsumer(new ISetter() {
-                    @Override
-                    public <T> T set(T value) throws Exception {
-                        return null; // just ignore
-                    }
-                })));
+        CommandLine cmd = createCommandLine(spec -> {}).setUnmatchedArgumentsAllowed(true);
         String[] argArray = cliArgs.toArray(new String[0]);
 
         try {
@@ -157,6 +154,16 @@ private CommandLine createCommandLineForCommand(List<String> cliArgs, List<Comma
 
                 currentSpec = subCommand.getCommandSpec();
 
+                currentSpec.addUnmatchedArgsBinding(CommandLine.Model.UnmatchedArgsBinding.forStringArrayConsumer(new ISetter() {
+                    @Override
+                    public <T> T set(T value) {
+                        if (value != null) {
+                            unrecognizedArgs.addAll(Arrays.asList((String[]) value));
+                        }
+                        return null; // doesn't matter
+                    }
+                }));
+
                 addHelp(currentSpec);
             }
 
@@ -326,6 +333,17 @@ private static boolean wasBuildEverRun() {
      * @param abstractCommand
      */
     public void validateConfig(List<String> cliArgs, AbstractCommand abstractCommand) {
+        unrecognizedArgs.removeIf(arg -> {
+            if (arg.contains("=")) {
+                arg = arg.substring(0, arg.indexOf("="));
+            }
+            PropertyMapper<?> mapper = PropertyMappers.getMapper(arg);
+            return mapper != null && mapper.hasWildcard() && allowedMappers.contains(mapper);
+        });
+        if (!unrecognizedArgs.isEmpty()) {
+            throw new KcUnmatchedArgumentException(abstractCommand.getCommandLine().orElseThrow(), unrecognizedArgs);
+        }
+
         if (cliArgs.contains(OPTIMIZED_BUILD_OPTION_LONG) && !wasBuildEverRun()) {
             throw new PropertyException(Messages.optimizedUsedForFirstStartup());
         }
@@ -363,53 +381,54 @@ public void validateConfig(List<String> cliArgs, AbstractCommand abstractCommand
                 Optional.ofNullable(PropertyMappers.getRuntimeMappers().get(category)).ifPresent(mappers::addAll);
                 Optional.ofNullable(PropertyMappers.getBuildTimeMappers().get(category)).ifPresent(mappers::addAll);
                 for (PropertyMapper<?> mapper : mappers) {
-                    ConfigValue configValue = Configuration.getConfigValue(mapper.getFrom());
-                    String configValueStr = configValue.getValue();
+                    mapper.getKcConfigValues().forEach(configValue -> {
+                        String configValueStr = configValue.getValue();
 
-                    // don't consider missing or anything below standard env properties
-                    if (configValueStr == null) {
-                        if (Environment.isRuntimeMode() && mapper.isEnabled() && mapper.isRequired()) {
-                            handleRequired(missingOption, mapper);
+                        // don't consider missing or anything below standard env properties
+                        if (configValueStr == null) {
+                            if (Environment.isRuntimeMode() && mapper.isEnabled() && mapper.isRequired()) {
+                                handleRequired(missingOption, mapper);
+                            }
+                            return;
                         }
-                        continue;
-                    }
-                    if (!isUserModifiable(configValue)) {
-                        continue;
-                    }
-
-                    if (disabledMappers.contains(mapper)) {
-                        if (!PropertyMappers.isDisabledMapper(mapper.getFrom())) {
-                            continue; // we found enabled mapper with the same name
+                        if (!isUserModifiable(configValue)) {
+                            return;
                         }
 
-                        // only check build-time for a rebuild, we'll check the runtime later
-                        if (!mapper.isRunTime() || !isRebuild()) {
-                            if (PropertyMapper.isCliOption(configValue)) {
-                                throw new KcUnmatchedArgumentException(abstractCommand.getCommandLine().orElseThrow(), List.of(mapper.getCliFormat()));
-                            } else {
-                                handleDisabled(mapper.isRunTime() ? disabledRunTime : disabledBuildTime, mapper);
+                        if (disabledMappers.contains(mapper)) {
+                            if (!PropertyMappers.isDisabledMapper(mapper.getFrom())) {
+                                return; // we found enabled mapper with the same name
                             }
+
+                            // only check build-time for a rebuild, we'll check the runtime later
+                            if (!mapper.isRunTime() || !isRebuild()) {
+                                if (PropertyMapper.isCliOption(configValue)) {
+                                    throw new KcUnmatchedArgumentException(abstractCommand.getCommandLine().orElseThrow(), List.of(mapper.getCliFormat()));
+                                } else {
+                                    handleDisabled(mapper.isRunTime() ? disabledRunTime : disabledBuildTime, mapper);
+                                }
+                            }
+                            return;
                         }
-                        continue;
-                    }
 
-                    if (mapper.isBuildTime() && !options.includeBuildTime) {
-                        String currentValue = getRawPersistedProperty(mapper.getFrom()).orElse(null);
-                        if (!configValueStr.equals(currentValue)) {
-                            ignoredBuildTime.add(mapper.getFrom());
-                            continue;
+                        if (mapper.isBuildTime() && !options.includeBuildTime) {
+                            String currentValue = getRawPersistedProperty(mapper.getFrom()).orElse(null);
+                            if (!configValueStr.equals(currentValue)) {
+                                ignoredBuildTime.add(mapper.getFrom());
+                                return;
+                            }
+                        }
+                        if (mapper.isRunTime() && !options.includeRuntime) {
+                            ignoredRunTime.add(mapper.getFrom());
+                            return;
                         }
-                    }
-                    if (mapper.isRunTime() && !options.includeRuntime) {
-                        ignoredRunTime.add(mapper.getFrom());
-                        continue;
-                    }
 
-                    mapper.validate(configValue);
+                        mapper.validate(configValue);
 
-                    mapper.getDeprecatedMetadata().ifPresent(metadata -> {
-                        handleDeprecated(deprecatedInUse, mapper, configValueStr, metadata);
-                    });
+                        mapper.getDeprecatedMetadata().ifPresent(metadata -> {
+                            handleDeprecated(deprecatedInUse, mapper, configValueStr, metadata);
+                        });
+                    });;
                 }
             }
 
@@ -641,7 +660,7 @@ private static void addHelp(CommandSpec currentSpec) {
         }
     }
 
-    private static IncludeOptions getIncludeOptions(List<String> cliArgs, AbstractCommand abstractCommand, String commandName) {
+    private IncludeOptions getIncludeOptions(List<String> cliArgs, AbstractCommand abstractCommand, String commandName) {
         IncludeOptions result = new IncludeOptions();
         if (abstractCommand == null) {
             return result;
@@ -659,7 +678,7 @@ private static IncludeOptions getIncludeOptions(List<String> cliArgs, AbstractCo
         return result;
     }
 
-    private static void addCommandOptions(List<String> cliArgs, CommandLine command) {
+    private void addCommandOptions(List<String> cliArgs, CommandLine command) {
         if (command != null && command.getCommand() instanceof AbstractCommand) {
             IncludeOptions options = getIncludeOptions(cliArgs, command.getCommand(), command.getCommandName());
 
@@ -671,7 +690,7 @@ private static void addCommandOptions(List<String> cliArgs, CommandLine command)
         }
     }
 
-    private static void addOptionsToCli(CommandLine commandLine, IncludeOptions includeOptions) {
+    private void addOptionsToCli(CommandLine commandLine, IncludeOptions includeOptions) {
         final Map<OptionCategory, List<PropertyMapper<?>>> mappers = new EnumMap<>(OptionCategory.class);
 
         // Since we can't run sanitizeDisabledMappers sooner, PropertyMappers.getRuntime|BuildTimeMappers() at this point
@@ -685,6 +704,8 @@ private static void addOptionsToCli(CommandLine commandLine, IncludeOptions incl
         }
 
         addMappedOptionsToArgGroups(commandLine, mappers);
+
+        allowedMappers = mappers.values().stream().flatMap(List::stream).collect(Collectors.toUnmodifiableSet());
     }
 
     private static <T extends Map<OptionCategory, List<PropertyMapper<?>>>> void combinePropertyMappers(T origMappers, T additionalMappers) {

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/cli/command/AbstractStartCommand.java

@@ -39,8 +39,6 @@ public abstract class AbstractStartCommand extends AbstractCommand implements Ru
     @Override
     public void run() {
         doBeforeRun();
-        HttpPropertyMappers.validateConfig();
-        HostnameV2PropertyMappers.validateConfig();
         validateConfig();
 
         if (isDevProfile()) {
@@ -56,6 +54,13 @@ protected void doBeforeRun() {
 
     }
 
+    @Override
+    protected void validateConfig() {
+        super.validateConfig(); // we want to run the generic validation here first to check for unknown options
+        HttpPropertyMappers.validateConfig();
+        HostnameV2PropertyMappers.validateConfig();
+    }
+
     @Override
     public List<OptionCategory> getOptionCategories() {
         EnumSet<OptionCategory> excludedCategories = excludedCategories();

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/configuration/ConfigArgsConfigSource.java

@@ -129,6 +129,8 @@ public void accept(String key, String value) {
                 PropertyMapper<?> mapper = PropertyMappers.getMapper(key);
 
                 if (mapper != null) {
+                    mapper = mapper.forKey(key);
+
                     String to = mapper.getTo();
 
                     if (to != null) {

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/configuration/KcEnvConfigSource.java

@@ -50,6 +50,8 @@ private static Map<String, String> buildProperties() {
                 PropertyMapper<?> mapper = PropertyMappers.getMapper(key);
 
                 if (mapper != null) {
+                    mapper = mapper.forEnvKey(key);
+
                     String to = mapper.getTo();
 
                     if (to != null) {

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/configuration/PropertyMappingInterceptor.java

@@ -22,12 +22,16 @@
 
 import io.smallrye.config.Priorities;
 import jakarta.annotation.Priority;
+import org.apache.commons.collections4.IteratorUtils;
 import org.apache.commons.collections4.iterators.FilterIterator;
 import org.keycloak.quarkus.runtime.Environment;
 import org.keycloak.quarkus.runtime.configuration.mappers.PropertyMapper;
 import org.keycloak.quarkus.runtime.configuration.mappers.PropertyMappers;
+import org.keycloak.quarkus.runtime.configuration.mappers.WildcardPropertyMapper;
 
 import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
 
 import static org.keycloak.quarkus.runtime.Environment.isRebuild;
 
@@ -49,7 +53,8 @@
 @Priority(Priorities.APPLICATION - 10)
 public class PropertyMappingInterceptor implements ConfigSourceInterceptor {
 
-    private static ThreadLocal<Boolean> disable = new ThreadLocal<>();
+    private static final ThreadLocal<Boolean> disable = new ThreadLocal<>();
+    private static final ThreadLocal<Boolean> disableAdditionalNames = new ThreadLocal<>();
 
     public static void disable() {
         disable.set(true);
@@ -73,7 +78,26 @@ static boolean isRuntime(String name) {
 
     @Override
     public Iterator<String> iterateNames(ConfigSourceInterceptorContext context) {
-        return filterRuntime(context.iterateNames());
+        // We need to iterate through names to get wildcard option names.
+        // Additionally, wildcardValuesTransformer might also trigger iterateNames.
+        // Hence we need to disable this to prevent infinite recursion.
+        // But we don't want to disable the whole interceptor, as wildcardValuesTransformer
+        // might still need mappers to work.
+        List<String> mappedWildcardNames = List.of();
+        if (!Boolean.TRUE.equals(disableAdditionalNames.get())) {
+            disableAdditionalNames.set(true);
+            try {
+                mappedWildcardNames = PropertyMappers.getWildcardMappers().stream()
+                        .map(WildcardPropertyMapper::getToWithWildcards)
+                        .flatMap(Set::stream)
+                        .toList();
+            } finally {
+                disableAdditionalNames.remove();
+            }
+        }
+
+        // this could be optimized by filtering the wildcard names in the stream above
+        return filterRuntime(IteratorUtils.chainedIterator(mappedWildcardNames.iterator(), context.iterateNames()));
     }
 
     @Override