[DocTool] Allow nested config sections (#9370)

The current smallrye-config parts of the doc-tool can only handle "config sections" directly on the `@ConfigMapping` type, but adding `@ConfigItem(section = ...)` on any attribute in a nested type is not handled.

This change was primarily intended to just allow that, but turned out into a bigger refactoring and eventually a leaner code base. Generation of the smallrye docs is now a single recursive walk, which eliminates the problem of the (currenlty broken) handling of property prefixes (actually two variants: the "plain" and the "markdown" one) and section prefixes.

As a side effect, this change also fixes actually wrong property names in the reference docs. Examples:
* currently wrong: `nessie.catalog.warehouseDefaults.default-warehouse`, correct: `nessie.catalog.default-warehouse`
* currently wrong: `nessie.catalog.service.adls.transport....`, currect: `nessie.catalog.service.adls....`

catalog/files/impl/src/main/java/org/projectnessie/catalog/files/adls/AdlsOptions.java

@@ -48,11 +48,11 @@ public interface AdlsOptions {
    * Default file-system configuration, default/fallback values for all file-systems are taken from
    * this one.
    */
-  @ConfigItem(section = "default-options", firstIsSectionDoc = true)
+  @ConfigItem(section = "default-options")
   Optional<? extends AdlsFileSystemOptions> defaultOptions();
 
   /** ADLS file-system specific options, per file system name. */
-  @ConfigItem(section = "buckets", firstIsSectionDoc = true)
+  @ConfigItem(section = "buckets")
   @ConfigPropertyName("filesystem-name")
   Map<String, ? extends AdlsNamedFileSystemOptions> fileSystems();
 

catalog/files/impl/src/main/java/org/projectnessie/catalog/files/gcs/GcsOptions.java

@@ -36,15 +36,15 @@ public interface GcsOptions {
   /**
    * Default bucket configuration, default/fallback values for all buckets are taken from this one.
    */
-  @ConfigItem(section = "default-options", firstIsSectionDoc = true)
+  @ConfigItem(section = "default-options")
   Optional<? extends GcsBucketOptions> defaultOptions();
 
   /**
    * Per-bucket configurations. The effective value for a bucket is taken from the per-bucket
    * setting. If no per-bucket setting is present, uses the defaults from the top-level GCS
    * settings.
    */
-  @ConfigItem(section = "buckets", firstIsSectionDoc = true)
+  @ConfigItem(section = "buckets")
   @ConfigPropertyName("bucket-name")
   Map<String, ? extends GcsNamedBucketOptions> buckets();
 

catalog/files/impl/src/main/java/org/projectnessie/catalog/files/s3/S3Options.java

@@ -72,14 +72,14 @@ default int effectiveStsClientsCacheMaxEntries() {
   /**
    * Default bucket configuration, default/fallback values for all buckets are taken from this one.
    */
-  @ConfigItem(section = "default-options", firstIsSectionDoc = true)
+  @ConfigItem(section = "default-options")
   Optional<? extends S3BucketOptions> defaultOptions();
 
   /**
    * Per-bucket configurations. The effective value for a bucket is taken from the per-bucket
    * setting. If no per-bucket setting is present, uses the values from top-level S3 settings.
    */
-  @ConfigItem(section = "buckets", firstIsSectionDoc = true)
+  @ConfigItem(section = "buckets")
   @ConfigPropertyName("bucket-name")
   Map<String, ? extends S3NamedBucketOptions> buckets();
 

tools/doc-generator/annotations/src/main/java/org/projectnessie/nessie/docgen/annotations/ConfigDocs.java

@@ -49,11 +49,8 @@ public interface ConfigDocs {
      */
     String section() default "";
 
-    /**
-     * For smallrye-configs only, the section docs are taken from docs of the very first property
-     * and that is omitted in the properties table.
-     */
-    boolean firstIsSectionDoc() default false;
+    /** For smallrye-configs only: the section docs are taken from property type's javadoc. */
+    boolean sectionDocFromType() default false;
   }
 
   /**

tools/doc-generator/doclet/src/main/java/org/projectnessie/nessie/docgen/DocGenDoclet.java

@@ -18,21 +18,22 @@
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.nio.file.StandardOpenOption.CREATE;
 import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
+import static org.projectnessie.nessie.docgen.SmallRyeConfigs.concatWithDot;
 
+import com.sun.source.doctree.DocCommentTree;
 import java.io.BufferedWriter;
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
 import javax.lang.model.SourceVersion;
 import javax.lang.model.element.Element;
-import javax.lang.model.element.TypeElement;
 import jdk.javadoc.doclet.Doclet;
 import jdk.javadoc.doclet.DocletEnvironment;
 import jdk.javadoc.doclet.Reporter;
@@ -81,7 +82,7 @@ public boolean process(String option, List<String> arguments) {
   @Override
   public boolean run(DocletEnvironment environment) {
     PropertiesConfigs propertiesConfigs = new PropertiesConfigs(environment);
-    SmallryeConfigs smallryeConfigs = new SmallryeConfigs(environment);
+    SmallRyeConfigs smallryeConfigs = new SmallRyeConfigs(environment);
 
     for (Element includedElement : environment.getIncludedElements()) {
       try {
@@ -92,6 +93,14 @@ public boolean run(DocletEnvironment environment) {
       }
     }
 
+    propertiesConfigPages(propertiesConfigs);
+
+    smallryeConfigPages(environment, smallryeConfigs);
+
+    return true;
+  }
+
+  private void propertiesConfigPages(PropertiesConfigs propertiesConfigs) {
     for (PropertiesConfigPageGroup page : propertiesConfigs.pages()) {
       System.out.println("Generating properties config pages for " + page.name());
       for (Map.Entry<String, Iterable<PropertiesConfigItem>> e : page.sectionItems().entrySet()) {
@@ -123,92 +132,153 @@ public boolean run(DocletEnvironment environment) {
         }
       }
     }
+  }
 
-    for (SmallRyeConfigSection configSection : smallryeConfigs.buildConfigSections(environment)) {
-      Path file =
-          outputDirectory.resolve("smallrye-" + safeFileName(configSection.fileName()) + ".md");
-      try (BufferedWriter fw = Files.newBufferedWriter(file, UTF_8, CREATE, TRUNCATE_EXISTING);
-          PrintWriter writer = new PrintWriter(fw)) {
-
-        TypeElement typeElem = configSection.element();
-        if (typeElem != null) {
-          MarkdownTypeFormatter typeFormatter =
-              new MarkdownTypeFormatter(configSection.element(), configSection.typeComment());
-          writer.println(typeFormatter.description().trim());
-          writer.println();
-        }
+  private void smallryeConfigPages(DocletEnvironment environment, SmallRyeConfigs smallryeConfigs) {
+    Map<String, SmallRyeConfigSectionPage> sectionPages = new HashMap<>();
 
-        List<SmallRyeConfigPropertyInfo> properties = configSection.properties();
-        SmallRyeConfigPropertyInfo first = properties.isEmpty() ? null : properties.get(0);
-        if (first != null && first.firstIsSectionDoc()) {
-          MarkdownTypeFormatter typeFormatter =
-              new MarkdownTypeFormatter(first.propertyElement(), first.doc());
-          writer.println(typeFormatter.description().trim());
-          writer.println();
-        }
-
-        if (!properties.isEmpty()) {
-          writer.println("| Property | Default Value | Type | Description |");
-          writer.println("|----------|---------------|------|-------------|");
-          properties.forEach(
-              prop ->
-                  writeProperty(
-                      prop, writer, configSection.prefix() + '.', smallryeConfigs, environment));
-        }
-      } catch (IOException ex) {
-        throw new RuntimeException(ex);
-      }
+    for (SmallRyeConfigMappingInfo mappingInfo : smallryeConfigs.configMappingInfos()) {
+      smallryeProcessRootMappingInfo(environment, smallryeConfigs, mappingInfo, sectionPages);
     }
 
-    return true;
+    sectionPages.values().stream()
+        .filter(p -> !p.isEmpty())
+        .forEach(
+            page -> {
+              System.out.printf(
+                  "... generating smallrye config page for section %s%n", page.section);
+              Path file = outputDirectory.resolve("smallrye-" + safeFileName(page.section) + ".md");
+              try (PrintWriter pw =
+                  new PrintWriter(
+                      Files.newBufferedWriter(file, UTF_8, CREATE, TRUNCATE_EXISTING))) {
+                page.writeTo(pw);
+              } catch (IOException e) {
+                throw new RuntimeException(e);
+              }
+            });
   }
 
-  private void writeProperty(
-      SmallRyeConfigPropertyInfo prop,
-      PrintWriter writer,
+  private void smallryeProcessRootMappingInfo(
+      DocletEnvironment environment,
+      SmallRyeConfigs smallryeConfigs,
+      SmallRyeConfigMappingInfo mappingInfo,
+      Map<String, SmallRyeConfigSectionPage> sectionPages) {
+    String effectiveSection = mappingInfo.prefix();
+    String propertyNamePrefix = mappingInfo.prefix();
+    smallryeProcessMappingInfo(
+        "",
+        environment,
+        smallryeConfigs,
+        effectiveSection,
+        mappingInfo,
+        propertyNamePrefix,
+        sectionPages);
+  }
+
+  private void smallryeProcessPropertyMappingInfo(
+      String logIndent,
+      DocletEnvironment environment,
+      SmallRyeConfigs smallryeConfigs,
+      String section,
+      SmallRyeConfigMappingInfo mappingInfo,
       String propertyNamePrefix,
-      SmallryeConfigs smallryeConfigs,
-      DocletEnvironment environment) {
-    MarkdownPropertyFormatter md = new MarkdownPropertyFormatter(prop);
-    if (!md.isHidden()) {
-      Optional<Class<?>> groupType = prop.groupType();
-      String propertyName = md.propertyName();
-      String suffix = md.propertySuffix();
-      if (!suffix.isEmpty()) {
-        propertyName += ".`_`<" + suffix + ">`_`";
-      }
-      String fullName = propertyNamePrefix + propertyName;
-
-      if (!prop.firstIsSectionDoc()) {
-        writer.print("| ");
-        String fullNameCode = ('`' + fullName + '`').replaceAll("``", "");
-        writer.print(fullNameCode);
-        writer.print(" | ");
-        String dv = prop.defaultValue();
-        if (dv != null) {
-          if (dv.isEmpty()) {
-            writer.print("(empty)");
-          } else {
-            writer.print('`');
-            writer.print(dv);
-            writer.print('`');
-          }
-        }
-        writer.print(" | ");
-        writer.print(md.propertyType());
-        writer.print(" | ");
-        writer.print(md.description().replaceAll("\n", "<br>"));
-        writer.println(" |");
-      }
+      Map<String, SmallRyeConfigSectionPage> sectionPages) {
+    smallryeProcessMappingInfo(
+        logIndent + "  ",
+        environment,
+        smallryeConfigs,
+        section,
+        mappingInfo,
+        propertyNamePrefix,
+        sectionPages);
+  }
 
-      if (groupType.isPresent()) {
-        String pre = fullName + '.';
-        smallryeConfigs
-            .getConfigMappingInfo(groupType.get())
-            .properties(environment)
-            .forEach(p -> writeProperty(p, writer, pre, smallryeConfigs, environment));
-      }
+  private void smallryeProcessMappingInfo(
+      String logIndent,
+      DocletEnvironment environment,
+      SmallRyeConfigs smallryeConfigs,
+      String effectiveSection,
+      SmallRyeConfigMappingInfo mappingInfo,
+      String propertyNamePrefix,
+      Map<String, SmallRyeConfigSectionPage> sectionPages) {
+
+    // Eagerly create page, so we have the comment from the type.
+    sectionPages.computeIfAbsent(
+        effectiveSection,
+        s -> new SmallRyeConfigSectionPage(s, mappingInfo.element(), mappingInfo.typeComment()));
+
+    mappingInfo
+        .properties(environment)
+        .forEach(
+            prop ->
+                smallryeProcessProperty(
+                    logIndent,
+                    environment,
+                    smallryeConfigs,
+                    mappingInfo,
+                    effectiveSection,
+                    prop,
+                    propertyNamePrefix,
+                    sectionPages));
+  }
+
+  private void smallryeProcessProperty(
+      String logIndent,
+      DocletEnvironment environment,
+      SmallRyeConfigs smallryeConfigs,
+      SmallRyeConfigMappingInfo mappingInfo,
+      String section,
+      SmallRyeConfigPropertyInfo propertyInfo,
+      String propertyNamePrefix,
+      Map<String, SmallRyeConfigSectionPage> sectionPages) {
+
+    String effectiveSection =
+        propertyInfo.prefixOverride().map(o -> concatWithDot(section, o)).orElse(section);
+
+    MarkdownPropertyFormatter md = new MarkdownPropertyFormatter(propertyInfo);
+    if (md.isHidden()) {
+      return;
+    }
+    String fullName =
+        formatPropertyName(propertyNamePrefix, md.propertyName(), md.propertySuffix());
+
+    SmallRyeConfigSectionPage page =
+        sectionPages.computeIfAbsent(
+            effectiveSection,
+            s -> {
+              DocCommentTree doc =
+                  propertyInfo.sectionDocFromType()
+                      ? propertyInfo
+                          .groupType()
+                          .map(smallryeConfigs::getConfigMappingInfo)
+                          .map(SmallRyeConfigMappingInfo::typeComment)
+                          .orElse(null)
+                      : propertyInfo.doc();
+              return new SmallRyeConfigSectionPage(s, mappingInfo.element(), doc);
+            });
+    propertyInfo.prefixOverride().ifPresent(o -> page.incrementSectionRef());
+    if (propertyInfo.isSettableType()) {
+      page.addProperty(fullName, propertyInfo, md);
     }
+
+    propertyInfo
+        .groupType()
+        .ifPresent(
+            groupType ->
+                smallryeProcessPropertyMappingInfo(
+                    logIndent + "  ",
+                    environment,
+                    smallryeConfigs,
+                    effectiveSection,
+                    smallryeConfigs.getConfigMappingInfo(groupType),
+                    fullName,
+                    sectionPages));
+  }
+
+  private String formatPropertyName(
+      String propertyNamePrefix, String propertyName, String propertySuffix) {
+    String r = concatWithDot(propertyNamePrefix, propertyName);
+    return propertySuffix.isEmpty() ? r : concatWithDot(r, "`_`<" + propertySuffix + ">`_`");
   }
 
   private String safeFileName(String str) {

tools/doc-generator/doclet/src/main/java/org/projectnessie/nessie/docgen/MarkdownFormatter.java

@@ -59,7 +59,7 @@ public abstract class MarkdownFormatter {
   private String javadocBody;
 
   public MarkdownFormatter(Element element, DocCommentTree commentTree) {
-    this.element = requireNonNull(element);
+    this.element = element;
 
     if (commentTree != null) {
 
@@ -136,7 +136,7 @@ public String description() {
     }
     if (javadocDeprecated != null) {
       sb.append("\n\n_Deprecated_ ").append(javadocDeprecated);
-    } else {
+    } else if (element != null) {
       boolean deprecated = element.getAnnotation(Deprecated.class) != null;
       if (deprecated) {
         sb.append("\n\n_Deprecated_ ");
@@ -568,7 +568,7 @@ private void value(ValueTree valueTree, Target target) {
       // TODO resolve reference properly
       ReferenceTree reference = valueTree.getReference();
       String signature = reference.getSignature().trim();
-      if (signature.startsWith("#")) {
+      if (signature.startsWith("#") && element != null) {
         Optional<? extends Element> referenced =
             element.getEnclosingElement().getEnclosedElements().stream()
                 .filter(enc -> enc.getSimpleName().toString().equals(signature.substring(1)))
@@ -607,7 +607,7 @@ private void link(LinkTree linkTree, Target target, boolean codeValue) {
       if (codeValue) {
         target.text.append('`');
       }
-      if (signature.startsWith("#")) {
+      if (signature.startsWith("#") && element != null) {
         Optional<? extends Element> referenced =
             element.getEnclosingElement().getEnclosedElements().stream()
                 .filter(enc -> enc.getSimpleName().toString().equals(signature.substring(1)))