Self-service errors: generate groups documentation [KVL-1186] (#11526)

* Add groups information to JSON

CHANGELOG_BEGIN
CHANGELOG_END

* Code cleanup

* Fix docs generation

* Fix KVErrors group explanation string

* Render error group explanations in generated docs

* Fix ErrorCodeDocumentationGeneratorSpec

docs/sphinx_ext/self_service_error_codes_extension.py

@@ -10,18 +10,21 @@
 import json
 
 error_codes_data = {}
+group_data = {}
 
 CONFIG_OPT = 'error_codes_json_export'
 
 def load_data(app, config):
     global error_codes_data
+    global group_data
 
     if CONFIG_OPT in config:
         file_name = config[CONFIG_OPT]
         try:
             with open(file_name) as f:
                 tmp = json.load(f)
                 error_codes_data = {error["code"]: error for error in tmp["errorCodes"]}
+                group_data = {group["className"]: group for group in tmp["groups"]}
         except EnvironmentError:
             print(f"Failed to open file: '{file_name}'")
             raise
@@ -77,8 +80,8 @@ def item_to_node(item: Dict[str, Any]) -> nodes.definition_list_item:
                 bold_text="Explanation: ",
                 non_bold_text=item['explanation'])
         definition_node += build_indented_bold_and_non_bold_node(
-            bold_text="Category: ",
-            non_bold_text=item['category'])
+             bold_text="Category: ",
+             non_bold_text=item['category'])
         if item["conveyance"]:
             definition_node += build_indented_bold_and_non_bold_node(
                 bold_text="Conveyance: ",
@@ -98,6 +101,9 @@ def item_to_node(item: Dict[str, Any]) -> nodes.definition_list_item:
 
         return node
 
+    def group_explanation_to_node(text: str) -> nodes.definition_list_item:
+        return text_node(nodes.paragraph, text)
+
     # A node of this tree is a dict that can contain
     #   1. further nodes and/or
     #   2. 'leaves' in the form of a list of error (code) data
@@ -107,8 +113,9 @@ def build_hierarchical_tree_of_error_data(data) -> defaultdict:
         root = defaultdict(create_node)
         for error_data in data:
             current = root
-            for group in error_data["hierarchicalGrouping"]:
-                current = current[group]
+            for grouping in error_data['hierarchicalGrouping']:
+                current = current[grouping['docName']]
+                current['explanation'] = group_data[grouping['className']]['explanation']
             if 'error-codes' in current:
                 current['error-codes'].append(error_data)
             else:
@@ -118,14 +125,16 @@ def build_hierarchical_tree_of_error_data(data) -> defaultdict:
     # DFS to traverse the error code data tree from `build_hierarchical_tree_of_error_data`
     # While traversing the tree, the presentation of the error codes on the documentation is built
     def dfs(tree, node, prefix: str) -> None:
+        if 'explanation' in tree and tree['explanation']:
+            node += group_explanation_to_node(tree['explanation'])
         if 'error-codes' in tree:
             dlist = nodes.definition_list()
             for code in tree['error-codes']:
                 dlist += item_to_node(item=code)
             node += dlist
         i = 1
         for subtopic, subtree in tree.items():
-            if subtopic == 'error-codes':
+            if subtopic in ['error-codes', 'explanation']:
                 continue
             subprefix = f"{prefix}{i}."
             i += 1

ledger/error/src/main/scala/com/daml/error/ErrorClass.scala

@@ -3,14 +3,24 @@
 
 package com.daml.error
 
+/** A grouping of errors.
+  *
+  * @param docName The name that will appear in the generated documentation for the grouping.
+  * @param group   If the grouping is defined by an [[ErrorGroup]], the associated instance.
+  */
+case class Grouping(
+    docName: String,
+    group: Option[ErrorGroup],
+)
+
 /** The classes [[ErrorClass]] and [[ErrorGroup]] are used to hierarchically structure error codes (their
   * hierarchical structure affects how they are displayed on the website)
   */
-case class ErrorClass(docNames: List[String]) {
-  def extend(docName: String = ""): ErrorClass = {
-    ErrorClass(docNames :+ docName)
-  }
+case class ErrorClass(groupings: List[Grouping]) {
+  def extend(grouping: Grouping): ErrorClass =
+    ErrorClass(groupings :+ grouping)
 }
+
 object ErrorClass {
   def root(): ErrorClass = ErrorClass(Nil)
 }

ledger/error/src/main/scala/com/daml/error/ErrorGroup.scala

@@ -4,7 +4,7 @@
 package com.daml.error
 
 abstract class ErrorGroup()(implicit parent: ErrorClass) {
-  private val fullClassName: String = getClass.getName
+  val fullClassName: String = getClass.getName
   // Hit https://github.com/scala/bug/issues/5425?orig=1 here: we cannot use .getSimpleName in deeply nested objects
   // TODO error codes: Switch to using .getSimpleName when switching to JDK 9+
   implicit val errorClass: ErrorClass = resolveErrorClass()
@@ -21,6 +21,6 @@ abstract class ErrorGroup()(implicit parent: ErrorClass) {
           s"Could not parse full class name: '${fullClassName}' for the error class name"
         )
       )
-    parent.extend(name)
+    parent.extend(Grouping(name, Some(this)))
   }
 }

ledger/error/src/main/scala/com/daml/error/generator/ErrorCodeDocumentationGenerator.scala

@@ -4,30 +4,28 @@
 package com.daml.error.generator
 
 import java.lang.reflect.Modifier
-
-import com.daml.error.ErrorCode
+import com.daml.error.{ErrorCode, ErrorGroup, Explanation, Resolution}
 import com.daml.error.generator.ErrorCodeDocumentationGenerator.{
   acceptedTypeNames,
   deprecatedTypeName,
   explanationTypeName,
   resolutionTypeName,
+  runtimeMirror,
 }
-import com.daml.error.{Explanation, Resolution}
-
 import org.reflections.Reflections
 
 import scala.reflect.runtime.{universe => ru}
 import scala.jdk.CollectionConverters._
+import scala.reflect.runtime.universe.typeOf
 
 /** Utility that indexes all error code implementations.
   *
-  * @param prefix The classpath prefix that should be scanned for finding subtypes of [[ErrorCode]].
+  * @param prefixes The classpath prefixes that should be scanned for finding subtypes of [[ErrorCode]].
   */
 class ErrorCodeDocumentationGenerator(prefixes: Array[String] = Array("com.daml")) {
 
-  def getDocItems: Seq[DocItem] = {
-    val errorCodes = getErrorCodeInstances
-
+  def getDocItems: (Seq[ErrorDocItem], Seq[GroupDocItem]) = {
+    val errorCodes = getInstances[ErrorCode]
     errorCodes.view.map(_.id).groupBy(identity).collect {
       case (code, occurrences) if occurrences.size > 1 =>
         sys.error(
@@ -36,53 +34,91 @@ class ErrorCodeDocumentationGenerator(prefixes: Array[String] = Array("com.daml"
         )
     }
 
-    errorCodes.map(convertToDocItem).sortBy(_.code)
+    val groups = getInstances[ErrorGroup]
+    groups.view.map(_.errorClass).groupBy(identity).collect {
+      case (group, occurrences) if occurrences.size > 1 =>
+        sys.error(
+          s"There are ${occurrences.size} groups named $group but we require each group class name to be unique! " +
+            s"Make these group class namers unique to make this assertion run through"
+        )
+    }
+
+    (errorCodes.map(convertToDocItem).sortBy(_.code), groups.map(convertToDocItem))
   }
 
-  private def getErrorCodeInstances: Seq[ErrorCode] =
+  private def getInstances[T: ru.TypeTag]: Seq[T] =
     new Reflections(prefixes)
-      .getSubTypesOf(classOf[ErrorCode])
+      .getSubTypesOf(runtimeMirror.runtimeClass(typeOf[T]))
       .asScala
       .view
       .collect {
         case clazz if !Modifier.isAbstract(clazz.getModifiers) =>
-          clazz.getDeclaredField("MODULE$").get(clazz).asInstanceOf[ErrorCode]
+          clazz.getDeclaredField("MODULE$").get(clazz).asInstanceOf[T]
       }
       .toSeq
 
-  private def convertToDocItem(error: ErrorCode): DocItem = {
+  private def convertToDocItem(error: ErrorCode): ErrorDocItem = {
     val ErrorDocumentationAnnotations(explanation, resolution) =
       getErrorDocumentationAnnotations(error)
 
-    DocItem(
+    ErrorDocItem(
       className = error.getClass.getName,
-      category = error.category.getClass.getSimpleName.replace("$", ""),
-      hierarchicalGrouping = error.parent.docNames.filter(_ != ""),
+      category = simpleClassName(error.category),
+      hierarchicalGrouping = error.parent.groupings.filter(_.docName.nonEmpty),
       conveyance = error.errorConveyanceDocString.getOrElse(""),
       code = error.id,
       explanation = explanation.getOrElse(Explanation("")),
       resolution = resolution.getOrElse(Resolution("")),
     )
   }
 
+  private def convertToDocItem(group: ErrorGroup): GroupDocItem = {
+    val GroupDocumentationAnnotations(explanation) =
+      getGroupDocumentationAnnotations(group)
+
+    GroupDocItem(
+      className = group.getClass.getName,
+      explanation = explanation.getOrElse(Explanation("")),
+    )
+  }
+
+  private def simpleClassName(any: Any): String =
+    any.getClass.getSimpleName.replace("$", "")
+
   private case class ErrorDocumentationAnnotations(
       explanation: Option[Explanation],
       resolution: Option[Resolution],
   )
 
   private def getErrorDocumentationAnnotations(error: ErrorCode): ErrorDocumentationAnnotations = {
-    val mirror = ru.runtimeMirror(getClass.getClassLoader)
-    val mirroredType = mirror.reflect(error)
+    val mirroredType = runtimeMirror.reflect(error)
     val annotations: Seq[ru.Annotation] = mirroredType.symbol.annotations
-    getAnnotations(annotations)
+    getErrorAnnotations(annotations)
+  }
+
+  private case class GroupDocumentationAnnotations(
+      explanation: Option[Explanation]
+  )
+
+  private def getGroupDocumentationAnnotations(group: ErrorGroup): GroupDocumentationAnnotations = {
+    val mirroredType = runtimeMirror.reflect(group)
+    GroupDocumentationAnnotations(
+      mirroredType.symbol.annotations
+        .find { annotation =>
+          isAnnotation(annotation, explanationTypeName)
+        }
+        .map { annotation =>
+          Explanation(parseAnnotationValue(annotation.tree))
+        }
+    )
   }
 
   private case class GetAnnotationsState(
       explanation: Option[Explanation],
       resolution: Option[Resolution],
   )
 
-  private def getAnnotations(
+  private def getErrorAnnotations(
       annotations: Seq[ru.Annotation]
   ): ErrorDocumentationAnnotations = {
 
@@ -158,6 +194,8 @@ class ErrorCodeDocumentationGenerator(prefixes: Array[String] = Array("com.daml"
 
 private object ErrorCodeDocumentationGenerator {
 
+  private val runtimeMirror: ru.Mirror = ru.runtimeMirror(getClass.getClassLoader)
+
   private val deprecatedTypeName = classOf[deprecated].getTypeName.replace("scala.", "")
   private val explanationTypeName = classOf[Explanation].getTypeName.replace("$", ".")
   private val resolutionTypeName = classOf[Resolution].getTypeName.replace("$", ".")

ledger/error/src/main/scala/com/daml/error/generator/DocItem.scala → ledger/error/src/main/scala/com/daml/error/generator/ErrorDocItem.scala

@@ -3,7 +3,7 @@
 
 package com.daml.error.generator
 
-import com.daml.error.{Explanation, Resolution}
+import com.daml.error.{Explanation, Grouping, Resolution}
 
 /** Contains error presentation data to be used for documentation rendering on the website.
   *
@@ -16,10 +16,10 @@ import com.daml.error.{Explanation, Resolution}
   * @param explanation The detailed error explanation.
   * @param resolution The suggested error resolution.
   */
-case class DocItem(
+case class ErrorDocItem(
     className: String, // TODO error codes: Rename to `errorCodeName` or `errorCodeClassName` to prevent confusion
     category: String,
-    hierarchicalGrouping: List[String],
+    hierarchicalGrouping: List[Grouping],
     conveyance: String,
     code: String,
     explanation: Explanation,

ledger/error/src/main/scala/com/daml/error/generator/GroupDocItem.scala

@@ -0,0 +1,16 @@
+// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package com.daml.error.generator
+
+import com.daml.error.Explanation
+
+/** Contains error presentation data to be used for documentation rendering on the website.
+  *
+  * @param className The group class name (see [[com.daml.error.ErrorGroup]]).
+  * @param explanation The detailed error explanation.
+  */
+case class GroupDocItem(
+    className: String,
+    explanation: Explanation,
+)

ledger/error/src/main/scala/com/daml/error/generator/Main.scala

@@ -3,6 +3,7 @@
 
 package com.daml.error.generator
 
+import com.daml.error.Grouping
 import io.circe.Encoder
 import io.circe.syntax._
 
@@ -12,9 +13,20 @@ import java.nio.file.{Files, Paths, StandardOpenOption}
   */
 object Main {
 
-  case class Output(errorCodes: Seq[DocItem])
+  case class Output(errorCodes: Seq[ErrorDocItem], groups: Seq[GroupDocItem])
 
-  implicit val errorCodeEncode: Encoder[DocItem] =
+  implicit val groupingEncode: Encoder[Grouping] =
+    Encoder.forProduct2(
+      "docName",
+      "className",
+    )(i =>
+      (
+        i.docName,
+        i.group.map(_.fullClassName),
+      )
+    )
+
+  implicit val errorCodeEncode: Encoder[ErrorDocItem] =
     Encoder.forProduct7(
       "className",
       "category",
@@ -35,13 +47,24 @@ object Main {
       )
     )
 
+  implicit val groupEncode: Encoder[GroupDocItem] =
+    Encoder.forProduct2(
+      "className",
+      "explanation",
+    )(i =>
+      (
+        i.className,
+        i.explanation.explanation,
+      )
+    )
+
   implicit val outputEncode: Encoder[Output] =
-    Encoder.forProduct1("errorCodes")(i => (i.errorCodes))
+    Encoder.forProduct2("errorCodes", "groups")(i => (i.errorCodes, i.groups))
 
   def main(args: Array[String]): Unit = {
-    val errorCodes = new ErrorCodeDocumentationGenerator().getDocItems
+    val (errorCodes, groups) = new ErrorCodeDocumentationGenerator().getDocItems
     val outputFile = Paths.get(args(0))
-    val output = Output(errorCodes)
+    val output = Output(errorCodes, groups)
     val outputText: String = output.asJson.spaces2
     val outputBytes = outputText.getBytes
     val _ = Files.write(outputFile, outputBytes, StandardOpenOption.CREATE_NEW)

ledger/error/src/test/suite/scala/com/daml/error/ErrorCodeSpec.scala

@@ -7,7 +7,7 @@ import ch.qos.logback.classic.Level
 import com.daml.error.ErrorCategory.TransientServerFailure
 import com.daml.error.utils.ErrorDetails
 import com.daml.error.utils.testpackage.SeriousError
-import com.daml.error.utils.testpackage.subpackage.NotSoSeriousError
+import com.daml.error.utils.testpackage.subpackage.MildErrors.NotSoSeriousError
 import com.daml.logging.{ContextualizedLogger, LoggingContext}
 import com.daml.platform.testing.LogCollector
 import io.grpc.protobuf.StatusProto

ledger/error/src/test/suite/scala/com/daml/error/ErrorGroupSpec.scala

@@ -23,9 +23,18 @@ class ErrorGroupSpec extends AnyFlatSpec with Matchers with BeforeAndAfter {
 
   it should "resolve correct error group names" in {
     ErrorGroupsFoo.ErrorGroupFoo1.ErrorGroupFoo2.ErrorGroupFoo3.errorClass shouldBe ErrorClass(
-      List("ErrorGroupFoo1", "ErrorGroupFoo2", "ErrorGroupFoo3")
+      List(
+        Grouping("ErrorGroupFoo1", Some(ErrorGroupsFoo.ErrorGroupFoo1)),
+        Grouping("ErrorGroupFoo2", Some(ErrorGroupsFoo.ErrorGroupFoo1.ErrorGroupFoo2)),
+        Grouping(
+          "ErrorGroupFoo3",
+          Some(ErrorGroupsFoo.ErrorGroupFoo1.ErrorGroupFoo2.ErrorGroupFoo3),
+        ),
+      )
+    )
+    ErrorGroupBar.errorClass shouldBe ErrorClass(
+      List(Grouping("ErrorGroupBar", Some(ErrorGroupBar)))
     )
-    ErrorGroupBar.errorClass shouldBe ErrorClass(List("ErrorGroupBar"))
   }
 
 }