From 87f166a15555d71ea5b261881282ae128ba236af Mon Sep 17 00:00:00 2001 From: Peter Findeisen Date: Mon, 7 Nov 2022 14:09:43 -0800 Subject: [PATCH] Post-review code changes, renaming, more comments. --- .../jmx/JmxMetricInsightInstaller.java | 4 +- .../jmx/engine/AttributeInfo.java | 7 +- .../jmx/engine/BeanAttributeExtractor.java | 7 ++ .../instrumentation/jmx/engine/BeanPack.java | 6 +- .../jmx/engine/JmxMetricInsight.java | 4 +- .../jmx/engine/MetricAttribute.java | 19 ------ .../jmx/engine/MetricAttributeExtractor.java | 23 ++++++- .../jmx/engine/MetricBanner.java | 11 ++- .../instrumentation/jmx/engine/MetricDef.java | 68 ++++++++++++++++--- .../jmx/engine/MetricExtractor.java | 5 +- .../jmx/engine/MetricRegistrar.java | 8 +-- .../instrumentation/jmx/yaml/JmxConfig.java | 2 +- .../jmx/yaml/MetricStructure.java | 7 +- .../instrumentation/jmx/yaml/RuleParser.java | 8 +-- 14 files changed, 125 insertions(+), 54 deletions(-) diff --git a/instrumentation/jmx-metrics/javaagent/src/main/java/io/opentelemetry/instrumentation/javaagent/jmx/JmxMetricInsightInstaller.java b/instrumentation/jmx-metrics/javaagent/src/main/java/io/opentelemetry/instrumentation/javaagent/jmx/JmxMetricInsightInstaller.java index 5afb57697e81..89cde5ae5db9 100644 --- a/instrumentation/jmx-metrics/javaagent/src/main/java/io/opentelemetry/instrumentation/javaagent/jmx/JmxMetricInsightInstaller.java +++ b/instrumentation/jmx-metrics/javaagent/src/main/java/io/opentelemetry/instrumentation/javaagent/jmx/JmxMetricInsightInstaller.java @@ -59,7 +59,7 @@ private static void addRulesForPlatform(String platform, MetricConfiguration con if (inputStream != null) { JmxMetricInsight.getLogger().log(FINE, "Opened input stream {0}", yamlResource); RuleParser parserInstance = RuleParser.get(); - parserInstance.addMetricDefs(conf, inputStream); + parserInstance.addMetricDefsTo(conf, inputStream); } else { JmxMetricInsight.getLogger().log(INFO, "No support found for {0}", platform); } @@ -85,7 +85,7 @@ private static void buildFromUserRules( JmxMetricInsight.getLogger().log(FINE, "JMX config file name: {0}", jmxDir); RuleParser parserInstance = RuleParser.get(); try (InputStream inputStream = Files.newInputStream(new File(jmxDir.trim()).toPath())) { - parserInstance.addMetricDefs(conf, inputStream); + parserInstance.addMetricDefsTo(conf, inputStream); } catch (Exception e) { JmxMetricInsight.getLogger().warning(e.getMessage()); } diff --git a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/AttributeInfo.java b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/AttributeInfo.java index 01b68bb13869..ac1f8b6d0c4c 100644 --- a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/AttributeInfo.java +++ b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/AttributeInfo.java @@ -5,6 +5,8 @@ package io.opentelemetry.instrumentation.jmx.engine; +import javax.annotation.Nullable; + /** * A class holding relevant information about an MBean attribute which will be used for collecting * metric values. The info comes directly from the relevant MBeans. @@ -12,9 +14,9 @@ class AttributeInfo { private boolean usesDoubles; - private String description; + @Nullable private String description; - AttributeInfo(Number sampleValue, String description) { + AttributeInfo(Number sampleValue, @Nullable String description) { if (sampleValue instanceof Byte || sampleValue instanceof Short || sampleValue instanceof Integer @@ -31,6 +33,7 @@ boolean usesDoubleValues() { return usesDoubles; } + @Nullable String getDescription() { return description; } diff --git a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/BeanAttributeExtractor.java b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/BeanAttributeExtractor.java index bfa9e3e01c91..d68f730de28e 100644 --- a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/BeanAttributeExtractor.java +++ b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/BeanAttributeExtractor.java @@ -10,6 +10,7 @@ import java.util.logging.Level; import java.util.logging.Logger; +import javax.annotation.Nullable; import javax.management.InstanceNotFoundException; import javax.management.MBeanAttributeInfo; import javax.management.MBeanInfo; @@ -98,6 +99,7 @@ String getAttributeName() { * @param objectName the ObjectName identifying the MBean * @return AttributeInfo if the attribute is properly recognized, or null */ + @Nullable AttributeInfo getAttributeInfo(MBeanServer server, ObjectName objectName) { if (logger.isLoggable(FINE)) { logger.log(FINE, "Resolving {0} for {1}", new Object[] {getAttributeName(), objectName}); @@ -168,6 +170,7 @@ AttributeInfo getAttributeInfo(MBeanServer server, ObjectName objectName) { * malfunctions will be silent to avoid flooding the log. * @return the attribute value, if found, or null if an error occurred */ + @Nullable private Object extractAttributeValue(MBeanServer server, ObjectName objectName, Logger logger) { try { Object value = server.getAttribute(objectName, baseName); @@ -206,10 +209,12 @@ private Object extractAttributeValue(MBeanServer server, ObjectName objectName, return null; } + @Nullable private Object extractAttributeValue(MBeanServer server, ObjectName objectName) { return extractAttributeValue(server, objectName, null); } + @Nullable Number extractNumericalAttribute(MBeanServer server, ObjectName objectName) { Object value = extractAttributeValue(server, objectName); if (value instanceof Number) { @@ -219,10 +224,12 @@ Number extractNumericalAttribute(MBeanServer server, ObjectName objectName) { } @Override + @Nullable public String extractValue(MBeanServer server, ObjectName objectName) { return extractStringAttribute(server, objectName); } + @Nullable private String extractStringAttribute(MBeanServer server, ObjectName objectName) { Object value = extractAttributeValue(server, objectName); if (value instanceof String) { diff --git a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/BeanPack.java b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/BeanPack.java index d0b2143712c8..8bef01c07e57 100644 --- a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/BeanPack.java +++ b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/BeanPack.java @@ -5,6 +5,7 @@ package io.opentelemetry.instrumentation.jmx.engine; +import javax.annotation.Nullable; import javax.management.ObjectName; import javax.management.QueryExp; @@ -14,7 +15,7 @@ */ public class BeanPack { // How to specify the MBean(s) - private final QueryExp queryExp; + @Nullable private final QueryExp queryExp; private final ObjectName[] namePatterns; /** @@ -24,11 +25,12 @@ public class BeanPack { * @param namePatterns an array of ObjectNames used to look for MBeans; usually they will be * patterns. If multiple patterns are provided, they work as logical OR. */ - public BeanPack(QueryExp queryExp, ObjectName... namePatterns) { + public BeanPack(@Nullable QueryExp queryExp, ObjectName... namePatterns) { this.queryExp = queryExp; this.namePatterns = namePatterns; } + @Nullable QueryExp getQueryExp() { return queryExp; } diff --git a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/JmxMetricInsight.java b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/JmxMetricInsight.java index 3a6b82f484de..44fe4e7269e0 100644 --- a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/JmxMetricInsight.java +++ b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/JmxMetricInsight.java @@ -5,7 +5,7 @@ package io.opentelemetry.instrumentation.jmx.engine; -import static java.util.logging.Level.CONFIG; +import static java.util.logging.Level.INFO; import io.opentelemetry.api.OpenTelemetry; import java.util.logging.Logger; @@ -36,7 +36,7 @@ private JmxMetricInsight(OpenTelemetry openTelemetry, long discoveryDelay) { public void start(MetricConfiguration conf) { if (conf.isEmpty()) { logger.log( - CONFIG, + INFO, "Empty JMX configuration, no metrics will be collected for InstrumentationScope " + INSTRUMENTATION_SCOPE); } else { diff --git a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricAttribute.java b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricAttribute.java index bdef32f65174..2dfff5509752 100644 --- a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricAttribute.java +++ b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricAttribute.java @@ -29,23 +29,4 @@ public String getAttributeName() { String acquireAttributeValue(MBeanServer server, ObjectName objectName) { return extractor.extractValue(server, objectName); } - - public static MetricAttributeExtractor fromConstant(String constantValue) { - return (a, b) -> { - return constantValue; - }; - } - - public static MetricAttributeExtractor fromObjectNameParameter(String parameterKey) { - if (parameterKey.isEmpty()) { - throw new IllegalArgumentException("Empty parameter name"); - } - return (dummy, objectName) -> { - return objectName.getKeyProperty(parameterKey); - }; - } - - public static MetricAttributeExtractor fromBeanAttribute(String attributeName) { - return BeanAttributeExtractor.fromName(attributeName); - } } diff --git a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricAttributeExtractor.java b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricAttributeExtractor.java index 851e0f52a0a3..61c9bc03c283 100644 --- a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricAttributeExtractor.java +++ b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricAttributeExtractor.java @@ -5,6 +5,7 @@ package io.opentelemetry.instrumentation.jmx.engine; +import javax.annotation.Nullable; import javax.management.MBeanServer; import javax.management.ObjectName; @@ -23,5 +24,25 @@ public interface MetricAttributeExtractor { * from an MBean attribute, or from the ObjectName parameter * @return the value of the attribute, can be null if extraction failed */ - String extractValue(MBeanServer server, ObjectName objectName); + @Nullable + String extractValue(@Nullable MBeanServer server, @Nullable ObjectName objectName); + + static MetricAttributeExtractor fromConstant(String constantValue) { + return (a, b) -> { + return constantValue; + }; + } + + static MetricAttributeExtractor fromObjectNameParameter(String parameterKey) { + if (parameterKey.isEmpty()) { + throw new IllegalArgumentException("Empty parameter name"); + } + return (dummy, objectName) -> { + return objectName.getKeyProperty(parameterKey); + }; + } + + static MetricAttributeExtractor fromBeanAttribute(String attributeName) { + return BeanAttributeExtractor.fromName(attributeName); + } } diff --git a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricBanner.java b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricBanner.java index 57b5c41c421d..19480843f6b6 100644 --- a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricBanner.java +++ b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricBanner.java @@ -5,6 +5,8 @@ package io.opentelemetry.instrumentation.jmx.engine; +import javax.annotation.Nullable; + /** * A class providing the user visible characteristics (name, type, description and units) of a * metric to be reported with OpenTelemetry. @@ -22,8 +24,8 @@ public enum Type { // How to report the metric using OpenTelemetry API private final String metricName; // used as Instrument name - private final String description; - private final String unit; + @Nullable private final String description; + @Nullable private final String unit; private final Type type; /** @@ -34,7 +36,8 @@ public enum Type { * @param unit a human readable unit of measurement * @param type the instrument typ to be used for the metric */ - public MetricBanner(String metricName, String description, String unit, Type type) { + public MetricBanner( + String metricName, @Nullable String description, String unit, @Nullable Type type) { this.metricName = metricName; this.description = description; this.unit = unit; @@ -45,10 +48,12 @@ String getMetricName() { return metricName; } + @Nullable String getDescription() { return description; } + @Nullable String getUnit() { return unit; } diff --git a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricDef.java b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricDef.java index 8f40e9bb3e51..f08e5a74bd50 100644 --- a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricDef.java +++ b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricDef.java @@ -9,17 +9,65 @@ * A class providing a complete definition on how to create an Open Telemetry metric out of the JMX * system: how to extract values from MBeans and how to model, name and decorate them with * attributes using OpenTelemetry Metric API. Objects of this class are immutable. - * - *

Example: The JVM provides an MBean with ObjectName "java.lang:type=Threading", and one of the - * MBean attributes is "ThreadCount". This MBean can be used to provide a metric showing the current - * number of threads run by the JVM as follows: - * - *

new MetricDef(new BeanPack(null, new ObjectName("java.lang:type=Threading")), - * - *

new MetricExtractor(new BeanAttributeExtractor("ThreadCount"), - * - *

new MetricBanner("process.runtime.jvm.threads", "Current number of threads", "1" ))); */ + +// Example: The rule described by the following YAML definition +// +// - bean: java.lang:name=*,type=MemoryPool +// metricAttribute: +// pool: param(name) +// type: beanattr(Type) +// mapping: +// Usage.used: +// metric: my.own.jvm.memory.pool.used +// type: updowncounter +// desc: Pool memory currently used +// unit: By +// Usage.max: +// metric: my.own.jvm.memory.pool.max +// type: updowncounter +// desc: Maximum obtainable memory pool size +// unit: By +// +// can be created using the following snippet: +// +// MetricAttribute poolAttribute = +// new MetricAttribute("pool", MetricAttributeExtractor.fromObjectNameParameter("name")); +// MetricAttribute typeAttribute = +// new MetricAttribute("type", MetricAttributeExtractor.fromBeanAttribute("Type")); +// +// MetricBanner poolUsedBanner = +// new MetricBanner( +// "my.own.jvm.memory.pool.used", +// "Pool memory currently used", +// "By", +// MetricBanner.Type.UPDOWNCOUNTER); +// MetricBanner poolLimitBanner = +// new MetricBanner( +// "my.own.jvm.memory.pool.limit", +// "Maximum obtainable memory pool size", +// "By", +// MetricBanner.Type.UPDOWNCOUNTER); +// +// MetricExtractor usageUsedExtractor = +// new MetricExtractor( +// new BeanAttributeExtractor("Usage", "used"), +// poolUsedBanner, +// poolAttribute, +// typeAttribute); +// MetricExtractor usageMaxExtractor = +// new MetricExtractor( +// new BeanAttributeExtractor("Usage", "max"), +// poolLimitBanner, +// poolAttribute, +// typeAttribute); +// +// MetricDef def = +// new MetricDef( +// new BeanPack(null, new ObjectName("java.lang:name=*,type=MemoryPool")), +// usageUsedExtractor, +// usageMaxExtractor); + public class MetricDef { // Describes the MBeans to use diff --git a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricExtractor.java b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricExtractor.java index 6d49081de9dd..6539c6076e3c 100644 --- a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricExtractor.java +++ b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricExtractor.java @@ -5,6 +5,8 @@ package io.opentelemetry.instrumentation.jmx.engine; +import javax.annotation.Nullable; + /** * A class holding the info needed to support a single metric: how to define it in OpenTelemetry and * how to provide the metric values. @@ -22,7 +24,7 @@ public class MetricExtractor { // Defines the Measurement attributes to be used when reporting the metric value. private final MetricAttribute[] attributes; - private volatile DetectionStatus status; + @Nullable private volatile DetectionStatus status; public MetricExtractor( BeanAttributeExtractor attributeExtractor, @@ -49,6 +51,7 @@ void setStatus(DetectionStatus status) { this.status = status; } + @Nullable DetectionStatus getStatus() { return status; } diff --git a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricRegistrar.java b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricRegistrar.java index 49ac9f2676ed..d207e317b21b 100644 --- a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricRegistrar.java +++ b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/engine/MetricRegistrar.java @@ -5,7 +5,7 @@ package io.opentelemetry.instrumentation.jmx.engine; -import static java.util.logging.Level.CONFIG; +import static java.util.logging.Level.INFO; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.common.Attributes; @@ -83,7 +83,7 @@ void enrollExtractor( } else { builder.buildWithCallback(longTypeCallback(extractor)); } - logger.log(CONFIG, "Created Counter for {0}", metricName); + logger.log(INFO, "Created Counter for {0}", metricName); } break; @@ -104,7 +104,7 @@ void enrollExtractor( } else { builder.buildWithCallback(longTypeCallback(extractor)); } - logger.log(CONFIG, "Created UpDownCounter for {0}", metricName); + logger.log(INFO, "Created UpDownCounter for {0}", metricName); } break; @@ -125,7 +125,7 @@ void enrollExtractor( } else { builder.ofLongs().buildWithCallback(longTypeCallback(extractor)); } - logger.log(CONFIG, "Created Gauge for {0}", metricName); + logger.log(INFO, "Created Gauge for {0}", metricName); } } } diff --git a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/yaml/JmxConfig.java b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/yaml/JmxConfig.java index 1749a098310c..d979dfb75752 100644 --- a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/yaml/JmxConfig.java +++ b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/yaml/JmxConfig.java @@ -37,7 +37,7 @@ public void setRules(List rules) { * @param configuration MetricConfiguration to add MetricDefs to * @throws an exception if the rule conversion cannot be performed */ - public void addMetricDefs(MetricConfiguration configuration) throws Exception { + void addMetricDefsTo(MetricConfiguration configuration) throws Exception { for (JmxRule rule : rules) { MetricDef metricDef = rule.buildMetricDef(); configuration.addMetricDef(metricDef); diff --git a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/yaml/MetricStructure.java b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/yaml/MetricStructure.java index 8e4044db4a48..026a09a285a9 100644 --- a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/yaml/MetricStructure.java +++ b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/yaml/MetricStructure.java @@ -6,6 +6,7 @@ package io.opentelemetry.instrumentation.jmx.yaml; import io.opentelemetry.instrumentation.jmx.engine.MetricAttribute; +import io.opentelemetry.instrumentation.jmx.engine.MetricAttributeExtractor; import io.opentelemetry.instrumentation.jmx.engine.MetricBanner; import java.util.ArrayList; import java.util.List; @@ -120,17 +121,17 @@ private static MetricAttribute buildMetricAttribute(String key, String target) { if (target.startsWith("param(")) { if (k > 0) { return new MetricAttribute( - key, MetricAttribute.fromObjectNameParameter(target.substring(6, k).trim())); + key, MetricAttributeExtractor.fromObjectNameParameter(target.substring(6, k).trim())); } } else if (target.startsWith("beanattr(")) { if (k > 0) { return new MetricAttribute( - key, MetricAttribute.fromBeanAttribute(target.substring(9, k).trim())); + key, MetricAttributeExtractor.fromBeanAttribute(target.substring(9, k).trim())); } } else if (target.startsWith("const(")) { if (k > 0) { return new MetricAttribute( - key, MetricAttribute.fromConstant(target.substring(6, k).trim())); + key, MetricAttributeExtractor.fromConstant(target.substring(6, k).trim())); } } diff --git a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/yaml/RuleParser.java b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/yaml/RuleParser.java index d200db1a9537..c5080db7b500 100644 --- a/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/yaml/RuleParser.java +++ b/instrumentation/jmx-metrics/library/src/main/java/io/opentelemetry/instrumentation/jmx/yaml/RuleParser.java @@ -5,7 +5,7 @@ package io.opentelemetry.instrumentation.jmx.yaml; -import static java.util.logging.Level.CONFIG; +import static java.util.logging.Level.INFO; import static java.util.logging.Level.WARNING; import io.opentelemetry.instrumentation.jmx.engine.MetricConfiguration; @@ -52,13 +52,13 @@ public JmxConfig loadConfig(InputStream is) throws Exception { * @param conf the metric configuration * @param is the InputStream with the YAML rules */ - public void addMetricDefs(MetricConfiguration conf, InputStream is) { + public void addMetricDefsTo(MetricConfiguration conf, InputStream is) { try { JmxConfig config = loadConfig(is); if (config != null) { - logger.log(CONFIG, "Found {0} metric rules", config.getRules().size()); - config.addMetricDefs(conf); + logger.log(INFO, "Found {0} metric rules", config.getRules().size()); + config.addMetricDefsTo(conf); } } catch (Exception exception) { logger.log(WARNING, "Failed to parse YAML rules: " + rootCause(exception));